2 writes to EndOfSentenceToken
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
29EndOfSentenceToken = modelProto.TrainerSpec.EosPiece ?? "</s>"; 76EndOfSentenceToken = eosToken;
7 references to EndOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\SentencePieceBaseModel.cs (2)
392token = model.EndOfSentenceToken; 630specialToken = model.EndOfSentenceToken;
Model\SentencePieceBpeModel.cs (2)
138tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 203tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length)));
Model\SentencePieceTokenizer.cs (1)
82public string EndOfSentenceToken => _model.EndOfSentenceToken;
Model\SentencePieceUnigramModel.cs (2)
99_suffixTokens = DefaultAffix(EndOfSentenceId, EndOfSentenceToken); 258_suffixTokens = suffixTokens is not null ? ToAffixArray(suffixTokens) : DefaultAffix(EndOfSentenceId, EndOfSentenceToken);