1 write to EndOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
29EndOfSentenceToken = modelProto.TrainerSpec.EosPiece ?? "</s>";
7 references to EndOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\SentencePieceBaseModel.cs (2)
348token = model.EndOfSentenceToken; 586specialToken = model.EndOfSentenceToken;
Model\SentencePieceBpeModel.cs (2)
138tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 203tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length)));
Model\SentencePieceTokenizer.cs (1)
76public string EndOfSentenceToken => _model.EndOfSentenceToken;
Model\SentencePieceUnigramModel.cs (2)
253tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(progressOffset, progressOffset))); 281tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(progressOffset, progressOffset)));