2 writes to EndOfSentenceToken
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
29EndOfSentenceToken = modelProto.TrainerSpec.EosPiece ?? "</s>"; 92EndOfSentenceToken = options.EndOfSentenceToken;
8 references to EndOfSentenceToken
Microsoft.ML.Tokenizers (8)
Model\SentencePieceBaseModel.cs (2)
409token = model.EndOfSentenceToken; 647specialToken = model.EndOfSentenceToken;
Model\SentencePieceBpeModel.cs (2)
184tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 249tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length)));
Model\SentencePieceTokenizer.cs (1)
86public string EndOfSentenceToken => _model.EndOfSentenceToken;
Model\SentencePieceUnigramModel.cs (3)
125_vocabReverse[EndOfSentenceId] = (EndOfSentenceToken, 0f, 0); 303tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(progressOffset, progressOffset))); 331tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(progressOffset, progressOffset)));