1 write to EndOfSentenceId
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
30EndOfSentenceId = Math.Max(0, modelProto.TrainerSpec.EosId);
10 references to EndOfSentenceId
Microsoft.ML.Tokenizers (10)
Model\SentencePieceBaseModel.cs (2)
346else if (id == model.EndOfSentenceId) 584else if (id == model.EndOfSentenceId)
Model\SentencePieceBpeModel.cs (4)
138tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 203tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 401accumulatedIds.Add(EndOfSentenceId); 497accumulatedIds.Add(EndOfSentenceId);
Model\SentencePieceTokenizer.cs (1)
91public int EndOfSentenceId => _model.EndOfSentenceId;
Model\SentencePieceUnigramModel.cs (3)
253tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(progressOffset, progressOffset))); 281tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(progressOffset, progressOffset))); 600ids.Add(EndOfSentenceId);