2 writes to EndOfSentenceId
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
30EndOfSentenceId = Math.Max(0, modelProto.TrainerSpec.EosId); 77EndOfSentenceId = eosId;
9 references to EndOfSentenceId
Microsoft.ML.Tokenizers (9)
Model\SentencePieceBaseModel.cs (2)
390else if (id == model.EndOfSentenceId) 628else if (id == model.EndOfSentenceId)
Model\SentencePieceBpeModel.cs (4)
138tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 203tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(text.Length, text.Length))); 401accumulatedIds.Add(EndOfSentenceId); 497accumulatedIds.Add(EndOfSentenceId);
Model\SentencePieceTokenizer.cs (1)
97public int EndOfSentenceId => _model.EndOfSentenceId;
Model\SentencePieceUnigramModel.cs (2)
99_suffixTokens = DefaultAffix(EndOfSentenceId, EndOfSentenceToken); 258_suffixTokens = suffixTokens is not null ? ToAffixArray(suffixTokens) : DefaultAffix(EndOfSentenceId, EndOfSentenceToken);