3 writes to EndOfSentenceId
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBaseModel.cs (1)
30
EndOfSentenceId
= Math.Max(0, modelProto.TrainerSpec.EosId);
Model\SentencePieceBpeModel.cs (1)
87
EndOfSentenceId
= endOfSentenceToken.Id;
Model\SentencePieceUnigramModel.cs (1)
143
EndOfSentenceId
= endOfSentenceToken;
11 references to EndOfSentenceId
Microsoft.ML.Tokenizers (11)
Model\SentencePieceBaseModel.cs (2)
407
else if (id == model.
EndOfSentenceId
)
645
else if (id == model.
EndOfSentenceId
)
Model\SentencePieceBpeModel.cs (4)
184
tokens.Add(new EncodedToken(
EndOfSentenceId
, EndOfSentenceToken, new Range(text.Length, text.Length)));
249
tokens.Add(new EncodedToken(
EndOfSentenceId
, EndOfSentenceToken, new Range(text.Length, text.Length)));
447
accumulatedIds.Add(
EndOfSentenceId
);
543
accumulatedIds.Add(
EndOfSentenceId
);
Model\SentencePieceTokenizer.cs (1)
101
public int EndOfSentenceId => _model.
EndOfSentenceId
;
Model\SentencePieceUnigramModel.cs (4)
125
_vocabReverse[
EndOfSentenceId
] = (EndOfSentenceToken, 0f, 0);
303
tokens.Add(new EncodedToken(
EndOfSentenceId
, EndOfSentenceToken, new Range(progressOffset, progressOffset)));
331
tokens.Add(new EncodedToken(
EndOfSentenceId
, EndOfSentenceToken, new Range(progressOffset, progressOffset)));
650
ids.Add(
EndOfSentenceId
);