2 writes to BeginningOfSentenceId
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
28BeginningOfSentenceId = Math.Max(0, modelProto.TrainerSpec.BosId); 75BeginningOfSentenceId = bosId;
9 references to BeginningOfSentenceId
Microsoft.ML.Tokenizers (9)
Model\SentencePieceBaseModel.cs (2)
386if (id == model.BeginningOfSentenceId) 624if (id == model.BeginningOfSentenceId)
Model\SentencePieceBpeModel.cs (4)
111tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 158tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 367accumulatedIds.Add(BeginningOfSentenceId); 431accumulatedIds.Add(BeginningOfSentenceId);
Model\SentencePieceTokenizer.cs (1)
92public int BeginningOfSentenceId => _model.BeginningOfSentenceId;
Model\SentencePieceUnigramModel.cs (2)
98_prefixTokens = DefaultAffix(BeginningOfSentenceId, BeginningOfSentenceToken); 257_prefixTokens = prefixTokens is not null ? ToAffixArray(prefixTokens) : DefaultAffix(BeginningOfSentenceId, BeginningOfSentenceToken);