1 write to BeginningOfSentenceId
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
28BeginningOfSentenceId = Math.Max(0, modelProto.TrainerSpec.BosId);
10 references to BeginningOfSentenceId
Microsoft.ML.Tokenizers (10)
Model\SentencePieceBaseModel.cs (2)
342if (id == model.BeginningOfSentenceId) 580if (id == model.BeginningOfSentenceId)
Model\SentencePieceBpeModel.cs (4)
111tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 158tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 367accumulatedIds.Add(BeginningOfSentenceId); 431accumulatedIds.Add(BeginningOfSentenceId);
Model\SentencePieceTokenizer.cs (1)
86public int BeginningOfSentenceId => _model.BeginningOfSentenceId;
Model\SentencePieceUnigramModel.cs (3)
221tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 271tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 574ids.Add(BeginningOfSentenceId);