3 writes to BeginningOfSentenceId
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBaseModel.cs (1)
28BeginningOfSentenceId = Math.Max(0, modelProto.TrainerSpec.BosId);
Model\SentencePieceBpeModel.cs (1)
81BeginningOfSentenceId = beginOfSentenceToken.Id;
Model\SentencePieceUnigramModel.cs (1)
137BeginningOfSentenceId = beginOfSentenceToken;
11 references to BeginningOfSentenceId
Microsoft.ML.Tokenizers (11)
Model\SentencePieceBaseModel.cs (2)
403if (id == model.BeginningOfSentenceId) 641if (id == model.BeginningOfSentenceId)
Model\SentencePieceBpeModel.cs (4)
157tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 204tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 413accumulatedIds.Add(BeginningOfSentenceId); 477accumulatedIds.Add(BeginningOfSentenceId);
Model\SentencePieceTokenizer.cs (1)
96public int BeginningOfSentenceId => _model.BeginningOfSentenceId;
Model\SentencePieceUnigramModel.cs (4)
124_vocabReverse[BeginningOfSentenceId] = (BeginningOfSentenceToken, 0f, 0); 271tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 321tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 624ids.Add(BeginningOfSentenceId);