2 writes to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
27BeginningOfSentenceToken = modelProto.TrainerSpec.BosPiece ?? "<s>"; 91BeginningOfSentenceToken = options.BeginningOfSentenceToken;
8 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (8)
Model\SentencePieceBaseModel.cs (2)
405token = model.BeginningOfSentenceToken; 643specialToken = model.BeginningOfSentenceToken;
Model\SentencePieceBpeModel.cs (2)
157tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 204tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0)));
Model\SentencePieceTokenizer.cs (1)
81public string BeginningOfSentenceToken => _model.BeginningOfSentenceToken;
Model\SentencePieceUnigramModel.cs (3)
124_vocabReverse[BeginningOfSentenceId] = (BeginningOfSentenceToken, 0f, 0); 271tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 321tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0)));