1 write to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
27BeginningOfSentenceToken = modelProto.TrainerSpec.BosPiece ?? "<s>";
7 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\SentencePieceBaseModel.cs (2)
344token = model.BeginningOfSentenceToken; 582specialToken = model.BeginningOfSentenceToken;
Model\SentencePieceBpeModel.cs (2)
111tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 158tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0)));
Model\SentencePieceTokenizer.cs (1)
71public string BeginningOfSentenceToken => _model.BeginningOfSentenceToken;
Model\SentencePieceUnigramModel.cs (2)
221tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 271tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0)));