2 writes to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
27BeginningOfSentenceToken = modelProto.TrainerSpec.BosPiece ?? "<s>"; 74BeginningOfSentenceToken = bosToken;
7 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\SentencePieceBaseModel.cs (2)
388token = model.BeginningOfSentenceToken; 626specialToken = model.BeginningOfSentenceToken;
Model\SentencePieceBpeModel.cs (2)
111tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 158tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0)));
Model\SentencePieceTokenizer.cs (1)
77public string BeginningOfSentenceToken => _model.BeginningOfSentenceToken;
Model\SentencePieceUnigramModel.cs (2)
98_prefixTokens = DefaultAffix(BeginningOfSentenceId, BeginningOfSentenceToken); 257_prefixTokens = prefixTokens is not null ? ToAffixArray(prefixTokens) : DefaultAffix(BeginningOfSentenceId, BeginningOfSentenceToken);