1 write to AddBeginningOfSentence
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
42
AddBeginningOfSentence
= addBos;
5 references to AddBeginningOfSentence
Microsoft.ML.Tokenizers (5)
Model\SentencePieceTokenizer.cs (5)
199
Tokens = EncodeToTokens(text, textSpan, out string? normalizedString,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization),
456
Tokens = EncodeToIds(text, textSpan,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderNormalization, out string? normalizedString, out int charsConsumed, settings.MaxTokenCount),
838
return CountTokens(text, textSpan,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization, out _, out _, settings.MaxTokenCount);
1166
tokenCount = CountTokens(text, textSpan,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization, out normalizedString, out int charsConsumed, settings.MaxTokenCount);
1215
=> GetIndexByTokenCountFromEnd(text is null ? textSpan : text.AsSpan(),
AddBeginningOfSentence
, AddEndOfSentence, maxTokenCount, considerNormalization, out normalizedString, out tokenCount);