1 write to AddEndOfSentence
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
43
AddEndOfSentence
= addEos;
5 references to AddEndOfSentence
Microsoft.ML.Tokenizers (5)
Model\SentencePieceTokenizer.cs (5)
199
Tokens = EncodeToTokens(text, textSpan, out string? normalizedString, AddBeginningOfSentence,
AddEndOfSentence
, settings.ConsiderPreTokenization, settings.ConsiderNormalization),
456
Tokens = EncodeToIds(text, textSpan, AddBeginningOfSentence,
AddEndOfSentence
, settings.ConsiderNormalization, out string? normalizedString, out int charsConsumed, settings.MaxTokenCount),
838
return CountTokens(text, textSpan, AddBeginningOfSentence,
AddEndOfSentence
, settings.ConsiderPreTokenization, settings.ConsiderNormalization, out _, out _, settings.MaxTokenCount);
1166
tokenCount = CountTokens(text, textSpan, AddBeginningOfSentence,
AddEndOfSentence
, settings.ConsiderPreTokenization, settings.ConsiderNormalization, out normalizedString, out int charsConsumed, settings.MaxTokenCount);
1215
=> GetIndexByTokenCountFromEnd(text is null ? textSpan : text.AsSpan(), AddBeginningOfSentence,
AddEndOfSentence
, maxTokenCount, considerNormalization, out normalizedString, out tokenCount);