1 write to AddBeginningOfSentence
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
143
AddBeginningOfSentence
= addBeginningOfSentence;
7 references to AddBeginningOfSentence
Microsoft.ML.Tokenizers (6)
Model\CodeGenTokenizer.cs (6)
176
if (
AddBeginningOfSentence
&& string.IsNullOrEmpty(BeginningOfSentenceToken))
286
=> EncodeToTokens(text, textSpan, AddPrefixSpace,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization);
490
Tokens = EncodeToIds(text, textSpan, AddPrefixSpace,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization,
670
=> CountTokens(text, textSpan, AddPrefixSpace,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization, out _, out _, settings.MaxTokenCount);
718
return LastIndexOf(text, textSpan, settings.MaxTokenCount, AddPrefixSpace,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderPreTokenization,
722
tokenCount = CountTokens(text, textSpan, AddPrefixSpace,
AddBeginningOfSentence
, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization, out normalizedString, out int charsConsumed, settings.MaxTokenCount);
Microsoft.ML.Tokenizers.Tests (1)
CodeGenTests.cs (1)
971
Assert.False(codeGenTokenizer.
AddBeginningOfSentence
);