1 write to AddBeginningOfSentence
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
143AddBeginningOfSentence = addBeginningOfSentence;
7 references to AddBeginningOfSentence
Microsoft.ML.Tokenizers (6)
Model\CodeGenTokenizer.cs (6)
176if (AddBeginningOfSentence && string.IsNullOrEmpty(BeginningOfSentenceToken)) 286=> EncodeToTokens(text, textSpan, AddPrefixSpace, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization); 492Tokens = EncodeToIds(text, textSpan, AddPrefixSpace, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization, 672=> CountTokens(text, textSpan, AddPrefixSpace, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization, out _, out _, settings.MaxTokenCount); 720return LastIndexOf(text, textSpan, settings.MaxTokenCount, AddPrefixSpace, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderPreTokenization, 724tokenCount = CountTokens(text, textSpan, AddPrefixSpace, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization, out normalizedText, out int charsConsumed, settings.MaxTokenCount);
Microsoft.ML.Tokenizers.Tests (1)
CodeGenTests.cs (1)
971Assert.False(codeGenTokenizer.AddBeginningOfSentence);