1 write to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
329BeginningOfSentenceToken = beginningOfSentenceToken;
7 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\BPETokenizer.cs (7)
455if (BeginningOfSentenceToken is not null) 457tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 515if (BeginningOfSentenceToken is not null) 581int count = BeginningOfSentenceToken is null ? 0 : 1; 663int count = BeginningOfSentenceToken is null ? 0 : 1; 729tokenCount += tokenCount < maxTokenCount && BeginningOfSentenceToken is not null ? 1 : 0; 734tokenCount += tokenCount < maxTokenCount && BeginningOfSentenceToken is not null ? 1 : 0;