1 write to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
324BeginningOfSentenceToken = beginningOfSentenceToken;
7 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\BPETokenizer.cs (7)
450if (BeginningOfSentenceToken is not null) 452tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 510if (BeginningOfSentenceToken is not null) 576int count = BeginningOfSentenceToken is null ? 0 : 1; 658int count = BeginningOfSentenceToken is null ? 0 : 1; 724tokenCount += tokenCount < maxTokenCount && BeginningOfSentenceToken is not null ? 1 : 0; 729tokenCount += tokenCount < maxTokenCount && BeginningOfSentenceToken is not null ? 1 : 0;