1 write to EndOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
335EndOfSentenceToken = endOfSentenceToken;
7 references to EndOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\BPETokenizer.cs (7)
469if (EndOfSentenceToken is not null) 471tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(charsConsumed, charsConsumed))); 536if (EndOfSentenceToken is not null && ids.Count < maxTokenCount) 597if (EndOfSentenceToken is not null && count < maxTokenCount) 677if (EndOfSentenceToken is not null && count < maxTokenCount) 714tokenCount = EndOfSentenceToken is null ? 0 : 1; 728tokenCount = EncodeToIdsFromEndWithCache(textSpanToEncode, null, maxTokenCount - (EndOfSentenceToken is null ? 0 : 1), out int charsConsumed, ref priorityQueue);