1 write to EndOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
340EndOfSentenceToken = endOfSentenceToken;
7 references to EndOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\BPETokenizer.cs (7)
474if (EndOfSentenceToken is not null) 476tokens.Add(new EncodedToken(EndOfSentenceId, EndOfSentenceToken, new Range(charsConsumed, charsConsumed))); 541if (EndOfSentenceToken is not null && ids.Count < maxTokenCount) 602if (EndOfSentenceToken is not null && count < maxTokenCount) 682if (EndOfSentenceToken is not null && count < maxTokenCount) 719tokenCount = EndOfSentenceToken is null ? 0 : 1; 733tokenCount = EncodeToIdsFromEndWithCache(textSpanToEncode, null, maxTokenCount - (EndOfSentenceToken is null ? 0 : 1), out int charsConsumed, ref priorityQueue);