2 references to CountTokens
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
560
count +=
CountTokens
(textSpanToEncode.Slice(split.Offset, split.Length), out int length, maxTokenCount - count);
571
count =
CountTokens
(textSpanToEncode, out charsConsumed, maxTokenCount);