2 references to CountTokensFromEnd
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
686
tokenCount +=
CountTokensFromEnd
(textSpanToEncode.Slice(split.Offset, split.Length), out int textIndex, maxTokenCount - tokenCount);
697
tokenCount =
CountTokensFromEnd
(textSpanToEncode, out int charsConsumed, maxTokenCount);