2 references to CountTokensFromEnd
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
687
tokenCount +=
CountTokensFromEnd
(textSpanToEncode.Slice(split.Offset, split.Length), out int textIndex, maxTokenCount - tokenCount);
698
tokenCount =
CountTokensFromEnd
(textSpanToEncode, out int charsConsumed, maxTokenCount);