2 references to EncodeToIdsFromEndWithCache
Microsoft.ML.Tokenizers (2)
Model\BPETokenizer.cs (2)
568
tokenCount +=
EncodeToIdsFromEndWithCache
(textSpanToEncode.Slice(split.Offset, split.Length), null, maxTokenCount - tokenCount, out int textIndex, ref priorityQueue);
577
tokenCount =
EncodeToIdsFromEndWithCache
(textSpanToEncode, null, maxTokenCount, out int charsConsumed, ref priorityQueue);