EncodeToIdsFromEndInternal

2 references to EncodeToIdsFromEndInternal

Microsoft.ML.Tokenizers (2)

Model\CodeGenTokenizer.cs (2)

983tokenCount += EncodeToIdsFromEndInternal(null, textSpanToEncode.Slice(split.Offset, split.Length), null, agenda, out int textIndex, maxTokenCount - tokenCount); 993tokenCount = EncodeToIdsFromEndInternal(addPrefixSpace ? null : text, textSpanToEncode, null, agenda, out int charsConsumed, maxTokenCount - tokenCount);