4 references to EncodeToIdsInternal
Microsoft.ML.Tokenizers (4)
Model\CodeGenTokenizer.cs (4)
632EncodeToIdsInternal(null, textSpanToEncode.Slice(split.Offset, split.Length), ids, agenda, out int length, maxTokenCount - ids.Count);
643EncodeToIdsInternal(addPrefixSpace ? null : (normalizedText ?? text), textSpanToEncode, ids, agenda, out charsConsumed, maxTokenCount - ids.Count);
844count += EncodeToIdsInternal(null, textSpanToEncode.Slice(split.Offset, split.Length), null, agenda, out int length, maxTokenCount - count);
855count = EncodeToIdsInternal(addPrefixSpace ? null : text, textSpanToEncode, null, agenda, out charsConsumed, maxTokenCount - count);