2 references to EncodeToIdsFromEndResult
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
722
return
EncodeToIdsFromEndResult
(value, accumulatedIds: null, maxTokens, text.Length, out textIndex);
752
return
EncodeToIdsFromEndResult
(encodedTokens, accumulatedIds: null, maxTokens, text.Length, out textIndex);