2 references to EncodeToIdsFromEndInternal
Microsoft.ML.Tokenizers (2)
Model\EnglishRobertaTokenizer.cs (2)
561
tokenCount +=
EncodeToIdsFromEndInternal
(textSpanToEncode.Slice(split.Offset, split.Length), null, out int textIndex, maxTokenCount - tokenCount);
570
tokenCount =
EncodeToIdsFromEndInternal
(textSpanToEncode, null, out int charsConsumed, maxTokenCount);