EncodeToIdsInternal

4 references to EncodeToIdsInternal

Microsoft.ML.Tokenizers (4)

Model\EnglishRobertaTokenizer.cs (4)

430EncodeToIdsInternal(textSpanToEncode.Slice(split.Offset, split.Length), ids, out int length, maxTokenCount - ids.Count); 441EncodeToIdsInternal(textSpanToEncode, ids, out textLength, maxTokenCount); 514count += EncodeToIdsInternal(textSpanToEncode.Slice(split.Offset, split.Length), null, out int length, maxTokenCount - count); 525count += EncodeToIdsInternal(textSpanToEncode, null, out charsConsumed, maxTokenCount);