2 references to EncodeInternal
Microsoft.ML.Tokenizers (2)
Model\EnglishRobertaTokenizer.cs (2)
326
foreach (EncodedToken t in
EncodeInternal
(textSpanToEncode.Slice(split.Offset, split.Length)))
336
return new EncodeResults<EncodedToken> { Tokens =
EncodeInternal
(textSpanToEncode), NormalizedText = normalizedString, CharsConsumed = charsConsumed };