2 references to EncodeInternal
Microsoft.ML.Tokenizers (2)
Model\EnglishRobertaTokenizer.cs (2)
337
foreach (EncodedToken t in
EncodeInternal
(textSpanToEncode.Slice(split.Offset, split.Length)))
347
return new EncodeResults<EncodedToken> { Tokens =
EncodeInternal
(textSpanToEncode), NormalizedText = normalizedText, CharsConsumed = charsConsumed };