6 references to EncodeToUtf8AndTransform
Microsoft.ML.Tokenizers (6)
Model\BpeTokenizer.cs (3)
1354
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(text, token, mapping);
1467
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(text, token, mapping);
1547
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(text, token, mapping);
Model\CodeGenTokenizer.cs (3)
462
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(textSpan, token, mapping);
1145
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(textSpan, token, mapping);
1209
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(textSpan, token, mapping);