6 references to EncodeToUtf8AndTransform
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (3)
1334
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(text, token, mapping);
1447
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(text, token, mapping);
1527
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(text, token, mapping);
Model\CodeGenTokenizer.cs (3)
462
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(textSpan, token, mapping);
1145
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(textSpan, token, mapping);
1209
int encodedLength = Helpers.
EncodeToUtf8AndTransform
(textSpan, token, mapping);