4 references to EncodeToUtf8
Microsoft.ML.Tokenizers (4)
Model\TiktokenTokenizer.cs (4)
327
int encodedLength = Helpers.
EncodeToUtf8
(text, arrayPoolArray, indexMappingSpan);
444
int encodedLength = Helpers.
EncodeToUtf8
(text, arrayPoolArray, indexMappingSpan);
609
int encodedLength = Helpers.
EncodeToUtf8
(text, arrayPoolArray, indexMappingSpan);
735
int encodedLength = Helpers.
EncodeToUtf8
(text, arrayPoolArray, indexMappingSpan);