5 references to GetUtf8Bytes
Microsoft.ML.Tokenizers (5)
Model\CodeGenTokenizer.cs (1)
1295
bytesIndex += Helpers.
GetUtf8Bytes
(removePrefixSpace ? specialToken.AsSpan().Slice(1) : specialToken.AsSpan(), bytes.AsSpan().Slice(bytesIndex));
Model\SentencePieceTokenizer.cs (4)
401
int bytesWritten = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
772
int bytesWritten = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
1091
int encodedCount = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
1471
int encodedCount = Helpers.
GetUtf8Bytes
(text.Slice(0, i + 1), utf8Bytes);