5 references to GetUtf8Bytes
Microsoft.ML.Tokenizers (5)
Model\CodeGenTokenizer.cs (1)
1293
bytesIndex += Helpers.
GetUtf8Bytes
(removePrefixSpace ? addedToken.AsSpan().Slice(1) : addedToken.AsSpan(), bytes.AsSpan().Slice(bytesIndex));
Model\SentencePieceBpeTokenizer.cs (4)
400
int bytesWritten = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
771
int bytesWritten = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
1090
int encodedCount = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
1470
int encodedCount = Helpers.
GetUtf8Bytes
(text.Slice(0, i + 1), utf8Bytes);