9 references to GetUtf8Bytes
Microsoft.ML.Tokenizers (9)
Model\CodeGenTokenizer.cs (1)
1298
bytesIndex += Helpers.
GetUtf8Bytes
(removePrefixSpace ? specialToken.AsSpan().Slice(1) : specialToken.AsSpan(), bytes.AsSpan().Slice(bytesIndex));
Model\SentencePieceBpeModel.cs (4)
236
int bytesWritten = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
536
int bytesWritten = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
799
int encodedCount = Helpers.
GetUtf8Bytes
(text.Slice(i), utf8Bytes);
1075
int encodedCount = Helpers.
GetUtf8Bytes
(text.Slice(0, i + 1), utf8Bytes);
Model\SentencePieceUnigramModel.cs (1)
299
int byteCount = Helpers.
GetUtf8Bytes
(text, byteSpan);
Utils\DoubleArrayTrie.cs (1)
758
encodingLength = Helpers.
GetUtf8Bytes
(pair.Key.AsSpan(), bytes);
Utils\OrdinalUtf8StringComparer.cs (2)
73
xLen = Helpers.
GetUtf8Bytes
(x.AsSpan(i), buffer1);
74
yLen = Helpers.
GetUtf8Bytes
(y.AsSpan(i), buffer2);