3 references to ConvertUtf8ToUtf16
Microsoft.ML.Tokenizers (3)
Model\CodeGenTokenizer.cs (1)
1480
if (!Helpers.
ConvertUtf8ToUtf16
(current, vocabBuffer.Slice(utf16CharsInBuffer), out int utf8BytesConsumed, out int utf16CharsWritten))
Model\TiktokenTokenizer.cs (2)
912
if (!Helpers.
ConvertUtf8ToUtf16
(tokenBytes.Span, tempBuffer, out int bytesConsumed, out incompleteCharsWritten))
945
if (!Helpers.
ConvertUtf8ToUtf16
(utf8Bytes.Slice(utf8BytesIncompleteIndex, utf8BytesIncompleteCount + tokenBytes.Length), tempBuffer, out int bytesConsumed, out int charsConsumed))