6 references to GetUtf16LengthFromUtf8Bytes
Microsoft.ML.Tokenizers (6)
Model\SentencePieceUnigramModel.cs (5)
162
int expectedCount = Helpers.
GetUtf16LengthFromUtf8Bytes
(utf8Bytes);
899
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(utf8UnknownBytes);
1147
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan);
1184
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan.Slice(ids[i].UtfStartOffset, ids[i].Utf8Length));
1395
consumedCharacters += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan.Slice(node.StartsAt, length));
Utils\Helpers.cs (1)
237
Debug.Assert(utf16Chars.Length >=
GetUtf16LengthFromUtf8Bytes
(utf8Bytes));