6 references to GetUtf16LengthFromUtf8Bytes
Microsoft.ML.Tokenizers (6)
Model\SentencePieceUnigramModel.cs (5)
212
int expectedCount = Helpers.
GetUtf16LengthFromUtf8Bytes
(utf8Bytes);
949
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(utf8UnknownBytes);
1197
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan);
1234
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan.Slice(ids[i].UtfStartOffset, ids[i].Utf8Length));
1445
consumedCharacters += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan.Slice(node.StartsAt, length));
Utils\Helpers.cs (1)
215
Debug.Assert(utf16Chars.Length >=
GetUtf16LengthFromUtf8Bytes
(utf8Bytes));