6 references to GetUtf16LengthFromUtf8Bytes
Microsoft.ML.Tokenizers (6)
Model\SentencePieceUnigramModel.cs (5)
437
int expectedCount = Helpers.
GetUtf16LengthFromUtf8Bytes
(utf8Bytes);
1185
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(utf8UnknownBytes);
1443
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan);
1480
charsConsumed += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan.Slice(ids[i].UtfStartOffset, ids[i].Utf8Length));
1701
consumedCharacters += Helpers.
GetUtf16LengthFromUtf8Bytes
(normalizationSpan.Slice(node.StartsAt, length));
Utils\Helpers.cs (1)
237
Debug.Assert(utf16Chars.Length >=
GetUtf16LengthFromUtf8Bytes
(utf8Bytes));