6 references to IsTrailByte
Microsoft.ML.Tokenizers.Tests (6)
src\Microsoft.ML.Tokenizers\Utils\Helpers.cs (6)
309
if (
IsTrailByte
(input[1]) && cp >= 0x0080 && IsValidCodepoint(cp))
318
if (
IsTrailByte
(input[1]) &&
IsTrailByte
(input[2]) && cp >= 0x0800 && IsValidCodepoint(cp))
327
if (
IsTrailByte
(input[1]) &&
IsTrailByte
(input[2]) &&
IsTrailByte
(input[3]) && cp >= 0x10000 && IsValidCodepoint(cp))