6 references to IsTrailByte
Microsoft.ML.Tokenizers.Tests (6)
src\Microsoft.ML.Tokenizers\Utils\Helpers.cs (6)
331
if (
IsTrailByte
(input[1]) && cp >= 0x0080 && IsValidCodepoint(cp))
340
if (
IsTrailByte
(input[1]) &&
IsTrailByte
(input[2]) && cp >= 0x0800 && IsValidCodepoint(cp))
349
if (
IsTrailByte
(input[1]) &&
IsTrailByte
(input[2]) &&
IsTrailByte
(input[3]) && cp >= 0x10000 && IsValidCodepoint(cp))