3 references to IsValidCodepoint
Microsoft.ML.Tokenizers.Tests (3)
src\Microsoft.ML.Tokenizers\Utils\Helpers.cs (3)
309
if (IsTrailByte(input[1]) && cp >= 0x0080 &&
IsValidCodepoint
(cp))
318
if (IsTrailByte(input[1]) && IsTrailByte(input[2]) && cp >= 0x0800 &&
IsValidCodepoint
(cp))
327
if (IsTrailByte(input[1]) && IsTrailByte(input[2]) && IsTrailByte(input[3]) && cp >= 0x10000 &&
IsValidCodepoint
(cp))