3 references to IsValidCodepoint
Microsoft.ML.Tokenizers.Tests (3)
src\Microsoft.ML.Tokenizers\Utils\Helpers.cs (3)
331
if (IsTrailByte(input[1]) && cp >= 0x0080 &&
IsValidCodepoint
(cp))
340
if (IsTrailByte(input[1]) && IsTrailByte(input[2]) && cp >= 0x0800 &&
IsValidCodepoint
(cp))
349
if (IsTrailByte(input[1]) && IsTrailByte(input[2]) && IsTrailByte(input[3]) && cp >= 0x10000 &&
IsValidCodepoint
(cp))