2 references to WordOrNonWordRegex
Microsoft.ML.Tokenizers (2)
PreTokenizer\PreTokenizer.cs (2)
98
return _wordOrNonWordPreTokenizer ??= new RegexPreTokenizer(
WordOrNonWordRegex
(), null);
101
return new RegexPreTokenizer(
WordOrNonWordRegex
(), specialTokens);