2 references to WordOrNonWordRegex
Microsoft.ML.Tokenizers (2)
PreTokenizer\PreTokenizer.cs (2)
92
return _wordOrNonWordPreTokenizer ??= new RegexPreTokenizer(
WordOrNonWordRegex
(), null);
95
return new RegexPreTokenizer(
WordOrNonWordRegex
(), specialTokensEncoder);