2 references to WhiteSpaceOrPunctuationRegex
Microsoft.ML.Tokenizers (2)
PreTokenizer\PreTokenizer.cs (2)
65
return _whiteSpaceOrPunctuationPreTokenizer ??= new RegexPreTokenizer(
WhiteSpaceOrPunctuationRegex
(), null);
68
return new RegexPreTokenizer(
WhiteSpaceOrPunctuationRegex
(), specialTokensEncoder);