1 write to _regex
Microsoft.ML.Tokenizers (1)
PreTokenizer\RegexPreTokenizer.cs (1)
34
_regex
= regex;
2 references to _regex
Microsoft.ML.Tokenizers (2)
PreTokenizer\RegexPreTokenizer.cs (2)
55
return SplitText(text,
_regex
, _specialTokensRegex);
106
return SplitText(buffer,
_regex
, _specialTokensRegex, text.Length);