1 write to _specialTokensRegex
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
90_specialTokensRegex = new Regex(string.Join("|", specialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
12 references to _specialTokensRegex
Microsoft.ML.Tokenizers (12)
Model\SentencePieceTokenizer.cs (12)
258if (_specialTokensRegex is not null) 272Debug.Assert(_specialTokensRegex is not null); 281foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, _specialTokensRegex!)) 581if (_specialTokensRegex is not null) 595Debug.Assert(_specialTokensRegex is not null); 611foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, _specialTokensRegex!)) 922return _specialTokensRegex is not null ? 929Debug.Assert(_specialTokensRegex is not null); 944foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, _specialTokensRegex!)) 1276if (_specialTokensRegex is not null) 1290Debug.Assert(_specialTokensRegex is not null); 1302(int Offset, int Length)[] splits = PreTokenizer.SplitText(text, _specialTokensRegex!).ToArray();