1 write to _specialTokensRegex
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
89_specialTokensRegex = new Regex(string.Join("|", specialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
12 references to _specialTokensRegex
Microsoft.ML.Tokenizers (12)
Model\SentencePieceTokenizer.cs (12)
257if (_specialTokensRegex is not null) 271Debug.Assert(_specialTokensRegex is not null); 280foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, _specialTokensRegex!)) 580if (_specialTokensRegex is not null) 594Debug.Assert(_specialTokensRegex is not null); 610foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, _specialTokensRegex!)) 921return _specialTokensRegex is not null ? 928Debug.Assert(_specialTokensRegex is not null); 943foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, _specialTokensRegex!)) 1275if (_specialTokensRegex is not null) 1289Debug.Assert(_specialTokensRegex is not null); 1301(int Offset, int Length)[] splits = PreTokenizer.SplitText(text, _specialTokensRegex!).ToArray();