2 writes to SpecialTokensRegex
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
51SpecialTokensRegex = new Regex(string.Join("|", specialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled); 97SpecialTokensRegex = new Regex(string.Join("|", specialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
24 references to SpecialTokensRegex
Microsoft.ML.Tokenizers (24)
Model\SentencePieceBpeModel.cs (12)
93if (SpecialTokensRegex is not null) 107Debug.Assert(SpecialTokensRegex is not null); 116foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, SpecialTokensRegex!)) 345if (SpecialTokensRegex is not null) 359Debug.Assert(SpecialTokensRegex is not null); 375foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, SpecialTokensRegex!)) 630return SpecialTokensRegex is not null ? 637Debug.Assert(SpecialTokensRegex is not null); 652foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, SpecialTokensRegex!)) 881if (SpecialTokensRegex is not null) 895Debug.Assert(SpecialTokensRegex is not null); 907(int Offset, int Length)[] splits = PreTokenizer.SplitText(text, SpecialTokensRegex!).ToArray();
Model\SentencePieceUnigramModel.cs (12)
392if (SpecialTokensRegex is not null) 492Debug.Assert(SpecialTokensRegex is not null); 503foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, SpecialTokensRegex!)) 867if (SpecialTokensRegex is not null) 928Debug.Assert(SpecialTokensRegex is not null); 937foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, SpecialTokensRegex!)) 1267if (SpecialTokensRegex is not null) 1309Debug.Assert(SpecialTokensRegex is not null); 1318foreach ((int Offset, int Length) in PreTokenizer.SplitText(text, SpecialTokensRegex!)) 1546if (SpecialTokensRegex is not null) 1583Debug.Assert(SpecialTokensRegex is not null); 1589(int Offset, int Length)[] splits = PreTokenizer.SplitText(text, SpecialTokensRegex!).ToArray();