1 write to _specialTokensRegex
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
89
_specialTokensRegex
= new Regex(string.Join("|", specialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
12 references to _specialTokensRegex
Microsoft.ML.Tokenizers (12)
Model\SentencePieceTokenizer.cs (12)
257
if (
_specialTokensRegex
is not null)
271
Debug.Assert(
_specialTokensRegex
is not null);
280
foreach ((int Offset, int Length) in PreTokenizer.SplitText(text,
_specialTokensRegex
!))
580
if (
_specialTokensRegex
is not null)
594
Debug.Assert(
_specialTokensRegex
is not null);
610
foreach ((int Offset, int Length) in PreTokenizer.SplitText(text,
_specialTokensRegex
!))
921
return
_specialTokensRegex
is not null ?
928
Debug.Assert(
_specialTokensRegex
is not null);
943
foreach ((int Offset, int Length) in PreTokenizer.SplitText(text,
_specialTokensRegex
!))
1275
if (
_specialTokensRegex
is not null)
1289
Debug.Assert(
_specialTokensRegex
is not null);
1301
(int Offset, int Length)[] splits = PreTokenizer.SplitText(text,
_specialTokensRegex
!).ToArray();