14 references to RegexPreTokenizer
Microsoft.ML.GenAI.LLaMA (1)
LlamaTokenizerHelper.cs (1)
52
var preTokenizer = new
RegexPreTokenizer
(new Regex(_re), _specialTokens);
Microsoft.ML.Tokenizers (12)
Model\CodeGenTokenizer.cs (1)
1879
new
RegexPreTokenizer
(TiktokenTokenizer.P50kBaseRegex(), CodeGenTokenizer.CodeGenSpecialTokens),
Model\Phi2Tokenizer.cs (1)
117
vocabStream, mergesStream, new
RegexPreTokenizer
(TiktokenTokenizer.P50kBaseRegex(), CodeGenTokenizer.CodeGenSpecialTokens), normalizer: null,
Model\TiktokenTokenizer.cs (3)
1255
new
RegexPreTokenizer
(tiktokenConfiguration.Regex, tiktokenConfiguration.SpecialTokens),
1395
new
RegexPreTokenizer
(tiktokenConfiguration.Regex, tiktokenConfiguration.SpecialTokens),
1435
new
RegexPreTokenizer
(tiktokenConfiguration.Regex, tiktokenConfiguration.SpecialTokens),
PreTokenizer\CompositePreTokenizer.cs (1)
56
list.Add(new
RegexPreTokenizer
(new Regex(string.Join("|", specialTokens.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled), null));
PreTokenizer\PreTokenizer.cs (6)
68
return _whiteSpaceOrPunctuationPreTokenizer ??= new
RegexPreTokenizer
(WhiteSpaceOrPunctuationRegex(), null);
71
return new
RegexPreTokenizer
(WhiteSpaceOrPunctuationRegex(), specialTokens);
98
return _wordOrNonWordPreTokenizer ??= new
RegexPreTokenizer
(WordOrNonWordRegex(), null);
101
return new
RegexPreTokenizer
(WordOrNonWordRegex(), specialTokens);
127
return _whiteSpacePreTokenizer ??= new
RegexPreTokenizer
(WhiteSpaceRegex(), null);
130
return new
RegexPreTokenizer
(WhiteSpaceRegex(), specialTokens);
Microsoft.ML.Tokenizers.Tests (1)
BpeTests.cs (1)
984
preTokenizers.Add(new
RegexPreTokenizer
(new Regex(pattern, RegexOptions.Compiled, TimeSpan.FromMilliseconds(DefaultTimeOutInMilliseconds)), null));