1 reference to Cl100kBaseRegexPattern
Microsoft.ML.Tokenizers (1)
Model\TiktokenTokenizer.cs (1)
1188
private static Regex Cl100kBaseRegex() => _cl100kBaseRegex ??= new Regex(
Cl100kBaseRegexPattern
, RegexOptions.Compiled, TimeSpan.FromMilliseconds(PreTokenizer.DefaultTimeOutInMilliseconds));