1 reference to P50kBaseRegexPattern
Microsoft.ML.Tokenizers (1)
Model\TiktokenTokenizer.cs (1)
1206
internal static Regex P50kBaseRegex() => _p50kBaseRegex ??= new Regex(
P50kBaseRegexPattern
, RegexOptions.Compiled, TimeSpan.FromMilliseconds(PreTokenizer.DefaultTimeOutInMilliseconds));