1 reference to P50kBaseRegexPattern
Microsoft.ML.Tokenizers (1)
Model\TiktokenTokenizer.cs (1)
1180
internal static Regex P50kBaseRegex() => _p50kBaseRegex ??= new Regex(
P50kBaseRegexPattern
, RegexOptions.Compiled);