1 reference to O200kBaseRegex
Microsoft.ML.Tokenizers (1)
Model\TiktokenTokenizer.cs (1)
1130
return (new Dictionary<string, int> { { EndOfText, 199999 }, { EndOfPrompt, 200018 } },
O200kBaseRegex
(), O200kBaseFile, Type.GetType(O200kBaseTypeName), O200kBasePackageName);