1 reference to O200kBaseFile
Microsoft.ML.Tokenizers (1)
Model\TiktokenTokenizer.cs (1)
1148
return (new Dictionary<string, int> { { EndOfText, 199999 }, { EndOfPrompt, 200018 } }, O200kBaseRegex(),
O200kBaseFile
, Type.GetType(O200kBaseTypeName), O200kBasePackageName);