1 reference to R50kBasePackageName
Microsoft.ML.Tokenizers (1)
Model\TiktokenTokenizer.cs (1)
1140
return (new Dictionary<string, int> { { EndOfText, 50256 } }, P50kBaseRegex(), R50RanksFile, Type.GetType(R50kBaseTypeName),
R50kBasePackageName
);