2 references to GetModelEncoding
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
1116
private static (Dictionary<string, int> SpecialTokens, Regex Regex, string VocabFile, Type? DataType, string PackageName) GetTiktokenConfigurations(string modelName) => GetTiktokenConfigurations(
GetModelEncoding
(modelName), modelName);
1434
=> CreateForModel(
GetModelEncoding
(modelName), modelName, extraSpecialTokens, normalizer);