11 references to CreateForModel
Microsoft.ML.Tokenizers.Data.Tests (1)
TokenizerDataTests.cs (1)
30var exception = Record.Exception(() => TiktokenTokenizer.CreateForModel(modelName));
Microsoft.ML.Tokenizers.Tests (10)
TiktokenTests.cs (10)
31public static Tokenizer GPT4 { get; } = TiktokenTokenizer.CreateForModel("gpt-4", _specialTokens); 32public static Tokenizer GPT2 { get; } = TiktokenTokenizer.CreateForModel("gpt2"); 33public static Tokenizer P50kBase { get; } = TiktokenTokenizer.CreateForModel("text-davinci-003"); 34public static Tokenizer R50kBase { get; } = TiktokenTokenizer.CreateForModel("ada"); 35public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.CreateForModel("text-davinci-edit-001"); 36public static Tokenizer GPT4o { get; } = TiktokenTokenizer.CreateForModel("gpt-4o"); 89tokenizer = TiktokenTokenizer.CreateForModel("gpt-4"); 444Tokenizer tokenizer = TiktokenTokenizer.CreateForModel(modelName); 471Tokenizer tokenizer1 = TiktokenTokenizer.CreateForModel(modelName); 511Tokenizer tokenizer = TiktokenTokenizer.CreateForModel(name);