13 references to CreateForModel
Microsoft.Extensions.AI.Integration.Tests (1)
ReducingChatClientTests.cs (1)
23
private static readonly Tokenizer _gpt4oTokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4o");
Microsoft.ML.Tokenizers.Data.Tests (1)
TokenizerDataTests.cs (1)
30
var exception = Record.Exception(() => TiktokenTokenizer.
CreateForModel
(modelName));
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
32
public static Tokenizer GPT4 { get; } = TiktokenTokenizer.
CreateForModel
("gpt-4", _specialTokens);
33
public static Tokenizer GPT2 { get; } = TiktokenTokenizer.
CreateForModel
("gpt2");
34
public static Tokenizer P50kBase { get; } = TiktokenTokenizer.
CreateForModel
("text-davinci-003");
35
public static Tokenizer R50kBase { get; } = TiktokenTokenizer.
CreateForModel
("ada");
36
public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.
CreateForModel
("text-davinci-edit-001");
37
public static Tokenizer GPT4o { get; } = TiktokenTokenizer.
CreateForModel
("gpt-4o");
38
public static Tokenizer Phi4 { get; } = TiktokenTokenizer.
CreateForModel
("phi-4");
92
tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4");
455
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
(modelName);
482
Tokenizer tokenizer1 = TiktokenTokenizer.
CreateForModel
(modelName);
526
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
(name);