15 references to CreateForModel
Microsoft.Extensions.AI.Integration.Tests (1)
ReducingChatClientTests.cs (1)
22
private static readonly Tokenizer _gpt4oTokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4o");
Microsoft.ML.Tokenizers.Data.Tests (1)
TokenizerDataTests.cs (1)
30
var exception = Record.Exception(() => TiktokenTokenizer.
CreateForModel
(modelName));
Microsoft.ML.Tokenizers.Tests (13)
TiktokenTests.cs (13)
32
public static Tokenizer GPT4 { get; } = TiktokenTokenizer.
CreateForModel
("gpt-4", _specialTokens);
33
public static Tokenizer GPT2 { get; } = TiktokenTokenizer.
CreateForModel
("gpt2");
34
public static Tokenizer P50kBase { get; } = TiktokenTokenizer.
CreateForModel
("text-davinci-003");
35
public static Tokenizer R50kBase { get; } = TiktokenTokenizer.
CreateForModel
("ada");
36
public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.
CreateForModel
("text-davinci-edit-001");
37
public static Tokenizer GPT4o { get; } = TiktokenTokenizer.
CreateForModel
("gpt-4o");
38
public static Tokenizer GPT5 { get; } = TiktokenTokenizer.
CreateForModel
("gpt-5");
39
public static Tokenizer Phi4 { get; } = TiktokenTokenizer.
CreateForModel
("phi-4");
40
public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.
CreateForModel
("gpt-oss-20b");
94
tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4");
477
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
(modelName);
506
Tokenizer tokenizer1 = TiktokenTokenizer.
CreateForModel
(modelName);
553
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
(name);