24 references to CreateForModel
Microsoft.Extensions.AI.Integration.Tests (1)
ReducingChatClientTests.cs (1)
21
private static readonly Tokenizer _gpt4oTokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4o");
Microsoft.Extensions.DataIngestion.Tests (9)
Chunkers\ChunkerOptionsTests.cs (1)
12
private static readonly Tokenizer _tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4");
Chunkers\HeaderChunkerTests.cs (8)
38
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")));
70
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 13 });
95
HeaderChunker lessThanContext = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 5 });
98
HeaderChunker sameAsContext = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 6 });
118
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 30 });
135
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 37 });
145
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 100 });
174
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4");
Microsoft.ML.Tokenizers.Data.Tests (1)
TokenizerDataTests.cs (1)
30
var exception = Record.Exception(() => TiktokenTokenizer.
CreateForModel
(modelName));
Microsoft.ML.Tokenizers.Tests (13)
TiktokenTests.cs (13)
32
public static Tokenizer GPT4 { get; } = TiktokenTokenizer.
CreateForModel
("gpt-4", _specialTokens);
33
public static Tokenizer GPT2 { get; } = TiktokenTokenizer.
CreateForModel
("gpt2");
34
public static Tokenizer P50kBase { get; } = TiktokenTokenizer.
CreateForModel
("text-davinci-003");
35
public static Tokenizer R50kBase { get; } = TiktokenTokenizer.
CreateForModel
("ada");
36
public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.
CreateForModel
("text-davinci-edit-001");
37
public static Tokenizer GPT4o { get; } = TiktokenTokenizer.
CreateForModel
("gpt-4o");
38
public static Tokenizer GPT5 { get; } = TiktokenTokenizer.
CreateForModel
("gpt-5");
39
public static Tokenizer Phi4 { get; } = TiktokenTokenizer.
CreateForModel
("phi-4");
40
public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.
CreateForModel
("gpt-oss-20b");
94
tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4");
477
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
(modelName);
506
Tokenizer tokenizer1 = TiktokenTokenizer.
CreateForModel
(modelName);
553
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
(name);