28 references to CreateForModel
Microsoft.Extensions.AI.Integration.Tests (1)
ReducingChatClientTests.cs (1)
21
private static readonly Tokenizer _gpt4oTokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4o");
Microsoft.Extensions.DataIngestion.Tests (12)
Chunkers\ChunkerOptionsTests.cs (1)
12
private static readonly Tokenizer _tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4");
Chunkers\HeaderChunkerTests.cs (8)
39
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")));
71
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 13 });
96
HeaderChunker lessThanContext = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 5 });
99
HeaderChunker sameAsContext = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 6 });
119
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 30 });
136
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 37 });
146
HeaderChunker chunker = new(new(TiktokenTokenizer.
CreateForModel
("gpt-4")) { MaxTokensPerChunk = 100 });
175
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4");
Chunkers\SectionChunkerTests.cs (1)
16
var tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4o");
Chunkers\SemanticSimilarityChunkerTests.cs (1)
26
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4o");
IngestionPipelineTests.cs (1)
226
private static IngestionChunker<string> CreateChunker() => new HeaderChunker(new(TiktokenTokenizer.
CreateForModel
("gpt-4")));
Microsoft.ML.Tokenizers.Data.Tests (1)
TokenizerDataTests.cs (1)
30
var exception = Record.Exception(() => TiktokenTokenizer.
CreateForModel
(modelName));
Microsoft.ML.Tokenizers.Tests (14)
TiktokenTests.cs (14)
32
public static Tokenizer GPT4 { get; } = TiktokenTokenizer.
CreateForModel
("gpt-4", _specialTokens);
33
public static Tokenizer GPT2 { get; } = TiktokenTokenizer.
CreateForModel
("gpt2");
34
public static Tokenizer P50kBase { get; } = TiktokenTokenizer.
CreateForModel
("text-davinci-003");
35
public static Tokenizer R50kBase { get; } = TiktokenTokenizer.
CreateForModel
("ada");
36
public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.
CreateForModel
("text-davinci-edit-001");
37
public static Tokenizer GPT4o { get; } = TiktokenTokenizer.
CreateForModel
("gpt-4o");
38
public static Tokenizer GPT5 { get; } = TiktokenTokenizer.
CreateForModel
("gpt-5");
39
public static Tokenizer GPT5_1 { get; } = TiktokenTokenizer.
CreateForModel
("gpt-5.1");
40
public static Tokenizer Phi4 { get; } = TiktokenTokenizer.
CreateForModel
("phi-4");
41
public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.
CreateForModel
("gpt-oss-20b");
95
tokenizer = TiktokenTokenizer.
CreateForModel
("gpt-4");
480
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
(modelName);
509
Tokenizer tokenizer1 = TiktokenTokenizer.
CreateForModel
(modelName);
557
Tokenizer tokenizer = TiktokenTokenizer.
CreateForModel
(name);