1332return new TiktokenTokenizer( 1360=> new TiktokenTokenizer(vocabFilePath, preTokenizer, specialTokens, normalizer, cacheSize); 1380=> new TiktokenTokenizer(vocabStream, preTokenizer, specialTokens, normalizer, cacheSize); 1411return new TiktokenTokenizer(encoder, decoder, vocab, preTokenizer, specialTokens, normalizer, cacheSize); 1475return new TiktokenTokenizer(vocabStream,

21private static readonly Tokenizer _gpt4oTokenizer = TiktokenTokenizer.CreateForModel("gpt-4o");

12private static readonly Tokenizer _tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");

39HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4"))); 71HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 13 }); 96HeaderChunker lessThanContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 5 }); 99HeaderChunker sameAsContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 6 }); 119HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 30 }); 136HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 37 }); 146HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 100 }); 175Tokenizer tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");

16var tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o");

17var tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o"); 25var tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o");

16var tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o");

26Tokenizer tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o");

226private static IngestionChunker<string> CreateChunker() => new HeaderChunker(new(TiktokenTokenizer.CreateForModel("gpt-4")));

1879new RegexPreTokenizer(TiktokenTokenizer.P50kBaseRegex(), CodeGenTokenizer.CodeGenSpecialTokens),

117vocabStream, mergesStream, new RegexPreTokenizer(TiktokenTokenizer.P50kBaseRegex(), CodeGenTokenizer.CodeGenSpecialTokens), normalizer: null,

1299private static TiktokenTokenizer CreateForModel( 1354public static TiktokenTokenizer Create( 1374public static TiktokenTokenizer Create( 1395public static async Task<TiktokenTokenizer> CreateAsync( 1427public static async Task<TiktokenTokenizer> CreateAsync( 1453public static TiktokenTokenizer CreateForModel( 1492public static async Task<TiktokenTokenizer> CreateForModelAsync( 1529public static TiktokenTokenizer CreateForModel(string modelName, IReadOnlyDictionary<string, int>? extraSpecialTokens = null, Normalizer? normalizer = null) 1539public static TiktokenTokenizer CreateForEncoding(string encodingName, IReadOnlyDictionary<string, int>? extraSpecialTokens = null, Normalizer? normalizer = null) 1573throw new ArgumentException($"The encoding name '{encodingName}' is not supported. The only supported encoding names are: {TiktokenTokenizer.Cl100kBaseEncodingName}, {TiktokenTokenizer.P50kBaseEncodingName}, {TiktokenTokenizer.P50kEditEncodingName}, and {TiktokenTokenizer.R50kBaseEncodingName}.", nameof(encodingName));

32return SplitText(text, TiktokenTokenizer.P50kBaseRegex()); 47return SplitText(text, TiktokenTokenizer.P50kBaseRegex());

30var exception = Record.Exception(() => TiktokenTokenizer.CreateForModel(modelName)); 53TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, preTokenizer: null, normalizer: null);

32public static Tokenizer GPT4 { get; } = TiktokenTokenizer.CreateForModel("gpt-4", _specialTokens); 33public static Tokenizer GPT2 { get; } = TiktokenTokenizer.CreateForModel("gpt2"); 34public static Tokenizer P50kBase { get; } = TiktokenTokenizer.CreateForModel("text-davinci-003"); 35public static Tokenizer R50kBase { get; } = TiktokenTokenizer.CreateForModel("ada"); 36public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.CreateForModel("text-davinci-edit-001"); 37public static Tokenizer GPT4o { get; } = TiktokenTokenizer.CreateForModel("gpt-4o"); 38public static Tokenizer GPT5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5"); 39public static Tokenizer GPT5_1 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.1"); 40public static Tokenizer GPT5_2 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.2"); 41public static Tokenizer GPT5_3 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.3"); 42public static Tokenizer GPT5_4 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.4"); 43public static Tokenizer GPT5_5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.5"); 44public static Tokenizer GPT5_6 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.6"); 45public static Tokenizer GPT6 { get; } = TiktokenTokenizer.CreateForModel("gpt-6"); 46public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4"); 47public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.CreateForModel("gpt-oss-20b"); 55Assert.True(GPT4 is TiktokenTokenizer); 56IReadOnlyDictionary<string, int>? specialTokens = (GPT4 as TiktokenTokenizer)!.SpecialTokens; 60string assemblyName = typeof(TiktokenTokenizer).Assembly.FullName!; 71Tokenizer tokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, GPT4.PreTokenizer, null, specialTokens); 76tokenizer = TiktokenTokenizer.Create(stream, GPT4.PreTokenizer, null, specialTokens); 80tokenizer = await TiktokenTokenizer.CreateAsync(tokenizerDataFileName, GPT4.PreTokenizer, normalizer: null, specialTokens); 85tokenizer = await TiktokenTokenizer.CreateAsync(stream, GPT4.PreTokenizer, normalizer: null, specialTokens); 91tokenizer = TiktokenTokenizer.CreateForModel("gpt-4", stream); 97tokenizer = await TiktokenTokenizer.CreateForModelAsync("gpt-3.5-turbo", stream); 101tokenizer = TiktokenTokenizer.CreateForModel("gpt-4"); 128TiktokenTokenizer tiktoken = (tokenizer as TiktokenTokenizer)!; 129TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, tokenizer.PreTokenizer, null, tiktoken.SpecialTokens); 153TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text); 170private void TestDecodingWithSpan(TiktokenTokenizer tokenizer, int[] ids, string expectedDecoded) 206TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 237TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text); 249TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 284TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 296foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5, GPT5_1, GPT5_2, GPT6 }) 349TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text); 368TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text); 387TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text); 406TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text); 498Tokenizer tokenizer = TiktokenTokenizer.CreateForModel(modelName); 499Assert.True(tokenizer is TiktokenTokenizer); 512Tokenizer tokenizer = TiktokenTokenizer.CreateForEncoding(encodingName); 513Assert.True(tokenizer is TiktokenTokenizer); 527Tokenizer tokenizer1 = TiktokenTokenizer.CreateForModel(modelName); 529Assert.True(tokenizer is TiktokenTokenizer); 530Assert.True(tokenizer1 is TiktokenTokenizer); 532TiktokenTokenizer tiktoken = (tokenizer as TiktokenTokenizer)!; 533TiktokenTokenizer tiktoken1 = (tokenizer1 as TiktokenTokenizer)!; 544Assert.Throws<ArgumentNullException>(() => TiktokenTokenizer.CreateForEncoding(null!)); 545Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("r50k_base_")); 546Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("p50k_base_")); 547Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("p50k_edit_")); 548Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("cl100k_base_")); 549Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("o200k_base_")); 550Assert.Throws<ArgumentException>(() => TiktokenTokenizer.CreateForEncoding("o200k_harmony_")); 581Tokenizer tokenizer = TiktokenTokenizer.CreateForModel(name); 582Assert.True(tokenizer is TiktokenTokenizer); 586int entriesCount = GetEncoder((tokenizer as TiktokenTokenizer)!)!.Count; 863private static IReadOnlyDictionary<ReadOnlyMemory<byte>, int>? GetEncoder(TiktokenTokenizer tiktoken) 864=> typeof(TiktokenTokenizer).GetProperty("Encoder", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(tiktoken) as IReadOnlyDictionary<ReadOnlyMemory<byte>, int>; 866private static IReadOnlyDictionary<int, ReadOnlyMemory<byte>>? GetDecoder(TiktokenTokenizer tiktoken) 867=> typeof(TiktokenTokenizer).GetProperty("Decoder", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(tiktoken) as IReadOnlyDictionary<int, ReadOnlyMemory<byte>>; 869private static IReadOnlyDictionary<string, int>? GetVocabulary(TiktokenTokenizer tiktoken) 870=> typeof(TiktokenTokenizer).GetProperty("Vocabulary", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(tiktoken) as IReadOnlyDictionary<string, int>;