9 instantiations of HeaderChunker
Microsoft.Extensions.DataIngestion.Tests (9)
Chunkers\HeaderChunkerTests.cs (8)
39HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4"))); 71HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 13 }); 96HeaderChunker lessThanContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 5 }); 99HeaderChunker sameAsContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 6 }); 119HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 30 }); 136HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 37 }); 146HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 100 }); 176HeaderChunker chunker = new(new(tokenizer) { MaxTokensPerChunk = 50 });
IngestionPipelineTests.cs (1)
226private static IngestionChunker<string> CreateChunker() => new HeaderChunker(new(TiktokenTokenizer.CreateForModel("gpt-4")));
9 references to HeaderChunker
Microsoft.Extensions.DataIngestion (1)
Chunkers\HeaderChunker.cs (1)
23/// Initializes a new instance of the <see cref="HeaderChunker"/> class.
Microsoft.Extensions.DataIngestion.Tests (8)
Chunkers\HeaderChunkerTests.cs (8)
39HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4"))); 71HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 13 }); 96HeaderChunker lessThanContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 5 }); 99HeaderChunker sameAsContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 6 }); 119HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 30 }); 136HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 37 }); 146HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 100 }); 176HeaderChunker chunker = new(new(tokenizer) { MaxTokensPerChunk = 50 });