15 instantiations of IngestionChunkerOptions
Microsoft.Extensions.DataIngestion.Tests (15)
Chunkers\ChunkerOptionsTests.cs (7)
17Assert.Throws<ArgumentNullException>("tokenizer", () => new IngestionChunkerOptions(null!)); 23IngestionChunkerOptions options = new(_tokenizer); 32IngestionChunkerOptions options = new(_tokenizer) { MaxTokensPerChunk = 100 }; 41IngestionChunkerOptions options = new(_tokenizer); 53IngestionChunkerOptions options = new(_tokenizer) 64IngestionChunkerOptions options = new(_tokenizer) { MaxTokensPerChunk = 1000 }; 73IngestionChunkerOptions options = new(_tokenizer) { OverlapTokens = 10 };
Chunkers\HeaderChunkerTests.cs (8)
38HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4"))); 70HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 13 }); 95HeaderChunker lessThanContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 5 }); 98HeaderChunker sameAsContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 6 }); 118HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 30 }); 135HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 37 }); 145HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 100 }); 175HeaderChunker chunker = new(new(tokenizer) { MaxTokensPerChunk = 50 });
9 references to IngestionChunkerOptions
Microsoft.Extensions.DataIngestion (3)
Chunkers\ElementsChunker.cs (1)
20internal ElementsChunker(IngestionChunkerOptions options)
Chunkers\HeaderChunker.cs (1)
26public HeaderChunker(IngestionChunkerOptions options)
Chunkers\IngestionChunkerOptions.cs (1)
21/// Initializes a new instance of the <see cref="IngestionChunkerOptions"/> class.
Microsoft.Extensions.DataIngestion.Tests (6)
Chunkers\ChunkerOptionsTests.cs (6)
23IngestionChunkerOptions options = new(_tokenizer); 32IngestionChunkerOptions options = new(_tokenizer) { MaxTokensPerChunk = 100 }; 41IngestionChunkerOptions options = new(_tokenizer); 53IngestionChunkerOptions options = new(_tokenizer) 64IngestionChunkerOptions options = new(_tokenizer) { MaxTokensPerChunk = 1000 }; 73IngestionChunkerOptions options = new(_tokenizer) { OverlapTokens = 10 };