18 references to IngestionChunkerOptions
Microsoft.Extensions.DataIngestion.Tests (18)
Chunkers\ChunkerOptionsTests.cs (7)
17Assert.Throws<ArgumentNullException>("tokenizer", () => new IngestionChunkerOptions(null!)); 23IngestionChunkerOptions options = new(_tokenizer); 32IngestionChunkerOptions options = new(_tokenizer) { MaxTokensPerChunk = 100 }; 41IngestionChunkerOptions options = new(_tokenizer); 53IngestionChunkerOptions options = new(_tokenizer) 64IngestionChunkerOptions options = new(_tokenizer) { MaxTokensPerChunk = 1000 }; 73IngestionChunkerOptions options = new(_tokenizer) { OverlapTokens = 10 };
Chunkers\HeaderChunkerTests.cs (8)
39HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4"))); 71HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 13 }); 96HeaderChunker lessThanContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 5 }); 99HeaderChunker sameAsContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 6 }); 119HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 30 }); 136HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 37 }); 146HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 100 }); 176HeaderChunker chunker = new(new(tokenizer) { MaxTokensPerChunk = 50 });
Chunkers\SectionChunkerTests.cs (1)
17return new SectionChunker(new(tokenizer) { MaxTokensPerChunk = maxTokensPerChunk, OverlapTokens = overlapTokens });
Chunkers\SemanticSimilarityChunkerTests.cs (1)
28new(tokenizer) { MaxTokensPerChunk = maxTokensPerChunk, OverlapTokens = overlapTokens });
IngestionPipelineTests.cs (1)
226private static IngestionChunker<string> CreateChunker() => new HeaderChunker(new(TiktokenTokenizer.CreateForModel("gpt-4")));