5 types derived from IngestionChunker
Microsoft.Extensions.DataIngestion (4)
Chunkers\DocumentTokenChunker.cs (1)
23public sealed class DocumentTokenChunker : IngestionChunker<string>
Chunkers\HeaderChunker.cs (1)
17public sealed class HeaderChunker : IngestionChunker<string>
Chunkers\SectionChunker.cs (1)
15public sealed class SectionChunker : IngestionChunker<string>
Chunkers\SemanticSimilarityChunker.cs (1)
18public sealed class SemanticSimilarityChunker : IngestionChunker<string>
Microsoft.Extensions.DataIngestion.Tests (1)
IngestionPipelineTests.cs (1)
176internal class ImageChunker : IngestionChunker<DataContent>
23 references to IngestionChunker
Microsoft.Extensions.DataIngestion (2)
IngestionPipeline.cs (2)
28private readonly IngestionChunker<T> _chunker; 43IngestionChunker<T> chunker,
Microsoft.Extensions.DataIngestion.Tests (21)
Chunkers\DocumentChunkerTests.cs (3)
14protected abstract IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500); 19var chunker = CreateDocumentChunker(); 27IngestionChunker<string> chunker = CreateDocumentChunker();
Chunkers\DocumentTokenChunkerTests.cs (1)
26IngestionChunker<string> chunker = CreateDocumentChunker();
Chunkers\NoOverlapTokenChunkerTests.cs (3)
14protected override IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500) 32IngestionChunker<string> chunker = CreateDocumentChunker(maxTokensPerChunk: 512); 53IngestionChunker<string> chunker = CreateDocumentChunker(maxTokensPerChunk: 200, overlapTokens: 0);
Chunkers\OverlapTokenChunkerTests.cs (1)
15protected override IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500)
Chunkers\SectionChunkerTests.cs (7)
14protected override IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500) 32IngestionChunker<string> chunker = CreateDocumentChunker(); 65IngestionChunker<string> chunker = CreateDocumentChunker(); 83IngestionChunker<string> chunker = CreateDocumentChunker(); 126IngestionChunker<string> chunker = CreateDocumentChunker(); 155IngestionChunker<string> chunker = CreateDocumentChunker(maxTokensPerChunk: 512); 176IngestionChunker<string> chunker = CreateDocumentChunker();
Chunkers\SemanticSimilarityChunkerTests.cs (5)
16protected override IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500) 24private static IngestionChunker<string> CreateSemanticSimilarityChunker(TestEmbeddingGenerator embeddingClient, int maxTokensPerChunk = 2_000, int overlapTokens = 500) 56IngestionChunker<string> chunker = CreateSemanticSimilarityChunker(customGenerator); 99IngestionChunker<string> chunker = CreateSemanticSimilarityChunker(customGenerator); 175IngestionChunker<string> chunker = CreateSemanticSimilarityChunker(customGenerator, 200, 0);
IngestionPipelineTests.cs (1)
226private static IngestionChunker<string> CreateChunker() => new HeaderChunker(new(TiktokenTokenizer.CreateForModel("gpt-4")));