4 types derived from IngestionChunker
Microsoft.Extensions.DataIngestion (3)
Chunkers\HeaderChunker.cs (1)
17public sealed class HeaderChunker : IngestionChunker<string>
Chunkers\SectionChunker.cs (1)
15public sealed class SectionChunker : IngestionChunker<string>
Chunkers\SemanticSimilarityChunker.cs (1)
18public sealed class SemanticSimilarityChunker : IngestionChunker<string>
Microsoft.Extensions.DataIngestion.Tests (1)
IngestionPipelineTests.cs (1)
176internal class ImageChunker : IngestionChunker<DataContent>
18 references to IngestionChunker
Microsoft.Extensions.DataIngestion (2)
IngestionPipeline.cs (2)
28private readonly IngestionChunker<T> _chunker; 43IngestionChunker<T> chunker,
Microsoft.Extensions.DataIngestion.Tests (16)
Chunkers\DocumentChunkerTests.cs (3)
14protected abstract IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500); 19var chunker = CreateDocumentChunker(); 27IngestionChunker<string> chunker = CreateDocumentChunker();
Chunkers\SectionChunkerTests.cs (7)
14protected override IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500) 32IngestionChunker<string> chunker = CreateDocumentChunker(); 65IngestionChunker<string> chunker = CreateDocumentChunker(); 83IngestionChunker<string> chunker = CreateDocumentChunker(); 126IngestionChunker<string> chunker = CreateDocumentChunker(); 155IngestionChunker<string> chunker = CreateDocumentChunker(maxTokensPerChunk: 512); 176IngestionChunker<string> chunker = CreateDocumentChunker();
Chunkers\SemanticSimilarityChunkerTests.cs (5)
16protected override IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500) 24private static IngestionChunker<string> CreateSemanticSimilarityChunker(TestEmbeddingGenerator embeddingClient, int maxTokensPerChunk = 2_000, int overlapTokens = 500) 56IngestionChunker<string> chunker = CreateSemanticSimilarityChunker(customGenerator); 99IngestionChunker<string> chunker = CreateSemanticSimilarityChunker(customGenerator); 175IngestionChunker<string> chunker = CreateSemanticSimilarityChunker(customGenerator, 200, 0);
IngestionPipelineTests.cs (1)
226private static IngestionChunker<string> CreateChunker() => new HeaderChunker(new(TiktokenTokenizer.CreateForModel("gpt-4")));