19 instantiations of IngestionChunk
Microsoft.Extensions.DataIngestion (3)
Chunkers\DocumentTokenChunker.cs (1)
94IngestionChunk<string> chunk = new IngestionChunk<string>(
Chunkers\ElementsChunker.cs (2)
199chunks.Add(new(_currentChunk.ToString(), document, context)); 208chunks.Add(new(_currentChunk.ToString(), document, context));
Microsoft.Extensions.DataIngestion.Tests (16)
IngestionPipelineTests.cs (1)
181.Select(image => new IngestionChunk<DataContent>(
Processors\ClassificationEnricherTests.cs (3)
122new(".NET developers need to integrate and interact with a growing variety of artificial intelligence (AI) services in their apps. " + 125new ("Rabbits are small mammals in the family Leporidae of the order Lagomorpha (along with the hare and the pika)." + 127new("This text does not belong to any category.", _document),
Processors\KeywordEnricherTests.cs (2)
122new("The Microsoft.Extensions.AI libraries provide a unified approach for representing generative AI components", _document), 123new("Rabbits are great pets. They are friendly and make excellent companions.", _document)
Processors\SentimentEnricherTests.cs (4)
109new("I love programming! It's so much fun and rewarding.", _document), 110new("I hate bugs. They are so frustrating and time-consuming.", _document), 111new("The weather is okay, not too bad but not great either.", _document), 112new("I hate you. I am sorry, I actually don't. I am not sure myself what my feelings are.", _document)
Processors\SummaryEnricherTests.cs (2)
106new("I love programming! It's so much fun and rewarding.", _document), 107new("I hate bugs. They are so frustrating and time-consuming.", _document)
Writers\VectorStoreWriterTests.cs (4)
29new("some content", document) 78new("first chunk", document) 85new("second chunk", document) 98new("different content", document)
89 references to IngestionChunk
Microsoft.Extensions.DataIngestion (38)
Chunkers\DocumentTokenChunker.cs (3)
43public override async IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(IngestionDocument document, [EnumeratorCancellation] CancellationToken cancellationToken = default) 92IngestionChunk<string> FinalizeChunk() 94IngestionChunk<string> chunk = new IngestionChunk<string>(
Chunkers\ElementsChunker.cs (2)
35internal IEnumerable<IngestionChunk<string>> Process(IngestionDocument document, string context, List<IngestionDocumentElement> elements) 38List<IngestionChunk<string>> chunks = [];
Chunkers\HeaderChunker.cs (5)
32public override async IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(IngestionDocument document, 46foreach (var chunk in SplitIntoChunks(document, headers, elements)) 62foreach (var chunk in SplitIntoChunks(document, headers, elements)) 68private IEnumerable<IngestionChunk<string>> SplitIntoChunks(IngestionDocument document, string?[] headers, List<IngestionDocumentElement> elements) 74foreach (var chunk in _elementsChunker.Process(document, chunkHeader, elements))
Chunkers\SectionChunker.cs (5)
29public override async IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(IngestionDocument document, [EnumeratorCancellation] CancellationToken cancellationToken = default) 33List<IngestionChunk<string>> chunks = []; 39foreach (var chunk in chunks) 47private void Process(IngestionDocument document, IngestionDocumentSection section, List<IngestionChunk<string>> chunks, string? parentContext = null) 79foreach (var chunk in _elementsChunker.Process(document, context, elements))
Chunkers\SemanticSimilarityChunker.cs (4)
47public override async IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(IngestionDocument document, 53foreach (var chunk in MakeChunks(document, distances)) 96private IEnumerable<IngestionChunk<string>> MakeChunks(IngestionDocument document, List<(IngestionDocumentElement element, float distance)> elementDistances) 109foreach (var chunk in _elementsChunker.Process(document, context, elementAccumulator))
IngestionPipeline.cs (1)
183IAsyncEnumerable<IngestionChunk<T>> chunks = _chunker.ProcessAsync(document, cancellationToken);
Processors\ClassificationEnricher.cs (2)
52public override IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(IAsyncEnumerable<IngestionChunk<string>> chunks, CancellationToken cancellationToken = default)
Processors\KeywordEnricher.cs (2)
60public override IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(IAsyncEnumerable<IngestionChunk<string>> chunks, CancellationToken cancellationToken = default)
Processors\SentimentEnricher.cs (2)
50public override IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(IAsyncEnumerable<IngestionChunk<string>> chunks, CancellationToken cancellationToken = default)
Processors\SummaryEnricher.cs (2)
45public override IAsyncEnumerable<IngestionChunk<string>> ProcessAsync(IAsyncEnumerable<IngestionChunk<string>> chunks, CancellationToken cancellationToken = default)
Utils\Batching.cs (4)
18internal static async IAsyncEnumerable<IngestionChunk<string>> ProcessAsync<TMetadata>(IAsyncEnumerable<IngestionChunk<string>> chunks, 31foreach (var chunk in batch) 64foreach (var chunk in batch)
Writers\VectorStoreWriter.cs (6)
35/// <param name="vectorStore">The <see cref="VectorStore"/> to use to store the <see cref="IngestionChunk{T}"/> instances.</param> 51/// The collection is initialized when <see cref="WriteAsync(IAsyncEnumerable{IngestionChunk{T}}, CancellationToken)"/> is called for the first time. 54/// Call <see cref="WriteAsync(IAsyncEnumerable{IngestionChunk{T}}, CancellationToken)"/> first.</exception> 59public override async Task WriteAsync(IAsyncEnumerable<IngestionChunk<T>> chunks, CancellationToken cancellationToken = default) 64await foreach (IngestionChunk<T> chunk in chunks.WithCancellation(cancellationToken)) 119private VectorStoreCollectionDefinition GetVectorStoreRecordDefinition(IngestionChunk<T> representativeChunk)
Microsoft.Extensions.DataIngestion.Abstractions (5)
IngestionChunk.cs (1)
21/// Initializes a new instance of the <see cref="IngestionChunk{T}"/> class.
IngestionChunker.cs (1)
21public abstract IAsyncEnumerable<IngestionChunk<T>> ProcessAsync(IngestionDocument document, CancellationToken cancellationToken = default);
IngestionChunkProcessor.cs (2)
21public abstract IAsyncEnumerable<IngestionChunk<T>> ProcessAsync(IAsyncEnumerable<IngestionChunk<T>> chunks, CancellationToken cancellationToken = default);
IngestionChunkWriter.cs (1)
23public abstract Task WriteAsync(IAsyncEnumerable<IngestionChunk<T>> chunks, CancellationToken cancellationToken = default);
Microsoft.Extensions.DataIngestion.Tests (46)
Chunkers\DocumentChunkerTests.cs (1)
29IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(emptyDoc).ToListAsync();
Chunkers\DocumentTokenChunkerTests.cs (2)
27IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 29IngestionChunk<string> chunk = Assert.Single(chunks);
Chunkers\HeaderChunkerTests.cs (5)
40IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 72IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 120IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 147IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(document).ToListAsync(); 177IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(document).ToListAsync();
Chunkers\NoOverlapTokenChunkerTests.cs (3)
33IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 54IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 56foreach (var chunk in chunks)
Chunkers\OverlapTokenChunkerTests.cs (3)
39IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 49var currentChunk = chunks[i]; 50var nextChunk = chunks[i + 1];
Chunkers\SectionChunkerTests.cs (7)
33IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 66IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 84IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 127IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 156IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 177IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 178IngestionChunk<string> chunk = Assert.Single(chunks);
Chunkers\SemanticSimilarityChunkerTests.cs (3)
57IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 100IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync(); 176IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(doc).ToListAsync();
IngestionPipelineTests.cs (1)
178public override IAsyncEnumerable<IngestionChunk<DataContent>> ProcessAsync(IngestionDocument document, CancellationToken cancellationToken = default)
Processors\ClassificationEnricherTests.cs (5)
58await foreach (var _ in sut.ProcessAsync(null!)) 90IReadOnlyList<IngestionChunk<string>> got = await sut.ProcessAsync(CreateChunks().ToAsyncEnumerable()).ToListAsync(); 109List<IngestionChunk<string>> chunks = CreateChunks(); 111IReadOnlyList<IngestionChunk<string>> got = await sut.ProcessAsync(chunks.ToAsyncEnumerable()).ToListAsync(); 120private static List<IngestionChunk<string>> CreateChunks() =>
Processors\KeywordEnricherTests.cs (5)
56await foreach (var _ in sut.ProcessAsync(null!)) 92IReadOnlyList<IngestionChunk<string>> got = await sut.ProcessAsync(chunks).ToListAsync(); 109List<IngestionChunk<string>> chunks = CreateChunks(); 111IReadOnlyList<IngestionChunk<string>> got = await sut.ProcessAsync(chunks.ToAsyncEnumerable()).ToListAsync(); 120private static List<IngestionChunk<string>> CreateChunks() =>
Processors\SentimentEnricherTests.cs (4)
42await foreach (var _ in sut.ProcessAsync(null!)) 96List<IngestionChunk<string>> chunks = CreateChunks(); 98IReadOnlyList<IngestionChunk<string>> got = await sut.ProcessAsync(chunks.ToAsyncEnumerable()).ToListAsync(); 107private static List<IngestionChunk<string>> CreateChunks() =>
Processors\SummaryEnricherTests.cs (4)
42await foreach (var _ in sut.ProcessAsync(null!)) 93List<IngestionChunk<string>> chunks = CreateChunks(); 95IReadOnlyList<IngestionChunk<string>> got = await sut.ProcessAsync(chunks.ToAsyncEnumerable()).ToListAsync(); 104private static List<IngestionChunk<string>> CreateChunks() =>
Writers\VectorStoreWriterTests.cs (3)
27List<IngestionChunk<string>> chunks = 76List<IngestionChunk<string>> chunks = 96List<IngestionChunk<string>> updatedChunks =