2 writes to Content
Microsoft.Extensions.DataIngestion.Abstractions (2)
IngestionChunk.cs (2)
36
Content
= (T)(object)Throw.IfNullOrEmpty((string)(object)content!);
40
Content
= Throw.IfNull(content);
39 references to Content
Microsoft.Extensions.DataIngestion (3)
Utils\Batching.cs (1)
33
contents.Add(new TextContent(chunk.
Content
));
Writers\VectorStoreWriter.cs (2)
82
[ContentName] = chunk.
Content
,
83
[EmbeddingName] = chunk.
Content
,
Microsoft.Extensions.DataIngestion.Tests (36)
Chunkers\HeaderChunkerTests.cs (18)
45
Assert.Equal($"Header 1 Header 1_1\nParagraph 1_1_1", chunks[0].
Content
, ignoreLineEndingDifferences: true);
47
Assert.Equal($"Header 1 Header 1_1 Header 1_1_1\nParagraph 1_1_1_1\nParagraph 1_1_1_2", chunks[1].
Content
, ignoreLineEndingDifferences: true);
49
Assert.Equal($"Header 1 Header 1_1 Header 1_1_2\nParagraph 1_1_2_1\nParagraph 1_1_2_2", chunks[2].
Content
, ignoreLineEndingDifferences: true);
51
Assert.Equal($"Header 1 Header 1_2\nParagraph 1_2_1", chunks[3].
Content
, ignoreLineEndingDifferences: true);
53
Assert.Equal($"Header 1 Header 1_2 Header 1_2_1\nParagraph 1_2_1_1", chunks[4].
Content
, ignoreLineEndingDifferences: true);
76
Assert.Equal($"Header A Header B Header C\nThis is a very long text.", chunks[0].
Content
, ignoreLineEndingDifferences: true);
78
Assert.Equal($"Header A Header B Header C\n It's expressed with plenty of tokens", chunks[1].
Content
, ignoreLineEndingDifferences: true);
125
chunks[0].
Content
, ignoreLineEndingDifferences: true);
127
Assert.Equal($"Header A Header B Header C\nWith some text after the new line.\nAnd following paragraph.", chunks[1].
Content
, ignoreLineEndingDifferences: true);
159
""", chunks[0].
Content
, ignoreLineEndingDifferences: true);
167
""", chunks[1].
Content
, ignoreLineEndingDifferences: true);
181
Assert.All(chunks, chunk => Assert.InRange(tokenizer.CountTokens(chunk.
Content
), 1, 50));
186
""", chunks[0].
Content
, ignoreLineEndingDifferences: true);
192
""", chunks[1].
Content
, ignoreLineEndingDifferences: true);
198
""", chunks[2].
Content
, ignoreLineEndingDifferences: true);
204
""", chunks[3].
Content
, ignoreLineEndingDifferences: true);
210
""", chunks[4].
Content
, ignoreLineEndingDifferences: true);
217
""", chunks[5].
Content
, ignoreLineEndingDifferences: true);
Chunkers\SectionChunkerTests.cs (11)
36
Assert.Equal(expectedResult, chunks[0].
Content
, ignoreLineEndingDifferences: true);
71
Assert.Equal(expectedResult1, chunks[0].
Content
, ignoreLineEndingDifferences: true);
72
Assert.Equal(expectedResult2, chunks[1].
Content
, ignoreLineEndingDifferences: true);
132
chunks[0].
Content
, ignoreLineEndingDifferences: true);
135
chunks[1].
Content
, ignoreLineEndingDifferences: true);
138
chunks[2].
Content
, ignoreLineEndingDifferences: true);
140
Assert.Equal("# Section title ## Subsection title\nThis is the last paragraph in subsection 1.2.", chunks[3].
Content
, ignoreLineEndingDifferences: true);
158
Assert.True(chunks[0].
Content
.Split(' ').Length <= 512);
159
Assert.True(chunks[1].
Content
.Split(' ').Length <= 512);
160
Assert.Equal(text, string.Join("", chunks.Select(c => c.
Content
)), ignoreLineEndingDifferences: true);
180
Assert.Equal(expectedResult, chunk.
Content
, ignoreLineEndingDifferences: true);
Chunkers\SemanticSimilarityChunkerTests.cs (6)
59
Assert.Equal(text, chunks[0].
Content
);
102
Assert.Equal(text1 + Environment.NewLine + text2, chunks[0].
Content
);
103
Assert.Equal(text3, chunks[1].
Content
);
184
chunks[0].
Content
, ignoreLineEndingDifferences: true);
200
chunks[1].
Content
, ignoreLineEndingDifferences: true);
206
""", chunks[2].
Content
, ignoreLineEndingDifferences: true);
Writers\VectorStoreWriterTests.cs (1)
51
Assert.Equal(chunks[0].
Content
, record["content"]);