File: Chunkers\DocumentChunkerTests.cs
Web Access
Project: src\test\Libraries\Microsoft.Extensions.DataIngestion.Tests\Microsoft.Extensions.DataIngestion.Tests.csproj (Microsoft.Extensions.DataIngestion.Tests)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Xunit;
 
namespace Microsoft.Extensions.DataIngestion.Chunkers.Tests
{
    public abstract class DocumentChunkerTests
    {
        protected abstract IngestionChunker<string> CreateDocumentChunker(int maxTokensPerChunk = 2_000, int overlapTokens = 500);
 
        [Fact]
        public async Task ProcessAsync_ThrowsArgumentNullException_WhenDocumentIsNull()
        {
            var chunker = CreateDocumentChunker();
            await Assert.ThrowsAsync<ArgumentNullException>("document", async () => await chunker.ProcessAsync(null!).ToListAsync());
        }
 
        [Fact]
        public async Task EmptyDocument()
        {
            IngestionDocument emptyDoc = new("emptyDoc");
            IngestionChunker<string> chunker = CreateDocumentChunker();
 
            IReadOnlyList<IngestionChunk<string>> chunks = await chunker.ProcessAsync(emptyDoc).ToListAsync();
            Assert.Empty(chunks);
        }
    }
}