3 references to DocumentTokenChunker
Microsoft.Extensions.DataIngestion.Tests (3)
Chunkers\NoOverlapTokenChunkerTests.cs (1)
17
return new
DocumentTokenChunker
(new(tokenizer) { MaxTokensPerChunk = maxTokensPerChunk, OverlapTokens = 0 });
Chunkers\OverlapTokenChunkerTests.cs (2)
18
return new
DocumentTokenChunker
(new(tokenizer) { MaxTokensPerChunk = maxTokensPerChunk, OverlapTokens = overlapTokens });
29
var chunker = new
DocumentTokenChunker
(new(tokenizer) { MaxTokensPerChunk = chunkSize, OverlapTokens = chunkOverlap });