3 instantiations of DocumentTokenChunker
Microsoft.Extensions.DataIngestion.Tests (3)
Chunkers\NoOverlapTokenChunkerTests.cs (1)
17
return new
DocumentTokenChunker
(new(tokenizer) { MaxTokensPerChunk = maxTokensPerChunk, OverlapTokens = 0 });
Chunkers\OverlapTokenChunkerTests.cs (2)
18
return new
DocumentTokenChunker
(new(tokenizer) { MaxTokensPerChunk = maxTokensPerChunk, OverlapTokens = overlapTokens });
29
var chunker = new
DocumentTokenChunker
(new(tokenizer) { MaxTokensPerChunk = chunkSize, OverlapTokens = chunkOverlap });
2 references to DocumentTokenChunker
Microsoft.Extensions.DataIngestion (1)
Chunkers\DocumentTokenChunker.cs (1)
30
/// Initializes a new instance of the <see cref="
DocumentTokenChunker
"/> class with the specified options.
Microsoft.Extensions.DataIngestion.Tests (1)
Chunkers\OverlapTokenChunkerTests.cs (1)
29
var
chunker = new DocumentTokenChunker(new(tokenizer) { MaxTokensPerChunk = chunkSize, OverlapTokens = chunkOverlap });