6 references to _tokenizer
Microsoft.Extensions.DataIngestion.Tests (6)
Chunkers\ChunkerOptionsTests.cs (6)
23
IngestionChunkerOptions options = new(
_tokenizer
);
32
IngestionChunkerOptions options = new(
_tokenizer
) { MaxTokensPerChunk = 100 };
41
IngestionChunkerOptions options = new(
_tokenizer
);
53
IngestionChunkerOptions options = new(
_tokenizer
)
64
IngestionChunkerOptions options = new(
_tokenizer
) { MaxTokensPerChunk = 1000 };
73
IngestionChunkerOptions options = new(
_tokenizer
) { OverlapTokens = 10 };