1 write to Tokenizer
Microsoft.Extensions.DataIngestion (1)
Chunkers\IngestionChunkerOptions.cs (1)
26
Tokenizer
= Throw.IfNull(tokenizer);
2 references to Tokenizer
Microsoft.Extensions.DataIngestion (2)
Chunkers\ElementsChunker.cs (1)
24
_tokenizer = options.
Tokenizer
;
Chunkers\IngestionChunkerOptions.cs (1)
30
/// Gets the <see cref="
Tokenizer
"/> instance used to process and tokenize input data.