1 write to _tokenizer
Microsoft.Extensions.DataIngestion (1)
Chunkers\ElementsChunker.cs (1)
24
_tokenizer
= options.Tokenizer;
2 references to _tokenizer
Microsoft.Extensions.DataIngestion (2)
Chunkers\ElementsChunker.cs (2)
153
int index =
_tokenizer
.GetIndexByTokenCount(
272
=>
_tokenizer
.CountTokens(input, considerNormalization: false);