1 write to _maxTokensPerChunk
Microsoft.Extensions.DataIngestion (1)
Chunkers\ElementsChunker.cs (1)
25
_maxTokensPerChunk
= options.MaxTokensPerChunk;
9 references to _maxTokensPerChunk
Microsoft.Extensions.DataIngestion (9)
Chunkers\ElementsChunker.cs (9)
28
_currentChunk = new(capacity:
_maxTokensPerChunk
);
44
if (contextTokenCount >=
_maxTokensPerChunk
)
71
if (elementTokenCount + totalTokenCount <=
_maxTokensPerChunk
)
89
if (contextTokenCount + headerTokenCount >=
_maxTokensPerChunk
)
94
if (headerTokenCount + totalTokenCount >=
_maxTokensPerChunk
)
111
if (totalTokenCount + lastRowTokens >
_maxTokensPerChunk
)
127
if (totalTokenCount + lastRowTokens >
_maxTokensPerChunk
)
155
maxTokenCount:
_maxTokensPerChunk
- totalTokenCount,
191
if (totalTokenCount ==
_maxTokensPerChunk
)