5 references to CountTokens
Microsoft.Extensions.DataIngestion (5)
Chunkers\ElementsChunker.cs (5)
40
int contextTokenCount =
CountTokens
(context.AsSpan());
70
int elementTokenCount =
CountTokens
(semanticContent.AsSpan());
86
int headerTokenCount =
CountTokens
(tableBuilder.AsSpan());
108
int lastRowTokens =
CountTokens
(tableBuilder.AsSpan(tableLength));
170
tokenCount =
CountTokens
(remainingContent.Slice(0, index));