15 writes to MaxTokensPerChunk
Microsoft.Extensions.DataIngestion.Tests (15)
Chunkers\ChunkerOptionsTests.cs (6)
32IngestionChunkerOptions options = new(_tokenizer) { MaxTokensPerChunk = 100 }; 43Assert.Throws<ArgumentOutOfRangeException>("value", () => options.MaxTokensPerChunk = 0); 44Assert.Throws<ArgumentOutOfRangeException>("value", () => options.MaxTokensPerChunk = -1); 64IngestionChunkerOptions options = new(_tokenizer) { MaxTokensPerChunk = 1000 }; 75Assert.Throws<ArgumentOutOfRangeException>("value", () => options.MaxTokensPerChunk = 10); 76Assert.Throws<ArgumentOutOfRangeException>("value", () => options.MaxTokensPerChunk = 5);
Chunkers\HeaderChunkerTests.cs (7)
71HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 13 }); 96HeaderChunker lessThanContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 5 }); 99HeaderChunker sameAsContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 6 }); 119HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 30 }); 136HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 37 }); 146HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 100 }); 176HeaderChunker chunker = new(new(tokenizer) { MaxTokensPerChunk = 50 });
Chunkers\SectionChunkerTests.cs (1)
17return new SectionChunker(new(tokenizer) { MaxTokensPerChunk = maxTokensPerChunk, OverlapTokens = overlapTokens });
Chunkers\SemanticSimilarityChunkerTests.cs (1)
28new(tokenizer) { MaxTokensPerChunk = maxTokensPerChunk, OverlapTokens = overlapTokens });
5 references to MaxTokensPerChunk
Microsoft.Extensions.DataIngestion (3)
Chunkers\ElementsChunker.cs (1)
25_maxTokensPerChunk = options.MaxTokensPerChunk;
Chunkers\IngestionChunkerOptions.cs (2)
64else if (MaxTokensPerChunk > DefaultOverlapTokens) 75if (Throw.IfLessThan(value, 0) >= MaxTokensPerChunk)
Microsoft.Extensions.DataIngestion.Tests (2)
Chunkers\ChunkerOptionsTests.cs (2)
25Assert.Equal(2000, options.MaxTokensPerChunk); 34Assert.Equal(100, options.MaxTokensPerChunk);