new

8 references to new

Microsoft.Extensions.DataIngestion.Tests (8)

Chunkers\HeaderChunkerTests.cs (8)

38HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4"))); 70HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 13 }); 95HeaderChunker lessThanContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 5 }); 98HeaderChunker sameAsContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 6 }); 118HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 30 }); 135HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 37 }); 145HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 100 }); 175HeaderChunker chunker = new(new(tokenizer) { MaxTokensPerChunk = 50 });