new

9 references to new

Microsoft.Extensions.DataIngestion.Tests (9)

Chunkers\HeaderChunkerTests.cs (8)

39HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4"))); 71HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 13 }); 96HeaderChunker lessThanContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 5 }); 99HeaderChunker sameAsContext = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 6 }); 119HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 30 }); 136HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 37 }); 146HeaderChunker chunker = new(new(TiktokenTokenizer.CreateForModel("gpt-4")) { MaxTokensPerChunk = 100 }); 176HeaderChunker chunker = new(new(tokenizer) { MaxTokensPerChunk = 50 });

IngestionPipelineTests.cs (1)

226private static IngestionChunker<string> CreateChunker() => new HeaderChunker(new(TiktokenTokenizer.CreateForModel("gpt-4")));