11 references to CausalLMDataset
Microsoft.ML.GenAI.Core (5)
Trainer\CasualLMSupervisedFineTuningTrainer.cs (1)
30CausalLMDataset trainDataset,
Trainer\CausalLMDataset.cs (4)
26public static CausalLMDataset Create(IEnumerable<IEnumerable<ChatMessage>> inputs, 50public static CausalLMDataset Create(IEnumerable<string> inputs, IEnumerable<string> outputs, Tokenizer tokenizer) 70public static CausalLMDataset Create(IEnumerable<IReadOnlyList<int>> inputIds, IEnumerable<IReadOnlyList<int>> labelIds) 84public static CausalLMDataset Create(IReadOnlyList<int> inputIds, IReadOnlyList<int> labelIds)
Microsoft.ML.GenAI.Core.Tests (3)
CasualLMDatasetTest.cs (3)
35var dataset = CausalLMDataset.Create(inputIds, outputIds) 67var dataset = CausalLMDataset.Create(inputIds, outputIds) 98var dataset = CausalLMDataset.Create(inputs, outputs, Llama3_1ChatTemplateBuilder.Instance, tokenizer)
Microsoft.ML.GenAI.Samples (3)
Llama\SFT_Llama_3_2_1B.cs (3)
46var input = CreateDataset(dataset, pipeline.TypedTokenizer, Llama3_1ChatTemplateBuilder.Instance); 101public static CausalLMDataset CreateDataset(IEnumerable<Data> dataset, Tokenizer tokenizer, IMEAIChatTemplateBuilder templateBuilder) 116return CausalLMDataset.Create(chatHistory.Select(c => c.trainChatHistory), chatHistory.Select(c => c.assistantMessage), templateBuilder, tokenizer);