6 references to Create
Microsoft.ML.GenAI.Core.Tests (1)
CasualLMDatasetTest.cs (1)
26
return LlamaTokenizer.
Create
(remoteStream);
Microsoft.ML.GenAI.Mistral (1)
MistralTokenizerHelper.cs (1)
100
var llamaTokenizer = LlamaTokenizer.
Create
(
Microsoft.ML.GenAI.Phi (1)
Phi3\Phi3TokenizerHelper.cs (1)
37
var llamaTokenizer = LlamaTokenizer.
Create
(
Microsoft.ML.Tokenizers.Tests (3)
LlamaTests.cs (3)
33
return LlamaTokenizer.
Create
(remoteStream);
40
return LlamaTokenizer.
Create
(remoteStream);
47
LlamaTokenizer tokenizer = LlamaTokenizer.
Create
(remoteStream, addBeginOfSentence: true, addEndOfSentence: false,