3 references to EncodeToIds
Microsoft.ML.GenAI.Mistral.Tests (1)
Mistral_7B_Instruct_V0_3Tests.cs (1)
128
var tokenizeIds = tokenizer.
EncodeToIds
(message, false, false);
Microsoft.ML.GenAI.Phi.Tests (2)
Phi3Tests.cs (2)
125
var endIds = tokenizer.
EncodeToIds
("<|end|>", addBeginningOfSentence: false, addEndOfSentence: false, considerPreTokenization: false, considerNormalization: false);
142
var tokenizeIds = tokenizer.
EncodeToIds
(message, true, false, considerPreTokenization: true);