2 references to _specialTokens
Microsoft.ML.GenAI.LLaMA (2)
LlamaTokenizerHelper.cs (2)
52
var preTokenizer = new RegexPreTokenizer(new Regex(_re),
_specialTokens
);
53
return TiktokenTokenizer.Create(File.OpenRead(modelFilePath), preTokenizer, normalizer: null, specialTokens:
_specialTokens
);