2 references to _specialTokens
Microsoft.ML.GenAI.LLaMA (2)
LlamaTokenizerHelper.cs (2)
52var preTokenizer = new RegexPreTokenizer(new Regex(_re), _specialTokens); 53return TiktokenTokenizer.Create(File.OpenRead(modelFilePath), preTokenizer, normalizer: null, specialTokens: _specialTokens);