1 reference to _re
Microsoft.ML.GenAI.LLaMA (1)
LlamaTokenizerHelper.cs (1)
52
var preTokenizer = new TiktokenPreTokenizer(new Regex(
_re
), _specialTokens);