4 references to CodeGenAddedTokens
Microsoft.ML.Tokenizers (4)
Model\CodeGenTokenizer.cs (2)
1896
new RegexPreTokenizer(TiktokenTokenizer.P50kBaseRegex(), CodeGenTokenizer.
CodeGenAddedTokens
),
1898
CodeGenTokenizer.
CodeGenAddedTokens
,
Model\Phi2Tokenizer.cs (2)
116
vocabStream, mergesStream, new RegexPreTokenizer(TiktokenTokenizer.P50kBaseRegex(), CodeGenTokenizer.
CodeGenAddedTokens
), normalizer: null,
117
CodeGenTokenizer.
CodeGenAddedTokens
, addPrefixSpace: addPrefixSpace, addBeginningOfSentence: addBeginOfSentence, addEndOfSentence: addEndOfSentence);