4 references to CodeGenSpecialTokens
Microsoft.ML.Tokenizers (4)
Model\CodeGenTokenizer.cs (2)
1897
new RegexPreTokenizer(TiktokenTokenizer.P50kBaseRegex(), CodeGenTokenizer.
CodeGenSpecialTokens
),
1899
CodeGenTokenizer.
CodeGenSpecialTokens
,
Model\Phi2Tokenizer.cs (2)
117
vocabStream, mergesStream, new RegexPreTokenizer(TiktokenTokenizer.P50kBaseRegex(), CodeGenTokenizer.
CodeGenSpecialTokens
), normalizer: null,
118
CodeGenTokenizer.
CodeGenSpecialTokens
, addPrefixSpace: addPrefixSpace, addBeginningOfSentence: addBeginOfSentence, addEndOfSentence: addEndOfSentence);