12 references to DefaultSpecialToken
Microsoft.ML.Tokenizers (12)
Model\CodeGenTokenizer.cs (6)
64
string? unknownToken =
DefaultSpecialToken
,
65
string? beginningOfSentenceToken =
DefaultSpecialToken
,
66
string? endOfSentenceToken =
DefaultSpecialToken
) :
96
string? unknownToken =
DefaultSpecialToken
,
97
string? beginningOfSentenceToken =
DefaultSpecialToken
,
98
string? endOfSentenceToken =
DefaultSpecialToken
) :
Model\Phi2Tokenizer.cs (6)
43
string? unknownToken =
DefaultSpecialToken
,
44
string? beginningOfSentenceToken =
DefaultSpecialToken
,
45
string? endOfSentenceToken =
DefaultSpecialToken
) :
74
string? unknownToken =
DefaultSpecialToken
,
75
string? beginningOfSentenceToken =
DefaultSpecialToken
,
76
string? endOfSentenceToken =
DefaultSpecialToken
) :