12 references to DefaultSpecialToken
Microsoft.ML.Tokenizers (12)
Model\CodeGenTokenizer.cs (6)
61
string? unknownToken =
DefaultSpecialToken
,
62
string? beginningOfSentenceToken =
DefaultSpecialToken
,
63
string? endOfSentenceToken =
DefaultSpecialToken
) :
93
string? unknownToken =
DefaultSpecialToken
,
94
string? beginningOfSentenceToken =
DefaultSpecialToken
,
95
string? endOfSentenceToken =
DefaultSpecialToken
) :
Model\Phi2Tokenizer.cs (6)
43
string? unknownToken =
DefaultSpecialToken
,
44
string? beginningOfSentenceToken =
DefaultSpecialToken
,
45
string? endOfSentenceToken =
DefaultSpecialToken
) :
74
string? unknownToken =
DefaultSpecialToken
,
75
string? beginningOfSentenceToken =
DefaultSpecialToken
,
76
string? endOfSentenceToken =
DefaultSpecialToken
) :