4 references to DefaultAffix
Microsoft.ML.Tokenizers (4)
Model\SentencePieceUnigramModel.cs (4)
98
_prefixTokens =
DefaultAffix
(BeginningOfSentenceId, BeginningOfSentenceToken);
99
_suffixTokens =
DefaultAffix
(EndOfSentenceId, EndOfSentenceToken);
257
_prefixTokens = prefixTokens is not null ? ToAffixArray(prefixTokens) :
DefaultAffix
(BeginningOfSentenceId, BeginningOfSentenceToken);
258
_suffixTokens = suffixTokens is not null ? ToAffixArray(suffixTokens) :
DefaultAffix
(EndOfSentenceId, EndOfSentenceToken);