2 references to ToAffixArray
Microsoft.ML.Tokenizers (2)
Model\SentencePieceUnigramModel.cs (2)
257_prefixTokens = prefixTokens is not null ? ToAffixArray(prefixTokens) : DefaultAffix(BeginningOfSentenceId, BeginningOfSentenceToken); 258_suffixTokens = suffixTokens is not null ? ToAffixArray(suffixTokens) : DefaultAffix(EndOfSentenceId, EndOfSentenceToken);