2 writes to _suffixTokens
Microsoft.ML.Tokenizers (2)
Model\SentencePieceUnigramModel.cs (2)
99
_suffixTokens
= DefaultAffix(EndOfSentenceId, EndOfSentenceToken);
258
_suffixTokens
= suffixTokens is not null ? ToAffixArray(suffixTokens) : DefaultAffix(EndOfSentenceId, EndOfSentenceToken);
4 references to _suffixTokens
Microsoft.ML.Tokenizers (4)
Model\SentencePieceUnigramModel.cs (4)
287
foreach (var (id, token) in
_suffixTokens
)
878
foreach (var (id, _) in
_suffixTokens
)
1278
foreach (var _ in
_suffixTokens
)
1527
foreach (var _ in
_suffixTokens
)