2 writes to _prefixTokens
Microsoft.ML.Tokenizers (2)
Model\SentencePieceUnigramModel.cs (2)
98
_prefixTokens
= DefaultAffix(BeginningOfSentenceId, BeginningOfSentenceToken);
257
_prefixTokens
= prefixTokens is not null ? ToAffixArray(prefixTokens) : DefaultAffix(BeginningOfSentenceId, BeginningOfSentenceToken);
4 references to _prefixTokens
Microsoft.ML.Tokenizers (4)
Model\SentencePieceUnigramModel.cs (4)
279
foreach (var (id, token) in
_prefixTokens
)
849
foreach (var (id, _) in
_prefixTokens
)
1249
foreach (var _ in
_prefixTokens
)
1557
foreach (var _ in
_prefixTokens
)