1 write to SpecialTokens
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
28
SpecialTokens
= specialTokens;
4 references to SpecialTokens
Microsoft.ML.Tokenizers (4)
Normalizer\SentencePieceNormalizer.cs (4)
118
if (
SpecialTokens
is not null)
153
if (
SpecialTokens
is not null)
181
foreach (var kvp in
SpecialTokens
!)
216
foreach (var kvp in
SpecialTokens
!)