1 write to SpecialTokens
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
37
SpecialTokens
= specialTokens;
4 references to SpecialTokens
Microsoft.ML.Tokenizers (4)
Normalizer\SentencePieceNormalizer.cs (4)
146
if (
SpecialTokens
is not null)
181
if (
SpecialTokens
is not null)
209
foreach (var kvp in
SpecialTokens
!)
244
foreach (var kvp in
SpecialTokens
!)