1 write to SpecialTokens
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
44
SpecialTokens
= specialTokens;
4 references to SpecialTokens
Microsoft.ML.Tokenizers (4)
Normalizer\SentencePieceNormalizer.cs (4)
194
if (
SpecialTokens
is not null)
229
if (
SpecialTokens
is not null)
257
foreach (var kvp in
SpecialTokens
!)
292
foreach (var kvp in
SpecialTokens
!)