1 write to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBpeTokenizer.cs (1)
75
SpecialTokens
= specialTokens;
5 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (5)
LlamaTests.cs (5)
433
normalizer = new SentencePieceNormalizer(removeExtraWhiteSpaces: false, addDummyPrefix: true, escapeWhiteSpaces: true, treatWhitespaceAsSuffix: false, specialTokens: (_llamaPhi3Tokenizer as LlamaTokenizer)!.
SpecialTokens
);
441
normalizer = new SentencePieceNormalizer(removeExtraWhiteSpaces: false, addDummyPrefix: true, escapeWhiteSpaces: true, treatWhitespaceAsSuffix: true, specialTokens: (_llamaPhi3Tokenizer as LlamaTokenizer)!.
SpecialTokens
);
654
Assert.True(tokenizer.
SpecialTokens
is not null);
662
foreach (var kvp in tokenizer.
SpecialTokens
)
684
string[] specialTokens = tokenizer.
SpecialTokens
.Keys.ToArray();