10 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (10)
LlamaTests.cs (5)
433
normalizer = new SentencePieceNormalizer(removeExtraWhiteSpaces: false, addDummyPrefix: true, escapeWhiteSpaces: true, treatWhitespaceAsSuffix: false, specialTokens: (_llamaPhi3Tokenizer as LlamaTokenizer)!.
SpecialTokens
);
441
normalizer = new SentencePieceNormalizer(removeExtraWhiteSpaces: false, addDummyPrefix: true, escapeWhiteSpaces: true, treatWhitespaceAsSuffix: true, specialTokens: (_llamaPhi3Tokenizer as LlamaTokenizer)!.
SpecialTokens
);
654
Assert.True(tokenizer.
SpecialTokens
is not null);
662
foreach (var kvp in tokenizer.
SpecialTokens
)
684
string[] specialTokens = tokenizer.
SpecialTokens
.Keys.ToArray();
UnigramTests.cs (5)
459
expectedIds[ids.Length + 1] = _unigramTokenizerWithSpecialTokens.
SpecialTokens
!["<pad>"];
479
expectedIds[shiftedIds.Length + 1] = _unigramTokenizerFromJson.
SpecialTokens
!["<pad>"];
668
expectedIds[ids.Length + 1] = _unigramTokenizerWithSpecialTokens.
SpecialTokens
!["<pad>"];
687
expectedIds[shiftedIds.Length + 1] = _unigramTokenizerFromJson.
SpecialTokens
!["<pad>"];
830
Assert.Equal(specialTokens, _unigramTokenizerFromJson.
SpecialTokens
);