1 write to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
44
SpecialTokens
= options.SpecialTokens;
5 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (5)
BertTokenizerTests.cs (3)
52
Assert.True(tokenizer.
SpecialTokens
!.ContainsKey("[SPECIAL]"));
124
Assert.True(tokenizer.
SpecialTokens
!.ContainsKey(tokenizer.UnknownToken));
125
Assert.True(tokenizer.
SpecialTokens
!.ContainsKey(tokenizer.ClassificationToken));
WordPieceTests.cs (2)
60
Assert.Null(tokenizer.
SpecialTokens
);
190
Assert.Equal(specialTokens, tokenizer.
SpecialTokens
);