3 writes to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
802
options.
SpecialTokens
= notNormalizedSpecialTokens;
Microsoft.ML.Tokenizers.Tests (2)
BertTokenizerTests.cs (1)
36
SpecialTokens
= specialTokens
WordPieceTests.cs (1)
188
WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile, new WordPieceOptions {
SpecialTokens
= specialTokens });
7 references to SpecialTokens
Microsoft.ML.Tokenizers (7)
Model\BertTokenizer.cs (3)
765
IReadOnlyDictionary<string, int>? specialTokensDict = options.
SpecialTokens
;
769
if (options.
SpecialTokens
is not null)
776
foreach (var kvp in options.
SpecialTokens
)
Model\WordPieceTokenizer.cs (4)
44
SpecialTokens = options.
SpecialTokens
;
45
SpecialTokensReverse = options.
SpecialTokens
is not null ? options.
SpecialTokens
.GroupBy(kvp => kvp.Value).ToDictionary(g => g.Key, g => g.First().Key) : null;
72
_preTokenizer = options.PreTokenizer ?? PreTokenizer.CreateWhiteSpace(options.
SpecialTokens
);