3 writes to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
802options.SpecialTokens = notNormalizedSpecialTokens;
Microsoft.ML.Tokenizers.Tests (2)
BertTokenizerTests.cs (1)
36SpecialTokens = specialTokens
WordPieceTests.cs (1)
188WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile, new WordPieceOptions { SpecialTokens = specialTokens });
7 references to SpecialTokens
Microsoft.ML.Tokenizers (7)
Model\BertTokenizer.cs (3)
765IReadOnlyDictionary<string, int>? specialTokensDict = options.SpecialTokens; 769if (options.SpecialTokens is not null) 776foreach (var kvp in options.SpecialTokens)
Model\WordPieceTokenizer.cs (4)
44SpecialTokens = options.SpecialTokens; 45SpecialTokensReverse = options.SpecialTokens is not null ? options.SpecialTokens.GroupBy(kvp => kvp.Value).ToDictionary(g => g.Key, g => g.First().Key) : null; 72_preTokenizer = options.PreTokenizer ?? PreTokenizer.CreateWhiteSpace(options.SpecialTokens);