3 writes to SpecialTokens
Microsoft.ML.Tokenizers (2)
Model\BertTokenizer.cs (2)
773options.SpecialTokens = dic; 791options.SpecialTokens = specialTokens;
Microsoft.ML.Tokenizers.Tests (1)
WordPieceTests.cs (1)
188WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile, new WordPieceOptions { SpecialTokens = specialTokens });
8 references to SpecialTokens
Microsoft.ML.Tokenizers (8)
Model\BertTokenizer.cs (4)
768if (options.SpecialTokens is not null) 772Dictionary<string, int> dic = options.SpecialTokens.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); 775foreach (var kvp in options.SpecialTokens) 801options.PreTokenizer ??= options.ApplyBasicTokenization ? PreTokenizer.CreateWordOrPunctuation(options.SplitOnSpecialTokens ? options.SpecialTokens : null) : PreTokenizer.CreateWhiteSpace();
Model\WordPieceTokenizer.cs (4)
44SpecialTokens = options.SpecialTokens; 45SpecialTokensReverse = options.SpecialTokens is not null ? options.SpecialTokens.ToDictionary(kvp => kvp.Value, kvp => kvp.Key) : null; 72_preTokenizer = options.PreTokenizer ?? PreTokenizer.CreateWhiteSpace(options.SpecialTokens);