3 references to SplitOnSpecialTokens
Microsoft.ML.Tokenizers (3)
Model\BertTokenizer.cs (3)
36
SplitOnSpecialTokens = options.
SplitOnSpecialTokens
;
766
if (options.
SplitOnSpecialTokens
)
809
options.PreTokenizer ??= options.ApplyBasicTokenization ? PreTokenizer.CreateWordOrPunctuation(options.
SplitOnSpecialTokens
? specialTokensDict : null) : PreTokenizer.CreateWhiteSpace();