1 reference to CreateWordOrPunctuation
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
809
options.PreTokenizer ??= options.ApplyBasicTokenization ? PreTokenizer.
CreateWordOrPunctuation
(options.SplitOnSpecialTokens ? specialTokensDict : null) : PreTokenizer.CreateWhiteSpace();