4 references to ApplyBasicTokenization
Microsoft.ML.Tokenizers (4)
Model\BertTokenizer.cs (4)
35
ApplyBasicTokenization = options.
ApplyBasicTokenization
;
763
options.Normalizer ??= options.
ApplyBasicTokenization
? new BertNormalizer(options.LowerCaseBeforeTokenization, options.IndividuallyTokenizeCjk, options.RemoveNonSpacingMarks) : null;
767
bool lowerCase = options.
ApplyBasicTokenization
&& options.LowerCaseBeforeTokenization;
801
options.PreTokenizer ??= options.
ApplyBasicTokenization
? PreTokenizer.CreateWordOrPunctuation(options.SplitOnSpecialTokens ? options.SpecialTokens : null) : PreTokenizer.CreateWhiteSpace();