1 write to PreTokenizer
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
801
options.
PreTokenizer
??= options.ApplyBasicTokenization ? PreTokenizer.CreateWordOrPunctuation(options.SplitOnSpecialTokens ? options.SpecialTokens : null) : PreTokenizer.CreateWhiteSpace();
6 references to PreTokenizer
Microsoft.ML.Tokenizers (6)
Model\WordPieceOptions.cs (1)
19
/// Gets or sets the <see cref="
PreTokenizer
"/> to override the default normalizer, if desired.
Model\WordPieceTokenizer.cs (5)
72
_preTokenizer = options.
PreTokenizer
?? PreTokenizer.CreateWhiteSpace(options.SpecialTokens);
128
/// If the <see cref="WordPieceOptions.
PreTokenizer
"/> is null, the whitespace pre-tokenizer will be used.
143
/// If the <see cref="WordPieceOptions.
PreTokenizer
"/> is null, the whitespace pre-tokenizer will be used.
184
/// If the <see cref="WordPieceOptions.
PreTokenizer
"/> is null, the whitespace pre-tokenizer will be used.
205
/// If the <see cref="WordPieceOptions.
PreTokenizer
"/> is null, the whitespace pre-tokenizer will be used.