1 write to _preTokenizer
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
72
_preTokenizer
= options.PreTokenizer ?? PreTokenizer.CreateWhiteSpace(options.SpecialTokens);
5 references to _preTokenizer
Microsoft.ML.Tokenizers (5)
Model\WordPieceTokenizer.cs (5)
243
public override PreTokenizer? PreTokenizer =>
_preTokenizer
;
285
_preTokenizer
,
408
_preTokenizer
,
560
_preTokenizer
,
622
_preTokenizer
,