1 write to _preTokenizer
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
77
_preTokenizer
= preTokenizer ?? PreTokenizer.CreateWhiteSpacePreTokenizer(specialTokens);
5 references to _preTokenizer
Microsoft.ML.Tokenizers (5)
Model\WordPieceTokenizer.cs (5)
298
public override PreTokenizer? PreTokenizer =>
_preTokenizer
;
340
_preTokenizer
,
463
_preTokenizer
,
615
_preTokenizer
,
677
_preTokenizer
,