1 write to _preTokenizer
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
311
_preTokenizer
= preTokenizer ?? PreTokenizer.CreateWordOrNonWord(); // Default to WordOrNonWord pre-tokenizer
6 references to _preTokenizer
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (6)
418
public override PreTokenizer? PreTokenizer =>
_preTokenizer
;
444
_preTokenizer
,
503
_preTokenizer
,
570
_preTokenizer
,
652
_preTokenizer
,
705
_preTokenizer
,