1 write to _preTokenizer
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
316
_preTokenizer
= preTokenizer ?? PreTokenizer.CreateWordOrNonWord(); // Default to WordOrNonWord pre-tokenizer
6 references to _preTokenizer
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (6)
423
public override PreTokenizer? PreTokenizer =>
_preTokenizer
;
449
_preTokenizer
,
508
_preTokenizer
,
575
_preTokenizer
,
657
_preTokenizer
,
710
_preTokenizer
,