1 write to _preTokenizer
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
243
_preTokenizer
= preTokenizer ?? PreTokenizer.CreateWordOrNonWord(); // Default to WordOrNonWord pre-tokenizer
6 references to _preTokenizer
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (6)
300
public override PreTokenizer? PreTokenizer =>
_preTokenizer
;
326
_preTokenizer
,
375
_preTokenizer
,
431
_preTokenizer
,
508
_preTokenizer
,
556
_preTokenizer
,