1 write to _preTokenizer
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
113
_preTokenizer
= preTokenizer;
9 references to _preTokenizer
Microsoft.ML.Tokenizers (9)
Model\CodeGenTokenizer.cs (9)
249
public override PreTokenizer? PreTokenizer =>
_preTokenizer
;
357
_preTokenizer
,
370
_preTokenizer
,
606
splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer,
_preTokenizer
, out normalizedString, out textSpanToEncode, out _);
610
splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer,
_preTokenizer
, out normalizedString, out textSpanToEncode, out _);
825
splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer,
_preTokenizer
, out normalizedString, out textSpanToEncode, out _);
829
splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer,
_preTokenizer
, out normalizedString, out textSpanToEncode, out _);
962
splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer,
_preTokenizer
, out normalizedString, out textSpanToEncode, out _);
966
splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer,
_preTokenizer
, out normalizedString, out textSpanToEncode, out _);