1 write to _preTokenizer
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
116_preTokenizer = preTokenizer;
9 references to _preTokenizer
Microsoft.ML.Tokenizers (9)
Model\CodeGenTokenizer.cs (9)
252public override PreTokenizer? PreTokenizer => _preTokenizer; 360_preTokenizer, 373_preTokenizer, 611splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 615splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 830splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 834splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 967splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 971splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);