1 write to _preTokenizer
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
113_preTokenizer = preTokenizer;
9 references to _preTokenizer
Microsoft.ML.Tokenizers (9)
Model\CodeGenTokenizer.cs (9)
249public override PreTokenizer? PreTokenizer => _preTokenizer; 357_preTokenizer, 370_preTokenizer, 608splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 612splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 827splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 831splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 964splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 968splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);