25 references to InitializeForEncoding
Microsoft.ML.Tokenizers (25)
Model\BPETokenizer.cs (5)
438IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 497IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 564IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 646IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 699IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\CodeGenTokenizer.cs (8)
354splits = InitializeForEncoding( 367splits = InitializeForEncoding( 611splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 615splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 830splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 834splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 967splits = InitializeForEncoding(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _); 971splits = InitializeForEncoding(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);
Model\EnglishRobertaTokenizer.cs (4)
321IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 422IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 509IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 556IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\TiktokenTokenizer.cs (4)
265IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 375IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 545IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 671IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\WordPieceTokenizer.cs (4)
279IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 402IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 554IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 616IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(