25 references to InitializeForEncoding
Microsoft.ML.Tokenizers (25)
Model\BPETokenizer.cs (5)
438
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
497
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
564
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
646
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
699
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
Model\CodeGenTokenizer.cs (8)
354
splits =
InitializeForEncoding
(
367
splits =
InitializeForEncoding
(
611
splits =
InitializeForEncoding
(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);
615
splits =
InitializeForEncoding
(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);
830
splits =
InitializeForEncoding
(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);
834
splits =
InitializeForEncoding
(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);
967
splits =
InitializeForEncoding
(null, span, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);
971
splits =
InitializeForEncoding
(text, textSpan, considerPreTokenization, considerNormalization, _normalizer, _preTokenizer, out normalizedText, out textSpanToEncode, out _);
Model\EnglishRobertaTokenizer.cs (4)
321
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
422
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
509
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
556
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
Model\TiktokenTokenizer.cs (4)
265
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
375
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
545
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
671
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
Model\WordPieceTokenizer.cs (4)
279
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
402
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
554
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
616
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(