17 references to InitializeForEncoding
Microsoft.ML.Tokenizers (17)
Model\BPETokenizer.cs (5)
438
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
497
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
564
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
646
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
699
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
Model\EnglishRobertaTokenizer.cs (4)
321
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
422
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
509
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
556
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
Model\TiktokenTokenizer.cs (4)
265
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
375
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
545
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
671
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
Model\WordPieceTokenizer.cs (4)
279
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
402
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
554
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
616
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(