13 references to InitializeForEncoding
Microsoft.ML.Tokenizers (13)
Model\BPETokenizer.cs (5)
275
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
324
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
380
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
457
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
505
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
Model\EnglishRobertaTokenizer.cs (4)
310
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
411
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
498
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
545
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
Model\TiktokenTokenizer.cs (4)
264
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
374
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
544
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(
670
IEnumerable<(int Offset, int Length)>? splits =
InitializeForEncoding
(