17 references to InitializeForEncoding
Microsoft.ML.Tokenizers (17)
Model\BPETokenizer.cs (5)
438IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 497IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 564IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 646IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 699IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\EnglishRobertaTokenizer.cs (4)
321IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 422IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 509IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 556IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\TiktokenTokenizer.cs (4)
265IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 375IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 545IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 671IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\WordPieceTokenizer.cs (4)
279IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 402IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 554IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 616IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(