17 references to InitializeForEncoding
Microsoft.ML.Tokenizers (17)
Model\BPETokenizer.cs (5)
320IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 369IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 425IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 502IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 550IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\EnglishRobertaTokenizer.cs (4)
321IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 422IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 509IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 556IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\TiktokenTokenizer.cs (4)
265IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 375IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 545IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 671IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\WordPieceTokenizer.cs (4)
279IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 402IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 554IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 616IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(