13 references to InitializeForEncoding
Microsoft.ML.Tokenizers (13)
Model\BPETokenizer.cs (5)
275IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 324IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 380IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 457IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 505IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\EnglishRobertaTokenizer.cs (4)
310IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 411IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 498IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 545IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(
Model\TiktokenTokenizer.cs (4)
264IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 374IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 544IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding( 670IEnumerable<(int Offset, int Length)>? splits = InitializeForEncoding(