11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
146TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text); 199TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 230TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text); 242TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 277TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 305TestDecodingWithSpan(tokenizer, encoded.ToArray(), text); 313TestDecodingWithSpan(tokenizer, encoded.ToArray(), text); 342TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text); 361TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text); 380TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text); 399TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);