11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
148TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
201TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
232TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
244TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
279TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
307TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
315TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
344TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
363TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
382TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
401TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);