11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
147TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
200TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
231TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
243TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
278TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
306TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
314TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
343TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
362TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
381TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
400TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);