11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
149TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
202TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
233TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
245TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
280TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
308TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
316TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
345TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
364TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
383TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
402TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);