11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
150TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
203TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
234TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
246TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
281TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
309TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
317TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
346TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
365TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
384TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
403TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);