11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
146TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
199TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
230TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
242TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
277TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
305TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
313TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
342TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
361TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
380TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
399TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);