11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
141TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
194TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
225TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
237TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
272TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
298TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text);
306TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text);
334TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
353TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
372TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
391TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);