11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
153TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
206TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
237TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
249TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
284TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
312TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
320TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
349TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
368TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
387TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
406TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);