11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
152TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
205TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
236TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
248TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
283TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
311TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
319TestDecodingWithSpan(tokenizer, encoded.ToArray(), text);
348TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
367TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
386TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
405TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);