11 references to TestDecodingWithSpan
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
144TestDecodingWithSpan((tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
197TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
228TestDecodingWithSpan((gpt4Tokenizer as TiktokenTokenizer)!, encoded.ToArray(), text);
240TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
275TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
301TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text);
309TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text);
337TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);
356TestDecodingWithSpan((P50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);
375TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);
394TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);