5 references to GPT2
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
101yield return new object[] { GPT2, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" }; // GPT2 uses the same encoding as R50kBase 322IReadOnlyList<int> encoded = GPT2.EncodeToIds(text); 323int idsCount = GPT2.CountTokens(text); 333Assert.Equal(text, GPT2.Decode(encoded)); 334TestDecodingWithSpan((GPT2 as TiktokenTokenizer)!, encoded.ToArray(), text);