5 references to GPT2
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
104
yield return new object[] {
GPT2
, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" }; // GPT2 uses the same encoding as R50kBase
325
IReadOnlyList<int> encoded =
GPT2
.EncodeToIds(text);
326
int idsCount =
GPT2
.CountTokens(text);
336
Assert.Equal(text,
GPT2
.Decode(encoded));
337
TestDecodingWithSpan((
GPT2
as TiktokenTokenizer)!, encoded.ToArray(), text);