11 references to GPT4o
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
104
yield return new object[] {
GPT4o
, @"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" };
285
IReadOnlyList<int> encoded =
GPT4o
.EncodeToIds(text);
286
int idsCount =
GPT4o
.CountTokens(text);
297
Assert.Equal(text,
GPT4o
.Decode(encoded));
298
TestDecodingWithSpan((
GPT4o
as TiktokenTokenizer)!, encoded.ToArray(), text);
302
encoded =
GPT4o
.EncodeToIds(text);
303
idsCount =
GPT4o
.CountTokens(text);
305
Assert.Equal(text,
GPT4o
.Decode(encoded));
306
TestDecodingWithSpan((
GPT4o
as TiktokenTokenizer)!, encoded.ToArray(), text);
308
IReadOnlyList<EncodedToken> result =
GPT4o
.EncodeToTokens(text, out string? normalizedText);
315
TokenizerTests.TestTokenLimits(
GPT4o
);