11 references to GPT4o
Microsoft.ML.Tokenizers.Tests (11)
TitokenTests.cs (11)
103
yield return new object[] {
GPT4o
, @"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" };
284
IReadOnlyList<int> encoded =
GPT4o
.EncodeToIds(text);
285
int idsCount =
GPT4o
.CountTokens(text);
296
Assert.Equal(text,
GPT4o
.Decode(encoded));
297
TestDecodingWithSpan((
GPT4o
as TiktokenTokenizer)!, encoded.ToArray(), text);
301
encoded =
GPT4o
.EncodeToIds(text);
302
idsCount =
GPT4o
.CountTokens(text);
304
Assert.Equal(text,
GPT4o
.Decode(encoded));
305
TestDecodingWithSpan((
GPT4o
as TiktokenTokenizer)!, encoded.ToArray(), text);
307
IReadOnlyList<EncodedToken> result =
GPT4o
.EncodeToTokens(text, out string? normalizedString);
314
TokenizerTests.TestTokenLimits(
GPT4o
);