11 references to GPT4o
Microsoft.ML.Tokenizers.Tests (11)
TitokenTests.cs (11)
103yield return new object[] { GPT4o, @"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" }; 284IReadOnlyList<int> encoded = GPT4o.EncodeToIds(text); 285int idsCount = GPT4o.CountTokens(text); 296Assert.Equal(text, GPT4o.Decode(encoded)); 297TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text); 301encoded = GPT4o.EncodeToIds(text); 302idsCount = GPT4o.CountTokens(text); 304Assert.Equal(text, GPT4o.Decode(encoded)); 305TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text); 307IReadOnlyList<EncodedToken> result = GPT4o.EncodeToTokens(text, out string? normalizedString); 314TokenizerTests.TestTokenLimits(GPT4o);