11 references to GPT4o
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
107yield return new object[] { GPT4o, @"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" }; 288IReadOnlyList<int> encoded = GPT4o.EncodeToIds(text); 289int idsCount = GPT4o.CountTokens(text); 300Assert.Equal(text, GPT4o.Decode(encoded)); 301TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text); 305encoded = GPT4o.EncodeToIds(text); 306idsCount = GPT4o.CountTokens(text); 308Assert.Equal(text, GPT4o.Decode(encoded)); 309TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text); 311IReadOnlyList<EncodedToken> result = GPT4o.EncodeToTokens(text, out string? normalizedText); 318TokenizerTests.TestTokenLimits(GPT4o);