11 references to GPT4o
Microsoft.ML.Tokenizers.Tests (11)
TiktokenTests.cs (11)
104yield return new object[] { GPT4o, @"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" }; 285IReadOnlyList<int> encoded = GPT4o.EncodeToIds(text); 286int idsCount = GPT4o.CountTokens(text); 297Assert.Equal(text, GPT4o.Decode(encoded)); 298TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text); 302encoded = GPT4o.EncodeToIds(text); 303idsCount = GPT4o.CountTokens(text); 305Assert.Equal(text, GPT4o.Decode(encoded)); 306TestDecodingWithSpan((GPT4o as TiktokenTokenizer)!, encoded.ToArray(), text); 308IReadOnlyList<EncodedToken> result = GPT4o.EncodeToTokens(text, out string? normalizedText); 315TokenizerTests.TestTokenLimits(GPT4o);