32 references to GPT4
Microsoft.ML.Tokenizers.Tests (32)
TiktokenTests.cs (32)
47TestGPT4TokenizationEncoding(GPT4);
50Assert.True(GPT4 is TiktokenTokenizer);
51IReadOnlyDictionary<string, int>? specialTokens = (GPT4 as TiktokenTokenizer)!.SpecialTokens;
66Tokenizer tokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, GPT4.PreTokenizer, null, specialTokens);
71tokenizer = TiktokenTokenizer.Create(stream, GPT4.PreTokenizer, null, specialTokens);
75tokenizer = await TiktokenTokenizer.CreateAsync(tokenizerDataFileName, GPT4.PreTokenizer, normalizer: null, specialTokens);
80tokenizer = await TiktokenTokenizer.CreateAsync(stream, GPT4.PreTokenizer, normalizer: null, specialTokens);
107yield return new object[] { GPT4, @"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" };
198IReadOnlyList<int> encoded = GPT4.EncodeToIds(text);
200Assert.Equal(text, GPT4.Decode(encoded));
201TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
203IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText);
204int idsCount = GPT4.CountTokens(text);
241IReadOnlyList<int> encoded = GPT4.EncodeToIds(text);
243Assert.Equal(text, GPT4.Decode(encoded));
244TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
246IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText);
251int idsCount = GPT4.CountTokens(text);
262IReadOnlyList<int> encoded = GPT4.EncodeToIds(text);
265IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText);
266int idsCount = GPT4.CountTokens(text);
275IReadOnlyList<int> encoded = GPT4.EncodeToIds(text);
276int idsCount = GPT4.CountTokens(text);
278Assert.Equal(text, GPT4.Decode(encoded));
279TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text);
281IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText);
616TestTokenizerEncodingForTokenizer(GPT4, text, expectedTokens, expectedOffsets, expectedIds);
737IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out _);
743Assert.Equal(expectedIds, GPT4.EncodeToIds(text));
744Assert.Equal(expectedIds.Length, GPT4.CountTokens(text));
748int length = GPT4.GetIndexByTokenCount(text, tokenCount, out _, out int count);
766int index = GPT4.GetIndexByTokenCountFromEnd(text, tokenCount, out _, out count);