52TestGPT4TokenizationEncoding(GPT4); 55Assert.True(GPT4 is TiktokenTokenizer); 56IReadOnlyDictionary<string, int>? specialTokens = (GPT4 as TiktokenTokenizer)!.SpecialTokens; 71Tokenizer tokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, GPT4.PreTokenizer, null, specialTokens); 76tokenizer = TiktokenTokenizer.Create(stream, GPT4.PreTokenizer, null, specialTokens); 80tokenizer = await TiktokenTokenizer.CreateAsync(tokenizerDataFileName, GPT4.PreTokenizer, normalizer: null, specialTokens); 85tokenizer = await TiktokenTokenizer.CreateAsync(stream, GPT4.PreTokenizer, normalizer: null, specialTokens); 112yield return new object[] { GPT4, @"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" }; 203IReadOnlyList<int> encoded = GPT4.EncodeToIds(text); 205Assert.Equal(text, GPT4.Decode(encoded)); 206TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 208IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 209int idsCount = GPT4.CountTokens(text); 246IReadOnlyList<int> encoded = GPT4.EncodeToIds(text); 248Assert.Equal(text, GPT4.Decode(encoded)); 249TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 251IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 256int idsCount = GPT4.CountTokens(text); 267IReadOnlyList<int> encoded = GPT4.EncodeToIds(text); 270IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 271int idsCount = GPT4.CountTokens(text); 280IReadOnlyList<int> encoded = GPT4.EncodeToIds(text); 281int idsCount = GPT4.CountTokens(text); 283Assert.Equal(text, GPT4.Decode(encoded)); 284TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 286IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 636TestTokenizerEncodingForTokenizer(GPT4, text, expectedTokens, expectedOffsets, expectedIds); 757IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out _); 763Assert.Equal(expectedIds, GPT4.EncodeToIds(text)); 764Assert.Equal(expectedIds.Length, GPT4.CountTokens(text)); 768int length = GPT4.GetIndexByTokenCount(text, tokenCount, out _, out int count); 786int index = GPT4.GetIndexByTokenCountFromEnd(text, tokenCount, out _, out count); 880IReadOnlyList<int> ids = GPT4.EncodeToIds(largeRepeatedInput); 881string decoded = GPT4.Decode(ids); 886IReadOnlyList<int> mixedIds = GPT4.EncodeToIds(largeMixedInput); 887string mixedDecoded = GPT4.Decode(mixedIds); 892IReadOnlyList<int> boundaryIds = GPT4.EncodeToIds(boundaryInput); 893string boundaryDecoded = GPT4.Decode(boundaryIds); 898IReadOnlyList<int> belowIds = GPT4.EncodeToIds(belowThresholdInput); 899string belowDecoded = GPT4.Decode(belowIds); 904IReadOnlyList<int> aboveIds = GPT4.EncodeToIds(aboveThresholdInput); 905string aboveDecoded = GPT4.Decode(aboveIds); 921IReadOnlyList<int> idsRepeated = GPT4.EncodeToIds(inputRepeated); 924string decodedRepeated = GPT4.Decode(idsRepeated); 929IReadOnlyList<int> idsMixed = GPT4.EncodeToIds(inputMixed); 930string decodedMixed = GPT4.Decode(idsMixed); 934IReadOnlyList<EncodedToken> tokens = GPT4.EncodeToTokens(inputRepeated, out string? normalizedText);