51TestGPT4TokenizationEncoding(GPT4); 54Assert.True(GPT4 is TiktokenTokenizer); 55IReadOnlyDictionary<string, int>? specialTokens = (GPT4 as TiktokenTokenizer)!.SpecialTokens; 70Tokenizer tokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, GPT4.PreTokenizer, null, specialTokens); 75tokenizer = TiktokenTokenizer.Create(stream, GPT4.PreTokenizer, null, specialTokens); 79tokenizer = await TiktokenTokenizer.CreateAsync(tokenizerDataFileName, GPT4.PreTokenizer, normalizer: null, specialTokens); 84tokenizer = await TiktokenTokenizer.CreateAsync(stream, GPT4.PreTokenizer, normalizer: null, specialTokens); 111yield return new object[] { GPT4, @"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" }; 202IReadOnlyList<int> encoded = GPT4.EncodeToIds(text); 204Assert.Equal(text, GPT4.Decode(encoded)); 205TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 207IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 208int idsCount = GPT4.CountTokens(text); 245IReadOnlyList<int> encoded = GPT4.EncodeToIds(text); 247Assert.Equal(text, GPT4.Decode(encoded)); 248TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 250IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 255int idsCount = GPT4.CountTokens(text); 266IReadOnlyList<int> encoded = GPT4.EncodeToIds(text); 269IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 270int idsCount = GPT4.CountTokens(text); 279IReadOnlyList<int> encoded = GPT4.EncodeToIds(text); 280int idsCount = GPT4.CountTokens(text); 282Assert.Equal(text, GPT4.Decode(encoded)); 283TestDecodingWithSpan((GPT4 as TiktokenTokenizer)!, encoded.ToArray(), text); 285IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 632TestTokenizerEncodingForTokenizer(GPT4, text, expectedTokens, expectedOffsets, expectedIds); 753IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out _); 759Assert.Equal(expectedIds, GPT4.EncodeToIds(text)); 760Assert.Equal(expectedIds.Length, GPT4.CountTokens(text)); 764int length = GPT4.GetIndexByTokenCount(text, tokenCount, out _, out int count); 782int index = GPT4.GetIndexByTokenCountFromEnd(text, tokenCount, out _, out count); 876IReadOnlyList<int> ids = GPT4.EncodeToIds(largeRepeatedInput); 877string decoded = GPT4.Decode(ids); 882IReadOnlyList<int> mixedIds = GPT4.EncodeToIds(largeMixedInput); 883string mixedDecoded = GPT4.Decode(mixedIds); 888IReadOnlyList<int> boundaryIds = GPT4.EncodeToIds(boundaryInput); 889string boundaryDecoded = GPT4.Decode(boundaryIds); 894IReadOnlyList<int> belowIds = GPT4.EncodeToIds(belowThresholdInput); 895string belowDecoded = GPT4.Decode(belowIds); 900IReadOnlyList<int> aboveIds = GPT4.EncodeToIds(aboveThresholdInput); 901string aboveDecoded = GPT4.Decode(aboveIds); 917IReadOnlyList<int> idsRepeated = GPT4.EncodeToIds(inputRepeated); 920string decodedRepeated = GPT4.Decode(idsRepeated); 925IReadOnlyList<int> idsMixed = GPT4.EncodeToIds(inputMixed); 926string decodedMixed = GPT4.Decode(idsMixed); 930IReadOnlyList<EncodedToken> tokens = GPT4.EncodeToTokens(inputRepeated, out string? normalizedText);