5 references to R50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
108yield return new object[] { R50kBase, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" }; 387IReadOnlyList<int> encoded = R50kBase.EncodeToIds(text); 388int idsCount = R50kBase.CountTokens(text); 398Assert.Equal(text, R50kBase.Decode(encoded)); 399TestDecodingWithSpan((R50kBase as TiktokenTokenizer)!, encoded.ToArray(), text);