5 references to R50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
108
yield return new object[] {
R50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" };
387
IReadOnlyList<int> encoded =
R50kBase
.EncodeToIds(text);
388
int idsCount =
R50kBase
.CountTokens(text);
398
Assert.Equal(text,
R50kBase
.Decode(encoded));
399
TestDecodingWithSpan((
R50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);