5 references to R50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
106
yield return new object[] {
R50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" };
382
IReadOnlyList<int> encoded =
R50kBase
.EncodeToIds(text);
383
int idsCount =
R50kBase
.CountTokens(text);
393
Assert.Equal(text,
R50kBase
.Decode(encoded));
394
TestDecodingWithSpan((
R50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);