5 references to R50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
114
yield return new object[] {
R50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" };
393
IReadOnlyList<int> encoded =
R50kBase
.EncodeToIds(text);
394
int idsCount =
R50kBase
.CountTokens(text);
404
Assert.Equal(text,
R50kBase
.Decode(encoded));
405
TestDecodingWithSpan((
R50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);