5 references to R50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
103
yield return new object[] {
R50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" };
379
IReadOnlyList<int> encoded =
R50kBase
.EncodeToIds(text);
380
int idsCount =
R50kBase
.CountTokens(text);
390
Assert.Equal(text,
R50kBase
.Decode(encoded));
391
TestDecodingWithSpan((
R50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);