5 references to R50kBase
Microsoft.ML.Tokenizers.Tests (5)
TitokenTests.cs (5)
102
yield return new object[] {
R50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" };
378
IReadOnlyList<int> encoded =
R50kBase
.EncodeToIds(text);
379
int idsCount =
R50kBase
.CountTokens(text);
389
Assert.Equal(text,
R50kBase
.Decode(encoded));
390
TestDecodingWithSpan((
R50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);