5 references to P50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
102
yield return new object[] {
P50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" };
341
IReadOnlyList<int> encoded =
P50kBase
.EncodeToIds(text);
342
int idsCount =
P50kBase
.CountTokens(text);
352
Assert.Equal(text,
P50kBase
.Decode(encoded));
353
TestDecodingWithSpan((
P50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);