5 references to P50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
108
yield return new object[] {
P50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" };
350
IReadOnlyList<int> encoded =
P50kBase
.EncodeToIds(text);
351
int idsCount =
P50kBase
.CountTokens(text);
361
Assert.Equal(text,
P50kBase
.Decode(encoded));
362
TestDecodingWithSpan((
P50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);