5 references to P50kBase
Microsoft.ML.Tokenizers.Tests (5)
TitokenTests.cs (5)
101
yield return new object[] {
P50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" };
340
IReadOnlyList<int> encoded =
P50kBase
.EncodeToIds(text);
341
int idsCount =
P50kBase
.CountTokens(text);
351
Assert.Equal(text,
P50kBase
.Decode(encoded));
352
TestDecodingWithSpan((
P50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);