5 references to P50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
107
yield return new object[] {
P50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" };
349
IReadOnlyList<int> encoded =
P50kBase
.EncodeToIds(text);
350
int idsCount =
P50kBase
.CountTokens(text);
360
Assert.Equal(text,
P50kBase
.Decode(encoded));
361
TestDecodingWithSpan((
P50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);