5 references to P50kBase
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
105
yield return new object[] {
P50kBase
, @"https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" };
344
IReadOnlyList<int> encoded =
P50kBase
.EncodeToIds(text);
345
int idsCount =
P50kBase
.CountTokens(text);
355
Assert.Equal(text,
P50kBase
.Decode(encoded));
356
TestDecodingWithSpan((
P50kBase
as TiktokenTokenizer)!, encoded.ToArray(), text);