4 references to P50kEdit
Microsoft.ML.Tokenizers.Tests (4)
TiktokenTests.cs (4)
360
IReadOnlyList<int> encoded =
P50kEdit
.EncodeToIds(text);
361
int idsCount =
P50kEdit
.CountTokens(text);
371
Assert.Equal(text,
P50kEdit
.Decode(encoded));
372
TestDecodingWithSpan((
P50kEdit
as TiktokenTokenizer)!, encoded.ToArray(), text);