4 references to P50kEdit
Microsoft.ML.Tokenizers.Tests (4)
TiktokenTests.cs (4)
360IReadOnlyList<int> encoded = P50kEdit.EncodeToIds(text); 361int idsCount = P50kEdit.CountTokens(text); 371Assert.Equal(text, P50kEdit.Decode(encoded)); 372TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);