4 references to P50kEdit
Microsoft.ML.Tokenizers.Tests (4)
TiktokenTests.cs (4)
368IReadOnlyList<int> encoded = P50kEdit.EncodeToIds(text); 369int idsCount = P50kEdit.CountTokens(text); 379Assert.Equal(text, P50kEdit.Decode(encoded)); 380TestDecodingWithSpan((P50kEdit as TiktokenTokenizer)!, encoded.ToArray(), text);