4 references to Phi4
Microsoft.ML.Tokenizers.Tests (4)
TiktokenTests.cs (4)
46TestGPT4TokenizationEncoding(Phi4); 609TestTokenizerEncodingForTokenizer(Phi4, text, expectedTokens, expectedOffsets, expectedIds); 782IReadOnlyList<int> encoded = Phi4.EncodeToIds(text); 784Assert.Equal(text, Phi4.Decode(encoded));