4 references to Phi4
Microsoft.ML.Tokenizers.Tests (4)
TiktokenTests.cs (4)
49
TestGPT4TokenizationEncoding(
Phi4
);
621
TestTokenizerEncodingForTokenizer(
Phi4
, text, expectedTokens, expectedOffsets, expectedIds);
794
IReadOnlyList<int> encoded =
Phi4
.EncodeToIds(text);
796
Assert.Equal(text,
Phi4
.Decode(encoded));