7 references to GptOss
Microsoft.ML.Tokenizers.Tests (7)
TiktokenTests.cs (7)
296foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5, GPT5_1, GPT5_2, GPT6 })
842}, GptOss.SpecialTokens);
846IReadOnlyList<int> ids = GptOss.EncodeToIds(text);
851Assert.Equal(text, GptOss.Decode(ids));
854GptOss.EncodeToTokens(text, out _).Select(t => t.Value).ToArray());
857GptOss.EncodeToTokens(text, out _).Select(t => (t.Offset.Start.Value, t.Offset.End.Value)).ToList());
859Assert.Equal(ids, GptOss.EncodeToTokens(text, out _).Select(t => t.Id).ToList());