33 references to UnknownToken
Microsoft.ML.Tokenizers.Tests (33)
BpeTests.cs (33)
27new Dictionary<string, int>() { { UnknownToken, 0 }, { "!", 5 }, { ",", 6 }, { ".", 7 }, { "B", 8 }, { "H", 9 }, { "T", 10 }, { "W", 11 }, { "a", 12 }, { "b", 13 }, { "c", 14 }, { "d", 15 }, { "e", 16 },
53new Dictionary<string, int>() { { "a", 1 }, { "b", 2 }, { UnknownToken, 3} },
56UnknownToken,
61new string[] { UnknownToken },
63UnknownToken,
98new Dictionary<string, int>() { { "a", 1 }, { "b", 2 }, { UnknownToken, 3} },
101UnknownToken,
106new string[] { "a", "b", UnknownToken },
108$"ab{UnknownToken}",
113new Dictionary<string, int>() { { "a", 1 }, { "b", 2 }, { UnknownToken, 3} },
116UnknownToken,
121new string[] { "a", "b", UnknownToken },
123$"ab{UnknownToken}",
128new Dictionary<string, int>() { { "a", 1 }, { "b", 2 }, { "ab", 3 }, { UnknownToken, 4} },
131UnknownToken,
136new string[] { "ab", UnknownToken },
138$"ab{UnknownToken}",
158new Dictionary<string, int>(){ { UnknownToken, 0} },
161UnknownToken,
166new string[] { UnknownToken, UnknownToken, UnknownToken },
168$"{UnknownToken}{UnknownToken}{UnknownToken}",
173new Dictionary<string, int>(){ { UnknownToken, 0} },
176UnknownToken,
181new string[] { UnknownToken },
183$"{UnknownToken}",
188new Dictionary<string, int>() { { UnknownToken, 0}, { "H", 1 }, { "e", 2 }, { "l", 3 }, { "o", 4 }, { "!", 5 }, { "He", 6 }, { "lo", 7}, { "llo", 8 },
192UnknownToken,
207UnknownToken,
222UnknownToken,