33 references to UnknownToken
Microsoft.ML.Tokenizers.Tests (33)
BpeTests.cs (33)
25new Dictionary<string, int>() { { UnknownToken, 0 }, { "!", 5 }, { ",", 6 }, { ".", 7 }, { "B", 8 }, { "H", 9 }, { "T", 10 }, { "W", 11 }, { "a", 12 }, { "b", 13 }, { "c", 14 }, { "d", 15 }, { "e", 16 },
51new Dictionary<string, int>() { { "a", 1 }, { "b", 2 }, { UnknownToken, 3} },
54UnknownToken,
59new string[] { UnknownToken },
61UnknownToken,
96new Dictionary<string, int>() { { "a", 1 }, { "b", 2 }, { UnknownToken, 3} },
99UnknownToken,
104new string[] { "a", "b", UnknownToken },
106$"ab{UnknownToken}",
111new Dictionary<string, int>() { { "a", 1 }, { "b", 2 }, { UnknownToken, 3} },
114UnknownToken,
119new string[] { "a", "b", UnknownToken },
121$"ab{UnknownToken}",
126new Dictionary<string, int>() { { "a", 1 }, { "b", 2 }, { "ab", 3 }, { UnknownToken, 4} },
129UnknownToken,
134new string[] { "ab", UnknownToken },
136$"ab{UnknownToken}",
156new Dictionary<string, int>(){ { UnknownToken, 0} },
159UnknownToken,
164new string[] { UnknownToken, UnknownToken, UnknownToken },
166$"{UnknownToken}{UnknownToken}{UnknownToken}",
171new Dictionary<string, int>(){ { UnknownToken, 0} },
174UnknownToken,
179new string[] { UnknownToken },
181$"{UnknownToken}",
186new Dictionary<string, int>() { { UnknownToken, 0}, { "H", 1 }, { "e", 2 }, { "l", 3 }, { "o", 4 }, { "!", 5 }, { "He", 6 }, { "lo", 7}, { "llo", 8 },
190UnknownToken,
205UnknownToken,
220UnknownToken,