13 references to JsonUnigramTokens
Microsoft.ML.Tokenizers.Tests (13)
UnigramTests.cs (13)
750Assert.Equal(new[] { "\u2581a", "\u2581b" }, JsonUnigramTokens(json, "a\u200Bb"));
859Assert.Equal(new[] { "\u2581Xab" }, JsonUnigramTokens(json, "ab"));
879Assert.Equal(new[] { "\u2581$0X" }, JsonUnigramTokens(json, "a"));
899Assert.Equal(new[] { "\u2581a", "\u2581,", "\u2581b" }, JsonUnigramTokens(json, "a,b"));
918Assert.Equal(new[] { "\u2581hello" }, JsonUnigramTokens(json, "HELLO"));
938Assert.Equal(new[] { "\u2581a", "\u2581b" }, JsonUnigramTokens(json, "A\u200BB"));
957Assert.Equal(new[] { "\u25811" }, JsonUnigramTokens(json, "\u2460"));
976Assert.Equal(new[] { "\u2581cafe" }, JsonUnigramTokens(json, "caf\u00e9"));
997Assert.Equal(new[] { "\u2581ab" }, JsonUnigramTokens(json, "a\uD834\uDD67b"));
1182Assert.Equal(new[] { "\u2581a", "\u2581b" }, JsonUnigramTokens(json, " a b "));
1213Assert.Equal(new[] { "\u25811" }, JsonUnigramTokens(json, "\uFF11"));
1214Assert.Equal(new[] { "\u2581a" }, JsonUnigramTokens(json, "A"));
1233string[] tokens = JsonUnigramTokens(json, "a\uD800b");