28 references to MakePiece
Microsoft.ML.Tokenizers.Tests (28)
SentencePieceTests.cs (28)
278MakePiece("<unk>", 0f, 2), // Unknown
279MakePiece("<s>", 0f, 3), // Control
280MakePiece("</s>", 0f, 3), // Control
281MakePiece("hello", -1.5f, 1), // Normal
282MakePiece("world", -2.0f, 4), // UserDefined
283MakePiece("<pad>", 0f, 5), // Unused
284MakePiece("<0x00>", 0f, 6)); // Byte
305model.WriteMessageField(1, MakePiece("test", -1f)); // piece
405MakePiece("a", -1f),
406MakePiece("b", -2f),
407MakePiece("c", -3f),
408MakePiece("ab", -0.5f),
409MakePiece("bc", -0.5f));
422MakePiece("\u2581", -1f), // ▁ (3-byte UTF-8)
423MakePiece("\u65E5\u672C\u8A9E", -2f), // 日本語 (CJK)
424MakePiece("\U0001F389", -3f), // 🎉 (4-byte UTF-8)
425MakePiece("caf\u00E9", -4f)); // café (Latin with diacritics)
472MakePiece("<unk>", 0f, 2), // Unknown (index 0)
473MakePiece("<s>", 0f, 3), // Control (index 1)
474MakePiece("</s>", 0f, 3), // Control (index 2)
475MakePiece("\u2581hello", -1.5f, 1), // Normal
476MakePiece("\u2581world", -2.0f, 1)); // Normal
492MakePiece("test", -1f));
651MakePiece("nan_score", float.NaN, 1),
652MakePiece("inf_score", float.PositiveInfinity, 1),
653MakePiece("neg_inf", float.NegativeInfinity, 1),
654MakePiece("subnormal", float.Epsilon, 1));
780MakePiece("test", -1f));