20 references to WriteStringField
Microsoft.ML.Tokenizers.Tests (20)
SentencePieceTests.cs (20)
225ts.WriteStringField(45, "[UNK]"); // unk_piece
226ts.WriteStringField(46, "[BOS]"); // bos_piece
227ts.WriteStringField(47, "[EOS]"); // eos_piece
228ts.WriteStringField(48, "[PAD]"); // pad_piece
247ns.WriteStringField(1, "identity"); // name
295ts.WriteStringField(47, "[EOS]"); // eos_piece (field 47 before field 3)
296ts.WriteStringField(46, "[BOS]"); // bos_piece
297ts.WriteStringField(45, "[UNK]"); // unk_piece
322ts.WriteStringField(45, "first"); // unk_piece = "first"
323ts.WriteStringField(45, "second"); // unk_piece = "second" (should win)
339piece.WriteStringField(1, "test");
352ns.WriteStringField(1, "identity");
450ts.WriteStringField(45, ""); // unk_piece = "" (zero-length)
526piece.WriteStringField(1, "test");
629ts.WriteStringField(200, "ext_value"); // extension field
696ts1.WriteStringField(45, "first"); // unk_piece = "first"
701ts2.WriteStringField(45, "second"); // unk_piece = "second"
724ts2.WriteStringField(45, "[UNK]"); // unk_piece (ONLY in second; no byte_fallback)
742piece.WriteStringField(1, "hello"); // only piece name, no score or type
810w.WriteStringField(1, text); // piece