1 write to Piece
Microsoft.ML.Tokenizers (1)
SentencepieceModel.cs (1)
249
result.
Piece
= SentencePieceProtobufReader.ReadString(data, end, ref pos);
4 references to Piece
Microsoft.ML.Tokenizers (4)
Model\SentencePieceBpeModel.cs (2)
31
_vocab.Add(new StringSpanOrdinalKey(piece.
Piece
), (i, piece.Score, (byte)piece.Type));
32
_vocabReverse.Add(i, piece.
Piece
);
Model\SentencePieceUnigramModel.cs (2)
49
string piece = modelProto.Pieces[i].
Piece
;
63
_vocabReverse[i] = (modelProto.Pieces[i].
Piece
, modelProto.Pieces[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown);