14 references to Pieces
Microsoft.ML.Tokenizers (14)
Model\SentencePieceBpeModel.cs (2)
28
for (int i = 0; i < modelProto.
Pieces
.Count; i++)
30
var piece = modelProto.
Pieces
[i];
Model\SentencePieceUnigramModel.cs (12)
30
_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.
Pieces
.Count];
35
for (int i = 0; i < modelProto.
Pieces
.Count; i++)
37
if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Normal ||
38
modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ||
39
modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unused)
41
string piece = modelProto.
Pieces
[i].Piece;
42
float score = modelProto.
Pieces
[i].Score;
43
_vocabReverse[i] = (piece, score, modelProto.
Pieces
[i].Type);
48
else if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Byte)
52
else if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unknown)
55
_vocabReverse[i] = (modelProto.
Pieces
[i].Piece, modelProto.
Pieces
[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown);