17 references to Pieces
Microsoft.ML.Tokenizers (17)
Model\SentencePieceBpeModel.cs (2)
28
for (int i = 0; i < modelProto.
Pieces
.Count; i++)
30
var piece = modelProto.
Pieces
[i];
Model\SentencePieceUnigramModel.cs (15)
31
if (modelProto.TrainerSpec.BosId >= modelProto.
Pieces
.Count ||
32
modelProto.TrainerSpec.EosId >= modelProto.
Pieces
.Count ||
33
modelProto.TrainerSpec.UnkId >= modelProto.
Pieces
.Count)
38
_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.
Pieces
.Count];
43
for (int i = 0; i < modelProto.
Pieces
.Count; i++)
45
if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Normal ||
46
modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ||
47
modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unused)
49
string piece = modelProto.
Pieces
[i].Piece;
50
float score = modelProto.
Pieces
[i].Score;
51
_vocabReverse[i] = (piece, score, modelProto.
Pieces
[i].Type);
56
else if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Byte)
60
else if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unknown)
63
_vocabReverse[i] = (modelProto.
Pieces
[i].Piece, modelProto.
Pieces
[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown);