18 references to Pieces
Microsoft.ML.Tokenizers (18)
Model\SentencePieceBpeModel.cs (2)
28
for (int i = 0; i < modelProto.
Pieces
.Count; i++)
30
var piece = modelProto.
Pieces
[i];
Model\SentencePieceUnigramModel.cs (15)
34
if (modelProto.TrainerSpec.BosId >= modelProto.
Pieces
.Count ||
35
modelProto.TrainerSpec.EosId >= modelProto.
Pieces
.Count ||
36
modelProto.TrainerSpec.UnkId >= modelProto.
Pieces
.Count)
41
_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.
Pieces
.Count];
46
for (int i = 0; i < modelProto.
Pieces
.Count; i++)
48
if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Normal ||
49
modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ||
50
modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unused)
52
string piece = modelProto.
Pieces
[i].Piece;
53
float score = modelProto.
Pieces
[i].Score;
54
_vocabReverse[i] = (piece, score, modelProto.
Pieces
[i].Type);
59
else if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Byte)
63
else if (modelProto.
Pieces
[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unknown)
66
_vocabReverse[i] = (modelProto.
Pieces
[i].Piece, modelProto.
Pieces
[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown);
SentencepieceModel.cs (1)
203
result.
Pieces
.Add(Types.SentencePiece.Parse(data, pos, pos + length));