5 references to Unknown
Microsoft.ML.Tokenizers (5)
Model\SentencePieceUnigramModel.cs (5)
63
else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.
Unknown
)
66
_vocabReverse[i] = (modelProto.Pieces[i].Piece, modelProto.Pieces[i].Score, ModelProto.Types.SentencePiece.Types.Type.
Unknown
);
90
_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.
Unknown
);
197
_vocabReverse[i] = (piece, score, ModelProto.Types.SentencePiece.Types.Type.
Unknown
);
239
_vocabReverse[unkId] = (unkToken, 0f, ModelProto.Types.SentencePiece.Types.Type.
Unknown
);