1 write to UnkId
Microsoft.ML.Tokenizers (1)
SentencepieceModel.cs (1)
362
result.
UnkId
= SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
5 references to UnkId
Microsoft.ML.Tokenizers (5)
Model\SentencePieceBaseModel.cs (1)
32
UnknownId = Math.Max(0, modelProto.TrainerSpec.
UnkId
);
Model\SentencePieceUnigramModel.cs (4)
33
modelProto.TrainerSpec.
UnkId
>= modelProto.Pieces.Count)
77
Debug.Assert(modelProto.TrainerSpec.
UnkId
>= 0);
81
_vocab[modelProto.TrainerSpec.UnkPiece] = modelProto.TrainerSpec.
UnkId
;
87
_vocabReverse[modelProto.TrainerSpec.
UnkId
] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown);