1 write to UnkPiece
Microsoft.ML.Tokenizers (1)
SentencepieceModel.cs (1)
378
result.
UnkPiece
= SentencePieceProtobufReader.ReadString(data, end, ref pos);
3 references to UnkPiece
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBaseModel.cs (1)
31
UnknownToken = modelProto.TrainerSpec.
UnkPiece
?? "<unk>";
Model\SentencePieceUnigramModel.cs (2)
81
_vocab[modelProto.TrainerSpec.
UnkPiece
] = modelProto.TrainerSpec.UnkId;
87
_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.
UnkPiece
, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown);