2 writes to UnkPiece
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
2357UnkPiece = other.UnkPiece; 2741UnkPiece = input.ReadString();
9 references to UnkPiece
Microsoft.ML.Tokenizers (9)
Model\SentencePieceBaseModel.cs (1)
31UnknownToken = modelProto.TrainerSpec.UnkPiece ?? "<unk>";
Model\SentencePieceUnigramModel.cs (2)
73_vocab[modelProto.TrainerSpec.UnkPiece] = modelProto.TrainerSpec.UnkId; 79_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown);
SentencepieceModel.cs (6)
1648if (UnkPiece != other.UnkPiece) return false; 1704if (HasUnkPiece) hash ^= UnkPiece.GetHashCode(); 2051output.WriteString(UnkPiece); 2212size += 2 + pb::CodedOutputStream.ComputeStringSize(UnkPiece); 2357UnkPiece = other.UnkPiece;