2 writes to EosPiece
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
2363EosPiece = other.EosPiece; 2749EosPiece = input.ReadString();
9 references to EosPiece
Microsoft.ML.Tokenizers (9)
Model\SentencePieceBaseModel.cs (1)
29EndOfSentenceToken = modelProto.TrainerSpec.EosPiece ?? "</s>";
Model\SentencePieceUnigramModel.cs (2)
75_vocab[modelProto.TrainerSpec.EosPiece] = modelProto.TrainerSpec.EosId; 78_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control);
SentencepieceModel.cs (6)
1650if (EosPiece != other.EosPiece) return false; 1706if (HasEosPiece) hash ^= EosPiece.GetHashCode(); 2059output.WriteString(EosPiece); 2218size += 2 + pb::CodedOutputStream.ComputeStringSize(EosPiece); 2363EosPiece = other.EosPiece;