1 write to EosId
Microsoft.ML.Tokenizers (1)
SentencepieceModel.cs (1)
370
result.
EosId
= SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
5 references to EosId
Microsoft.ML.Tokenizers (5)
Model\SentencePieceBaseModel.cs (1)
30
EndOfSentenceId = Math.Max(0, modelProto.TrainerSpec.
EosId
);
Model\SentencePieceUnigramModel.cs (4)
32
modelProto.TrainerSpec.
EosId
>= modelProto.Pieces.Count ||
79
Debug.Assert(modelProto.TrainerSpec.
EosId
>= 0);
83
_vocab[modelProto.TrainerSpec.EosPiece] = modelProto.TrainerSpec.
EosId
;
86
_vocabReverse[modelProto.TrainerSpec.
EosId
] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control);