1 write to EosId
Microsoft.ML.Tokenizers (1)
SentencepieceModel.cs (1)
370
result.
EosId
= SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
5 references to EosId
Microsoft.ML.Tokenizers (5)
Model\SentencePieceBaseModel.cs (1)
30
EndOfSentenceId = Math.Max(0, modelProto.TrainerSpec.
EosId
);
Model\SentencePieceUnigramModel.cs (4)
35
modelProto.TrainerSpec.
EosId
>= modelProto.Pieces.Count ||
82
Debug.Assert(modelProto.TrainerSpec.
EosId
>= 0);
86
_vocab[modelProto.TrainerSpec.EosPiece] = modelProto.TrainerSpec.
EosId
;
89
_vocabReverse[modelProto.TrainerSpec.
EosId
] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control);