1 write to BosId
Microsoft.ML.Tokenizers (1)
SentencepieceModel.cs (1)
366
result.
BosId
= SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
5 references to BosId
Microsoft.ML.Tokenizers (5)
Model\SentencePieceBaseModel.cs (1)
28
BeginningOfSentenceId = Math.Max(0, modelProto.TrainerSpec.
BosId
);
Model\SentencePieceUnigramModel.cs (4)
31
if (modelProto.TrainerSpec.
BosId
>= modelProto.Pieces.Count ||
78
Debug.Assert(modelProto.TrainerSpec.
BosId
>= 0);
82
_vocab[modelProto.TrainerSpec.BosPiece] = modelProto.TrainerSpec.
BosId
;
85
_vocabReverse[modelProto.TrainerSpec.
BosId
] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control);