2 writes to TrainerSpec
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
4145
TrainerSpec
= new global::Sentencepiece.TrainerSpec();
4240
TrainerSpec
= new global::Sentencepiece.TrainerSpec();
22 references to TrainerSpec
Microsoft.ML.Tokenizers (22)
Model\LlamaTokenizer.cs (2)
47
if (modelProto.
TrainerSpec
.ModelType != TrainerSpec.Types.ModelType.Bpe)
61
modelProto.
TrainerSpec
.TreatWhitespaceAsSuffix,
Model\SentencePieceTokenizer.cs (12)
63
BeginningOfSentenceToken = modelProto.
TrainerSpec
.BosPiece ?? "<s>";
64
BeginningOfSentenceId = modelProto.
TrainerSpec
.BosId <= 0 ? 1 : modelProto.
TrainerSpec
.BosId;
65
EndOfSentenceToken = modelProto.
TrainerSpec
.EosPiece ?? "</s>";
66
EndOfSentenceId = modelProto.
TrainerSpec
.EosId <= 0 ? 1 : modelProto.
TrainerSpec
.EosId;
67
UnknownToken = modelProto.
TrainerSpec
.UnkPiece ?? "<unk>";
68
UnknownId = modelProto.
TrainerSpec
.UnkId < 0 ? 0 : modelProto.
TrainerSpec
.UnkId;
72
TreatWhitespaceAsSuffix = modelProto.
TrainerSpec
.TreatWhitespaceAsSuffix;
73
ByteFallback = modelProto.
TrainerSpec
.ByteFallback;
76
_normalizer = new SentencePieceNormalizer(modelProto.NormalizerSpec.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.
TrainerSpec
.TreatWhitespaceAsSuffix, specialTokens);
SentencepieceModel.cs (8)
4014
if (!object.Equals(
TrainerSpec
, other.
TrainerSpec
)) return false;
4029
if (trainerSpec_ != null) hash ^=
TrainerSpec
.GetHashCode();
4087
output.WriteMessage(
TrainerSpec
);
4116
size += 1 + pb::CodedOutputStream.ComputeMessageSize(
TrainerSpec
);
4147
TrainerSpec
.MergeFrom(other.
TrainerSpec
);
4242
input.ReadMessage(
TrainerSpec
);