2 writes to TrainerSpec
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
4145TrainerSpec = new global::Sentencepiece.TrainerSpec(); 4240TrainerSpec = new global::Sentencepiece.TrainerSpec();
22 references to TrainerSpec
Microsoft.ML.Tokenizers (22)
Model\LlamaTokenizer.cs (2)
47if (modelProto.TrainerSpec.ModelType != TrainerSpec.Types.ModelType.Bpe) 61modelProto.TrainerSpec.TreatWhitespaceAsSuffix,
Model\SentencePieceTokenizer.cs (12)
63BeginningOfSentenceToken = modelProto.TrainerSpec.BosPiece ?? "<s>"; 64BeginningOfSentenceId = modelProto.TrainerSpec.BosId <= 0 ? 1 : modelProto.TrainerSpec.BosId; 65EndOfSentenceToken = modelProto.TrainerSpec.EosPiece ?? "</s>"; 66EndOfSentenceId = modelProto.TrainerSpec.EosId <= 0 ? 1 : modelProto.TrainerSpec.EosId; 67UnknownToken = modelProto.TrainerSpec.UnkPiece ?? "<unk>"; 68UnknownId = modelProto.TrainerSpec.UnkId < 0 ? 0 : modelProto.TrainerSpec.UnkId; 72TreatWhitespaceAsSuffix = modelProto.TrainerSpec.TreatWhitespaceAsSuffix; 73ByteFallback = modelProto.TrainerSpec.ByteFallback; 76_normalizer = new SentencePieceNormalizer(modelProto.NormalizerSpec.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.TrainerSpec.TreatWhitespaceAsSuffix, specialTokens);
SentencepieceModel.cs (8)
4014if (!object.Equals(TrainerSpec, other.TrainerSpec)) return false; 4029if (trainerSpec_ != null) hash ^= TrainerSpec.GetHashCode(); 4087output.WriteMessage(TrainerSpec); 4116size += 1 + pb::CodedOutputStream.ComputeMessageSize(TrainerSpec); 4147TrainerSpec.MergeFrom(other.TrainerSpec); 4242input.ReadMessage(TrainerSpec);