2 writes to TrainerSpec
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
4145TrainerSpec = new global::Sentencepiece.TrainerSpec(); 4240TrainerSpec = new global::Sentencepiece.TrainerSpec();
40 references to TrainerSpec
Microsoft.ML.Tokenizers (40)
Model\LlamaTokenizer.cs (1)
47if (modelProto.TrainerSpec.ModelType != TrainerSpec.Types.ModelType.Bpe)
Model\SentencePieceBaseModel.cs (9)
27BeginningOfSentenceToken = modelProto.TrainerSpec.BosPiece ?? "<s>"; 28BeginningOfSentenceId = Math.Max(0, modelProto.TrainerSpec.BosId); 29EndOfSentenceToken = modelProto.TrainerSpec.EosPiece ?? "</s>"; 30EndOfSentenceId = Math.Max(0, modelProto.TrainerSpec.EosId); 31UnknownToken = modelProto.TrainerSpec.UnkPiece ?? "<unk>"; 32UnknownId = Math.Max(0, modelProto.TrainerSpec.UnkId); 35TreatWhitespaceAsSuffix = modelProto.TrainerSpec.TreatWhitespaceAsSuffix; 36ByteFallback = modelProto.TrainerSpec.ByteFallback; 58modelProto.TrainerSpec.TreatWhitespaceAsSuffix,
Model\SentencePieceTokenizer.cs (2)
25_model = modelProto.TrainerSpec.ModelType switch 29_ => throw new ArgumentException($"The model type '{modelProto.TrainerSpec.ModelType}' is not supported.", nameof(modelProto))
Model\SentencePieceUnigramModel.cs (20)
69Debug.Assert(modelProto.TrainerSpec.UnkId >= 0); 70Debug.Assert(modelProto.TrainerSpec.BosId >= 0); 71Debug.Assert(modelProto.TrainerSpec.EosId >= 0); 73_vocab[modelProto.TrainerSpec.UnkPiece] = modelProto.TrainerSpec.UnkId; 74_vocab[modelProto.TrainerSpec.BosPiece] = modelProto.TrainerSpec.BosId; 75_vocab[modelProto.TrainerSpec.EosPiece] = modelProto.TrainerSpec.EosId; 77_vocabReverse[modelProto.TrainerSpec.BosId] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 78_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 79_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 81if (modelProto.TrainerSpec.PadId >= 0) 83_vocab[modelProto.TrainerSpec.PadPiece] = modelProto.TrainerSpec.PadId; 84_vocabReverse[modelProto.TrainerSpec.PadId] = (modelProto.TrainerSpec.PadPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control);
SentencepieceModel.cs (8)
4014if (!object.Equals(TrainerSpec, other.TrainerSpec)) return false; 4029if (trainerSpec_ != null) hash ^= TrainerSpec.GetHashCode(); 4087output.WriteMessage(TrainerSpec); 4116size += 1 + pb::CodedOutputStream.ComputeMessageSize(TrainerSpec); 4147TrainerSpec.MergeFrom(other.TrainerSpec); 4242input.ReadMessage(TrainerSpec);