2 writes to TrainerSpec
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
4145TrainerSpec = new global::Sentencepiece.TrainerSpec(); 4240TrainerSpec = new global::Sentencepiece.TrainerSpec();
43 references to TrainerSpec
Microsoft.ML.Tokenizers (43)
Model\LlamaTokenizer.cs (1)
47if (modelProto.TrainerSpec.ModelType != TrainerSpec.Types.ModelType.Bpe)
Model\SentencePieceBaseModel.cs (9)
27BeginningOfSentenceToken = modelProto.TrainerSpec.BosPiece ?? "<s>"; 28BeginningOfSentenceId = Math.Max(0, modelProto.TrainerSpec.BosId); 29EndOfSentenceToken = modelProto.TrainerSpec.EosPiece ?? "</s>"; 30EndOfSentenceId = Math.Max(0, modelProto.TrainerSpec.EosId); 31UnknownToken = modelProto.TrainerSpec.UnkPiece ?? "<unk>"; 32UnknownId = Math.Max(0, modelProto.TrainerSpec.UnkId); 35TreatWhitespaceAsSuffix = modelProto.TrainerSpec.TreatWhitespaceAsSuffix; 36ByteFallback = modelProto.TrainerSpec.ByteFallback; 58modelProto.TrainerSpec.TreatWhitespaceAsSuffix,
Model\SentencePieceTokenizer.cs (2)
25_model = modelProto.TrainerSpec.ModelType switch 29_ => throw new ArgumentException($"The model type '{modelProto.TrainerSpec.ModelType}' is not supported.", nameof(modelProto))
Model\SentencePieceUnigramModel.cs (23)
31if (modelProto.TrainerSpec.BosId >= modelProto.Pieces.Count || 32modelProto.TrainerSpec.EosId >= modelProto.Pieces.Count || 33modelProto.TrainerSpec.UnkId >= modelProto.Pieces.Count) 77Debug.Assert(modelProto.TrainerSpec.UnkId >= 0); 78Debug.Assert(modelProto.TrainerSpec.BosId >= 0); 79Debug.Assert(modelProto.TrainerSpec.EosId >= 0); 81_vocab[modelProto.TrainerSpec.UnkPiece] = modelProto.TrainerSpec.UnkId; 82_vocab[modelProto.TrainerSpec.BosPiece] = modelProto.TrainerSpec.BosId; 83_vocab[modelProto.TrainerSpec.EosPiece] = modelProto.TrainerSpec.EosId; 85_vocabReverse[modelProto.TrainerSpec.BosId] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 86_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 87_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 89if (modelProto.TrainerSpec.PadId >= 0) 91_vocab[modelProto.TrainerSpec.PadPiece] = modelProto.TrainerSpec.PadId; 92_vocabReverse[modelProto.TrainerSpec.PadId] = (modelProto.TrainerSpec.PadPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control);
SentencepieceModel.cs (8)
4014if (!object.Equals(TrainerSpec, other.TrainerSpec)) return false; 4029if (trainerSpec_ != null) hash ^= TrainerSpec.GetHashCode(); 4087output.WriteMessage(TrainerSpec); 4116size += 1 + pb::CodedOutputStream.ComputeMessageSize(TrainerSpec); 4147TrainerSpec.MergeFrom(other.TrainerSpec); 4242input.ReadMessage(TrainerSpec);