2 instantiations of ModelProto
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
3878private static readonly pb::MessageParser<ModelProto> _parser = new pb::MessageParser<ModelProto>(() => new ModelProto()); 3921return new ModelProto(this);
70 references to ModelProto
Microsoft.ML.Tokenizers (70)
Model\LlamaTokenizer.cs (3)
19internal LlamaTokenizer(ModelProto modelProto, bool addBos, bool addEos, IReadOnlyDictionary<string, int>? addedTokens = null) : base(modelProto, addBos, addEos, addedTokens) 40ModelProto modelProto = ModelProto.Parser.ParseFrom(modelStream);
Model\SentencePieceBaseModel.cs (1)
18internal SentencePieceBaseModel(ModelProto modelProto, bool addBos = false, bool addEos = false, IReadOnlyDictionary<string, int>? specialTokens = null)
Model\SentencePieceBpeModel.cs (12)
26internal SentencePieceBpeModel(ModelProto modelProto, bool addBos, bool addEos, IReadOnlyDictionary<string, int>? specialTokens = null) : base(modelProto, addBos, addEos, specialTokens) 34if (piece.Type == ModelProto.Types.SentencePiece.Types.Type.Byte) 56_vocab.Add(new StringSpanOrdinalKey(item.Token), (id, item.Score, (byte)ModelProto.Types.SentencePiece.Types.Type.Normal)); 226if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 314if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 504if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 619if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 772if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 878if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 1048if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 1154if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 1254if (leftId.Type == (byte)ModelProto.Types.SentencePiece.Types.Type.Unused)
Model\SentencePieceTokenizer.cs (3)
23internal SentencePieceTokenizer(ModelProto modelProto, bool addBos, bool addEos, IReadOnlyDictionary<string, int>? specialTokens = null) 461ModelProto modelProto = ModelProto.Parser.ParseFrom(modelStream);
Model\SentencePieceUnigramModel.cs (17)
21private readonly (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[] _vocabReverse; 27public SentencePieceUnigramModel(ModelProto modelProto, bool addBos, bool addEos, IReadOnlyDictionary<string, int>? specialTokens = null) : base(modelProto, addBos, addEos, specialTokens) 30_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.Pieces.Count]; 37if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Normal || 38modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined || 39modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 48else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Byte) 52else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unknown) 55_vocabReverse[i] = (modelProto.Pieces[i].Piece, modelProto.Pieces[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown); 77_vocabReverse[modelProto.TrainerSpec.BosId] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 78_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 79_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 84_vocabReverse[modelProto.TrainerSpec.PadId] = (modelProto.TrainerSpec.PadPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 94List<(string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)> vocabReverse = new(250_000); 103vocabReverse.Add((item.Token, item.Score, ModelProto.Types.SentencePiece.Types.Type.Normal)); 556if (_vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 566float score = _vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ? length * _maxScore - 0.1f : _vocabReverse[ret].Score;
SentencepieceModel.cs (34)
104new pbr::GeneratedClrTypeInfo(typeof(global::Sentencepiece.ModelProto), global::Sentencepiece.ModelProto.Parser, new[]{ "Pieces", "TrainerSpec", "NormalizerSpec", "SelfTestData", "DenormalizerSpec" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Sentencepiece.ModelProto.Types.SentencePiece), global::Sentencepiece.ModelProto.Types.SentencePiece.Parser, new[]{ "Piece", "Score", "Type" }, null, new[]{ typeof(global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type) }, null, null)}) 3873internal sealed partial class ModelProto : pb::IExtendableMessage<ModelProto> 3878private static readonly pb::MessageParser<ModelProto> _parser = new pb::MessageParser<ModelProto>(() => new ModelProto()); 3880private pb::ExtensionSet<ModelProto> _extensions; 3881private pb::ExtensionSet<ModelProto> _Extensions { get { return _extensions; } } 3884public static pb::MessageParser<ModelProto> Parser { get { return _parser; } } 3908public ModelProto(ModelProto other) : this() { 3920public ModelProto Clone() { 3926private static readonly pb::FieldCodec<global::Sentencepiece.ModelProto.Types.SentencePiece> _repeated_pieces_codec 3927= pb::FieldCodec.ForMessage(10, global::Sentencepiece.ModelProto.Types.SentencePiece.Parser); 3928private readonly pbc::RepeatedField<global::Sentencepiece.ModelProto.Types.SentencePiece> pieces_ = new pbc::RepeatedField<global::Sentencepiece.ModelProto.Types.SentencePiece>(); 3934public pbc::RepeatedField<global::Sentencepiece.ModelProto.Types.SentencePiece> Pieces { 4001return Equals(other as ModelProto); 4006public bool Equals(ModelProto other) { 4138public void MergeFrom(ModelProto other) { 4271public TValue GetExtension<TValue>(pb::Extension<ModelProto, TValue> extension) { 4274public pbc::RepeatedField<TValue> GetExtension<TValue>(pb::RepeatedExtension<ModelProto, TValue> extension) { 4277public pbc::RepeatedField<TValue> GetOrInitializeExtension<TValue>(pb::RepeatedExtension<ModelProto, TValue> extension) { 4280public void SetExtension<TValue>(pb::Extension<ModelProto, TValue> extension, TValue value) { 4283public bool HasExtension<TValue>(pb::Extension<ModelProto, TValue> extension) { 4286public void ClearExtension<TValue>(pb::Extension<ModelProto, TValue> extension) { 4289public void ClearExtension<TValue>(pb::RepeatedExtension<ModelProto, TValue> extension) { 4316get { return global::Sentencepiece.ModelProto.Descriptor.NestedTypes[0]; } 4408private readonly static global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type TypeDefaultValue = global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type.Normal; 4410private global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type type_; 4413public global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type Type { 4624Type = (global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type) input.ReadEnum();