2 instantiations of SentencePiece
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
4304private static readonly pb::MessageParser<SentencePiece> _parser = new pb::MessageParser<SentencePiece>(() => new SentencePiece()); 4347return new SentencePiece(this);
42 references to SentencePiece
Microsoft.ML.Tokenizers (42)
Model\SentencePieceBpeTokenizer.cs (11)
50var piece = modelProto.Pieces[i]; 54if (piece.Type == ModelProto.Types.SentencePiece.Types.Type.Byte) 344if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 432if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 693if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 808if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 1017if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 1123if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 1397if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 1503if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 2250if (leftId.Type == (byte)ModelProto.Types.SentencePiece.Types.Type.Unused)
SentencepieceModel.cs (31)
104new pbr::GeneratedClrTypeInfo(typeof(global::Sentencepiece.ModelProto), global::Sentencepiece.ModelProto.Parser, new[]{ "Pieces", "TrainerSpec", "NormalizerSpec", "SelfTestData", "DenormalizerSpec" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Sentencepiece.ModelProto.Types.SentencePiece), global::Sentencepiece.ModelProto.Types.SentencePiece.Parser, new[]{ "Piece", "Score", "Type" }, null, new[]{ typeof(global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type) }, null, null)}) 3926private static readonly pb::FieldCodec<global::Sentencepiece.ModelProto.Types.SentencePiece> _repeated_pieces_codec 3927= pb::FieldCodec.ForMessage(10, global::Sentencepiece.ModelProto.Types.SentencePiece.Parser); 3928private readonly pbc::RepeatedField<global::Sentencepiece.ModelProto.Types.SentencePiece> pieces_ = new pbc::RepeatedField<global::Sentencepiece.ModelProto.Types.SentencePiece>(); 3934public pbc::RepeatedField<global::Sentencepiece.ModelProto.Types.SentencePiece> Pieces { 4299public sealed partial class SentencePiece : pb::IExtendableMessage<SentencePiece> 4304private static readonly pb::MessageParser<SentencePiece> _parser = new pb::MessageParser<SentencePiece>(() => new SentencePiece()); 4306private pb::ExtensionSet<SentencePiece> _extensions; 4307private pb::ExtensionSet<SentencePiece> _Extensions { get { return _extensions; } } 4311public static pb::MessageParser<SentencePiece> Parser { get { return _parser; } } 4335public SentencePiece(SentencePiece other) : this() { 4346public SentencePiece Clone() { 4408private readonly static global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type TypeDefaultValue = global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type.Normal; 4410private global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type type_; 4413public global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type Type { 4436return Equals(other as SentencePiece); 4441public bool Equals(SentencePiece other) { 4555public void MergeFrom(SentencePiece other) { 4624Type = (global::Sentencepiece.ModelProto.Types.SentencePiece.Types.Type) input.ReadEnum(); 4632public TValue GetExtension<TValue>(pb::Extension<SentencePiece, TValue> extension) { 4635public pbc::RepeatedField<TValue> GetExtension<TValue>(pb::RepeatedExtension<SentencePiece, TValue> extension) { 4638public pbc::RepeatedField<TValue> GetOrInitializeExtension<TValue>(pb::RepeatedExtension<SentencePiece, TValue> extension) { 4641public void SetExtension<TValue>(pb::Extension<SentencePiece, TValue> extension, TValue value) { 4644public bool HasExtension<TValue>(pb::Extension<SentencePiece, TValue> extension) { 4647public void ClearExtension<TValue>(pb::Extension<SentencePiece, TValue> extension) { 4650public void ClearExtension<TValue>(pb::RepeatedExtension<SentencePiece, TValue> extension) {