33 references to Type
Microsoft.ML.Tokenizers (33)
Model\SentencePieceBpeModel.cs (10)
34if (piece.Type == ModelProto.Types.SentencePiece.Types.Type.Byte) 180if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 268if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 458if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 573if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 726if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 832if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 1002if (type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused) 1108if (id.Type != (byte)ModelProto.Types.SentencePiece.Types.Type.Unused || 1208if (leftId.Type == (byte)ModelProto.Types.SentencePiece.Types.Type.Unused)
Model\SentencePieceUnigramModel.cs (20)
22private readonly (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[] _vocabReverse; 41_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[modelProto.Pieces.Count]; 48if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Normal || 49modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined || 50modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 59else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Byte) 63else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unknown) 66_vocabReverse[i] = (modelProto.Pieces[i].Piece, modelProto.Pieces[i].Score, ModelProto.Types.SentencePiece.Types.Type.Unknown); 88_vocabReverse[modelProto.TrainerSpec.BosId] = (modelProto.TrainerSpec.BosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 89_vocabReverse[modelProto.TrainerSpec.EosId] = (modelProto.TrainerSpec.EosPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 90_vocabReverse[modelProto.TrainerSpec.UnkId] = (modelProto.TrainerSpec.UnkPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 95_vocabReverse[modelProto.TrainerSpec.PadId] = (modelProto.TrainerSpec.PadPiece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 174_vocabReverse = new (string Piece, float Score, ModelProto.Types.SentencePiece.Types.Type Type)[pieces!.Count]; 197_vocabReverse[i] = (piece, score, ModelProto.Types.SentencePiece.Types.Type.Unknown); 201_vocabReverse[i] = (piece, score, ModelProto.Types.SentencePiece.Types.Type.Control); 205_vocabReverse[i] = (piece, score, ModelProto.Types.SentencePiece.Types.Type.Normal); 239_vocabReverse[unkId] = (unkToken, 0f, ModelProto.Types.SentencePiece.Types.Type.Unknown); 253_vocabReverse[controlId] = (piece, 0f, ModelProto.Types.SentencePiece.Types.Type.Control); 781if (_vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 791float score = _vocabReverse[ret].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined ? length * _maxScore - 0.1f : _vocabReverse[ret].Score;
SentencepieceModel.cs (3)
234internal Types.Type Type { get; set; } = Types.Type.Normal; 257result.Type = (Types.Type)SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);