1 write to Type
Microsoft.ML.Tokenizers (1)
SentencepieceModel.cs (1)
257result.Type = (Types.Type)SentencePieceProtobufReader.ReadRawVarint32(data, end, ref pos);
8 references to Type
Microsoft.ML.Tokenizers (8)
Model\SentencePieceBpeModel.cs (2)
31_vocab.Add(new StringSpanOrdinalKey(piece.Piece), (i, piece.Score, (byte)piece.Type)); 34if (piece.Type == ModelProto.Types.SentencePiece.Types.Type.Byte)
Model\SentencePieceUnigramModel.cs (6)
45if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Normal || 46modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.UserDefined || 47modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unused) 51_vocabReverse[i] = (piece, score, modelProto.Pieces[i].Type); 56else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Byte) 60else if (modelProto.Pieces[i].Type == ModelProto.Types.SentencePiece.Types.Type.Unknown)