From f42338c30b051a95f8e30674b9b9a372555d654e Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 10 Sep 2020 15:13:28 -0700 Subject: [PATCH] Add back multi-sig support and fix the multi-sig check for harmony nodes (#3337) * Revert "Revert "Add multi-sig merged messaging logic (#3300)"" This reverts commit cbd11331b46d62112f1457a2d1c23c0641451d66. * Allow harmony nodes to sign multisig from multiple accounts --- api/proto/message/message.pb.go | 149 +++++++------- api/proto/message/message.proto | 1 + consensus/checks.go | 14 +- consensus/consensus.go | 5 +- consensus/consensus_service.go | 20 +- consensus/consensus_service_test.go | 2 +- consensus/consensus_v2.go | 16 +- consensus/construct.go | 81 ++++++-- consensus/construct_test.go | 8 +- consensus/double_sign.go | 185 ++++++++++-------- consensus/fbft_log.go | 71 ++++--- consensus/leader.go | 67 ++++--- consensus/quorum/one-node-one-vote.go | 9 +- consensus/quorum/one-node-staked-vote.go | 31 ++- consensus/quorum/one-node-staked-vote_test.go | 2 +- consensus/quorum/quorum.go | 56 ++++-- consensus/threshold.go | 5 +- consensus/validator.go | 77 ++++++-- consensus/view_change.go | 61 ++++-- consensus/votepower/roster.go | 15 +- crypto/bls/mask.go | 57 +++++- crypto/bls/mask_test.go | 6 + crypto/pki/utils.go | 20 -- crypto/pki/utils_test.go | 21 -- node/node.go | 50 +++-- node/node_explorer.go | 4 +- staking/slash/double-sign.go | 116 +++++++---- staking/slash/double-sign_test.go | 25 +-- staking/slash/test/copy.go | 2 +- staking/slash/test/copy_test.go | 4 +- 30 files changed, 765 insertions(+), 415 deletions(-) delete mode 100644 crypto/pki/utils.go delete mode 100644 crypto/pki/utils_test.go diff --git a/api/proto/message/message.pb.go b/api/proto/message/message.pb.go index 4e01f75b9..601bce19f 100644 --- a/api/proto/message/message.pb.go +++ b/api/proto/message/message.pb.go @@ -650,13 +650,14 @@ type ConsensusRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ViewId uint64 `protobuf:"varint,1,opt,name=view_id,json=viewId,proto3" json:"view_id,omitempty"` - BlockNum uint64 `protobuf:"varint,2,opt,name=block_num,json=blockNum,proto3" json:"block_num,omitempty"` - ShardId uint32 `protobuf:"varint,3,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` - BlockHash []byte `protobuf:"bytes,4,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` - Block []byte `protobuf:"bytes,5,opt,name=block,proto3" json:"block,omitempty"` - SenderPubkey []byte `protobuf:"bytes,6,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` - Payload []byte `protobuf:"bytes,7,opt,name=payload,proto3" json:"payload,omitempty"` + ViewId uint64 `protobuf:"varint,1,opt,name=view_id,json=viewId,proto3" json:"view_id,omitempty"` + BlockNum uint64 `protobuf:"varint,2,opt,name=block_num,json=blockNum,proto3" json:"block_num,omitempty"` + ShardId uint32 `protobuf:"varint,3,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` + BlockHash []byte `protobuf:"bytes,4,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` + Block []byte `protobuf:"bytes,5,opt,name=block,proto3" json:"block,omitempty"` + SenderPubkey []byte `protobuf:"bytes,6,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` + Payload []byte `protobuf:"bytes,7,opt,name=payload,proto3" json:"payload,omitempty"` + SenderPubkeyBitmap []byte `protobuf:"bytes,8,opt,name=sender_pubkey_bitmap,json=senderPubkeyBitmap,proto3" json:"sender_pubkey_bitmap,omitempty"` } func (x *ConsensusRequest) Reset() { @@ -740,6 +741,13 @@ func (x *ConsensusRequest) GetPayload() []byte { return nil } +func (x *ConsensusRequest) GetSenderPubkeyBitmap() []byte { + if x != nil { + return x.SenderPubkeyBitmap + } + return nil +} + type DrandRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1032,7 +1040,7 @@ var file_message_proto_rawDesc = []byte{ 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, - 0x22, 0xd7, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x52, 0x65, + 0x22, 0x89, 0x02, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x6e, 0x73, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x76, 0x69, 0x65, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x76, 0x69, 0x65, 0x77, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, @@ -1045,67 +1053,70 @@ var file_message_proto_rawDesc = []byte{ 0x65, 0x6e, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x97, 0x01, 0x0a, 0x0c, 0x44, - 0x72, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x08, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x42, 0x02, 0x18, - 0x01, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0d, 0x73, 0x65, - 0x6e, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, - 0x6b, 0x65, 0x79, 0x12, 0x21, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x61, 0x73, - 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x09, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x1c, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x07, 0x70, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x22, 0xad, 0x03, 0x0a, 0x11, 0x56, 0x69, 0x65, 0x77, 0x43, 0x68, 0x61, - 0x6e, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x76, 0x69, - 0x65, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x76, 0x69, 0x65, - 0x77, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x75, 0x6d, - 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x73, - 0x65, 0x6e, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, - 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, - 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x50, - 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, - 0x25, 0x0a, 0x0e, 0x76, 0x69, 0x65, 0x77, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x5f, 0x73, 0x69, - 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x76, 0x69, 0x65, 0x77, 0x63, 0x68, 0x61, - 0x6e, 0x67, 0x65, 0x53, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x69, 0x65, 0x77, 0x69, 0x64, - 0x5f, 0x73, 0x69, 0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x76, 0x69, 0x65, 0x77, - 0x69, 0x64, 0x53, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x32, 0x5f, 0x61, 0x67, 0x67, 0x73, - 0x69, 0x67, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x32, 0x41, 0x67, 0x67, - 0x73, 0x69, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x32, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, - 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x32, 0x42, 0x69, 0x74, 0x6d, 0x61, - 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x33, 0x5f, 0x61, 0x67, 0x67, 0x73, 0x69, 0x67, 0x73, 0x18, - 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x33, 0x41, 0x67, 0x67, 0x73, 0x69, 0x67, 0x73, - 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x33, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x18, 0x0c, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x33, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x12, 0x25, 0x0a, - 0x0e, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, - 0x0d, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x42, - 0x6c, 0x6f, 0x63, 0x6b, 0x2a, 0x50, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x53, 0x45, 0x4e, 0x53, 0x55, 0x53, - 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x07, 0x53, 0x54, 0x41, 0x4b, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x1a, - 0x02, 0x08, 0x01, 0x12, 0x0d, 0x0a, 0x05, 0x44, 0x52, 0x41, 0x4e, 0x44, 0x10, 0x02, 0x1a, 0x02, - 0x08, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x5f, 0x53, 0x55, 0x50, - 0x50, 0x4f, 0x52, 0x54, 0x10, 0x03, 0x2a, 0xd1, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x16, 0x4e, 0x45, 0x57, 0x4e, 0x4f, 0x44, - 0x45, 0x5f, 0x42, 0x45, 0x41, 0x43, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x4b, 0x49, 0x4e, 0x47, - 0x10, 0x00, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x4e, 0x4e, 0x4f, 0x55, 0x4e, - 0x43, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, 0x10, - 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, 0x44, 0x10, 0x03, 0x12, - 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x43, - 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x54, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0e, 0x0a, 0x0a, 0x56, 0x49, - 0x45, 0x57, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x45, - 0x57, 0x56, 0x49, 0x45, 0x57, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0a, 0x44, 0x52, 0x41, 0x4e, 0x44, - 0x5f, 0x49, 0x4e, 0x49, 0x54, 0x10, 0x0a, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x14, 0x0a, 0x0c, 0x44, - 0x52, 0x41, 0x4e, 0x44, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x0b, 0x1a, 0x02, 0x08, - 0x01, 0x12, 0x17, 0x0a, 0x0f, 0x4c, 0x4f, 0x54, 0x54, 0x45, 0x52, 0x59, 0x5f, 0x52, 0x45, 0x51, - 0x55, 0x45, 0x53, 0x54, 0x10, 0x0c, 0x1a, 0x02, 0x08, 0x01, 0x32, 0x41, 0x0a, 0x0d, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x30, 0x0a, 0x07, 0x50, - 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x10, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x11, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x65, + 0x6e, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x5f, 0x62, 0x69, 0x74, 0x6d, + 0x61, 0x70, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, + 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x22, 0x97, 0x01, 0x0a, + 0x0c, 0x44, 0x72, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, + 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x42, + 0x02, 0x18, 0x01, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0d, + 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, + 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12, 0x21, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, + 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x09, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x1c, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x07, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0xad, 0x03, 0x0a, 0x11, 0x56, 0x69, 0x65, 0x77, 0x43, + 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, + 0x76, 0x69, 0x65, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x76, + 0x69, 0x65, 0x77, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, + 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, + 0x75, 0x6d, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x64, 0x12, 0x23, 0x0a, + 0x0d, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x73, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6b, + 0x65, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, + 0x6b, 0x65, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x6c, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x50, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x76, 0x69, 0x65, 0x77, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x5f, + 0x73, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x76, 0x69, 0x65, 0x77, 0x63, + 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x69, 0x65, 0x77, + 0x69, 0x64, 0x5f, 0x73, 0x69, 0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x76, 0x69, + 0x65, 0x77, 0x69, 0x64, 0x53, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x32, 0x5f, 0x61, 0x67, + 0x67, 0x73, 0x69, 0x67, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x32, 0x41, + 0x67, 0x67, 0x73, 0x69, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x32, 0x5f, 0x62, 0x69, 0x74, + 0x6d, 0x61, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x32, 0x42, 0x69, 0x74, + 0x6d, 0x61, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x33, 0x5f, 0x61, 0x67, 0x67, 0x73, 0x69, 0x67, + 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6d, 0x33, 0x41, 0x67, 0x67, 0x73, 0x69, + 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x33, 0x5f, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x18, + 0x0c, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x33, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x12, + 0x25, 0x0a, 0x0e, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, + 0x64, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x2a, 0x50, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x53, 0x45, 0x4e, 0x53, + 0x55, 0x53, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x07, 0x53, 0x54, 0x41, 0x4b, 0x49, 0x4e, 0x47, 0x10, + 0x01, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x0d, 0x0a, 0x05, 0x44, 0x52, 0x41, 0x4e, 0x44, 0x10, 0x02, + 0x1a, 0x02, 0x08, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x5f, 0x53, + 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x10, 0x03, 0x2a, 0xd1, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x16, 0x4e, 0x45, 0x57, 0x4e, + 0x4f, 0x44, 0x45, 0x5f, 0x42, 0x45, 0x41, 0x43, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x4b, 0x49, + 0x4e, 0x47, 0x10, 0x00, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x41, 0x4e, 0x4e, 0x4f, + 0x55, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, + 0x45, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, 0x44, 0x10, + 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x04, 0x12, 0x0d, 0x0a, + 0x09, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x54, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0e, 0x0a, 0x0a, + 0x56, 0x49, 0x45, 0x57, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, + 0x4e, 0x45, 0x57, 0x56, 0x49, 0x45, 0x57, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0a, 0x44, 0x52, 0x41, + 0x4e, 0x44, 0x5f, 0x49, 0x4e, 0x49, 0x54, 0x10, 0x0a, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x14, 0x0a, + 0x0c, 0x44, 0x52, 0x41, 0x4e, 0x44, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x0b, 0x1a, + 0x02, 0x08, 0x01, 0x12, 0x17, 0x0a, 0x0f, 0x4c, 0x4f, 0x54, 0x54, 0x45, 0x52, 0x59, 0x5f, 0x52, + 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x0c, 0x1a, 0x02, 0x08, 0x01, 0x32, 0x41, 0x0a, 0x0d, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x30, 0x0a, + 0x07, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x10, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x11, 0x2e, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/proto/message/message.proto b/api/proto/message/message.proto index f21156709..0368aca75 100644 --- a/api/proto/message/message.proto +++ b/api/proto/message/message.proto @@ -87,6 +87,7 @@ message ConsensusRequest { bytes block = 5; bytes sender_pubkey = 6; bytes payload = 7; + bytes sender_pubkey_bitmap = 8; } message DrandRequest { diff --git a/consensus/checks.go b/consensus/checks.go index 91a8a49f1..ff4ce7354 100644 --- a/consensus/checks.go +++ b/consensus/checks.go @@ -62,7 +62,7 @@ func (consensus *Consensus) isRightBlockNumAndViewID(recvMsg *FBFTMessage, Uint64("MsgViewID", recvMsg.ViewID). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("blockNum", consensus.blockNum). - Str("ValidatorPubKey", recvMsg.SenderPubkey.Bytes.Hex()). + Interface("ValidatorPubKey", recvMsg.SenderPubkeys). Msg("BlockNum/viewID not match") return false } @@ -74,12 +74,18 @@ func (consensus *Consensus) onAnnounceSanityChecks(recvMsg *FBFTMessage) bool { msg_pb.MessageType_ANNOUNCE, recvMsg.BlockNum, recvMsg.ViewID, ) if len(logMsgs) > 0 { + if len(logMsgs[0].SenderPubkeys) != 1 || len(recvMsg.SenderPubkeys) != 1 { + consensus.getLogger().Debug(). + Interface("signers", recvMsg.SenderPubkeys). + Msg("[OnAnnounce] Announce message have 0 or more than 1 signers") + return false + } if logMsgs[0].BlockHash != recvMsg.BlockHash && - bytes.Equal(logMsgs[0].SenderPubkey.Bytes[:], recvMsg.SenderPubkey.Bytes[:]) { + bytes.Equal(logMsgs[0].SenderPubkeys[0].Bytes[:], recvMsg.SenderPubkeys[0].Bytes[:]) { consensus.getLogger().Debug(). - Str("logMsgSenderKey", logMsgs[0].SenderPubkey.Bytes.Hex()). + Str("logMsgSenderKey", logMsgs[0].SenderPubkeys[0].Bytes.Hex()). Str("logMsgBlockHash", logMsgs[0].BlockHash.Hex()). - Str("recvMsg.SenderPubkey", recvMsg.SenderPubkey.Bytes.Hex()). + Str("recvMsg.SenderPubkeys", recvMsg.SenderPubkeys[0].Bytes.Hex()). Uint64("recvMsg.BlockNum", recvMsg.BlockNum). Uint64("recvMsg.ViewID", recvMsg.ViewID). Str("recvMsgBlockHash", recvMsg.BlockHash.Hex()). diff --git a/consensus/consensus.go b/consensus/consensus.go index fa6f93267..096a93ef8 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -47,6 +47,8 @@ type Consensus struct { aggregatedCommitSig *bls_core.Sign prepareBitmap *bls_cosi.Mask commitBitmap *bls_cosi.Mask + multiSigBitmap *bls_cosi.Mask // Bitmap for parsing multisig bitmap from validators + multiSigMutex sync.RWMutex // Commits collected from view change // for each viewID, we need keep track of corresponding sigs and bitmap // until one of the viewID has enough votes (>=2f+1) @@ -122,6 +124,8 @@ type Consensus struct { BlockPeriod time.Duration // The time due for next block proposal NextBlockDue time.Time + // Temporary flag to control whether multi-sig signing is enabled + MultiSig bool } // SetCommitDelay sets the commit message delay. If set to non-zero, @@ -179,7 +183,6 @@ func New( // FBFT related consensus.FBFTLog = NewFBFTLog() consensus.phase = FBFTAnnounce - // TODO Refactor consensus.block* into State? consensus.current = State{mode: Normal} // FBFT timeout consensus.consensusTimeout = createTimeout() diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index aaf12df0a..42d7ee700 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -5,6 +5,8 @@ import ( "sync/atomic" "time" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/crypto/bls" "github.com/ethereum/go-ethereum/common" @@ -97,6 +99,7 @@ func (consensus *Consensus) UpdatePublicKeys(pubKeys []bls_cosi.PublicKeyWrapper } consensus.pubKeyLock.Unlock() // reset states after update public keys + // TODO: incorporate bitmaps in the decider, so their state can't be inconsistent. consensus.UpdateBitmaps() consensus.ResetState() @@ -111,7 +114,8 @@ func NewFaker() *Consensus { return &Consensus{} } -// Sign on the hash of the message +// Sign on the hash of the message with the private keys and return the signature. +// If multiple keys are provided, the aggregated signature will be returned. func (consensus *Consensus) signMessage(message []byte, priKey *bls_core.SecretKey) []byte { hash := hash.Keccak256(message) signature := priKey.SignHash(hash[:]) @@ -158,8 +162,12 @@ func (consensus *Consensus) UpdateBitmaps() { members := consensus.Decider.Participants() prepareBitmap, _ := bls_cosi.NewMask(members, nil) commitBitmap, _ := bls_cosi.NewMask(members, nil) + multiSigBitmap, _ := bls_cosi.NewMask(members, nil) consensus.prepareBitmap = prepareBitmap consensus.commitBitmap = commitBitmap + consensus.multiSigMutex.Lock() + consensus.multiSigBitmap = multiSigBitmap + consensus.multiSigMutex.Unlock() } // ResetState resets the state of the consensus @@ -226,7 +234,10 @@ func (consensus *Consensus) checkViewID(msg *FBFTMessage) error { consensus.current.SetMode(Normal) consensus.viewID = msg.ViewID consensus.current.SetViewID(msg.ViewID) - consensus.LeaderPubKey = msg.SenderPubkey + if len(msg.SenderPubkeys) != 1 { + return errors.New("Leader message can not have multiple sender keys") + } + consensus.LeaderPubKey = msg.SenderPubkeys[0] consensus.IgnoreViewIDCheck.UnSet() consensus.consensusTimeout[timeoutConsensus].Start() utils.Logger().Debug(). @@ -349,6 +360,11 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.BlockPeriod = 5 * time.Second + // TODO: remove once multisig is fully upgraded in the network + if consensus.ChainReader.Config().ChainID != params.MainnetChainID || curEpoch.Cmp(big.NewInt(1000)) > 0 { + consensus.MultiSig = true + } + isFirstTimeStaking := consensus.ChainReader.Config().IsStaking(nextEpoch) && len(curHeader.ShardState()) > 0 && !consensus.ChainReader.Config().IsStaking(curEpoch) diff --git a/consensus/consensus_service_test.go b/consensus/consensus_service_test.go index 6f127051e..d3c4aeb01 100644 --- a/consensus/consensus_service_test.go +++ b/consensus/consensus_service_test.go @@ -43,7 +43,7 @@ func TestPopulateMessageFields(t *testing.T) { keyBytes := bls.SerializedPublicKey{} keyBytes.FromLibBLSPublicKey(blsPriKey.GetPublicKey()) - consensusMsg := consensus.populateMessageFields(msg.GetConsensus(), consensus.blockHash[:], + consensusMsg := consensus.populateMessageFieldsAndSender(msg.GetConsensus(), consensus.blockHash[:], keyBytes) if consensusMsg.ViewId != 2 { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 7677b8f8d..b475c047b 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -110,7 +110,7 @@ func (consensus *Consensus) finalizeCommits() { return } // Construct committed message - network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil, leaderPriKey) + network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil, []*bls.PrivateKeyWrapper{leaderPriKey}) if err != nil { consensus.getLogger().Warn().Err(err). Msg("[FinalizeCommits] Unable to construct Committed message") @@ -272,20 +272,18 @@ func (consensus *Consensus) tryCatchup() { consensus.getLogger().Debug().Msg("[TryCatchup] parent block hash not match") break } - consensus.getLogger().Info().Msg("[TryCatchup] block found to commit") - preparedMsgs := consensus.FBFTLog.GetMessagesByTypeSeqHash( - msg_pb.MessageType_PREPARED, committedMsg.BlockNum, committedMsg.BlockHash, - ) - msg := consensus.FBFTLog.FindMessageByMaxViewID(preparedMsgs) - if msg == nil { + if len(committedMsg.SenderPubkeys) != 1 { + consensus.getLogger().Error().Msg("[TryCatchup] Leader message can not have multiple sender keys") break } - consensus.getLogger().Info().Msg("[TryCatchup] prepared message found to commit") + + consensus.getLogger().Info().Msg("[TryCatchup] block found to commit") atomic.AddUint64(&consensus.blockNum, 1) atomic.StoreUint64(&consensus.viewID, committedMsg.ViewID+1) - consensus.LeaderPubKey = committedMsg.SenderPubkey + + consensus.LeaderPubKey = committedMsg.SenderPubkeys[0] consensus.getLogger().Info().Msg("[TryCatchup] Adding block to chain") diff --git a/consensus/construct.go b/consensus/construct.go index e4d1b880d..36b662455 100644 --- a/consensus/construct.go +++ b/consensus/construct.go @@ -2,6 +2,9 @@ package consensus import ( "bytes" + "errors" + + protobuf "github.com/golang/protobuf/proto" "github.com/harmony-one/harmony/crypto/bls" @@ -24,13 +27,31 @@ type NetworkMessage struct { // Populates the common basic fields for all consensus message. func (consensus *Consensus) populateMessageFields( - request *msg_pb.ConsensusRequest, blockHash []byte, pubKey bls.SerializedPublicKey, + request *msg_pb.ConsensusRequest, blockHash []byte, ) *msg_pb.ConsensusRequest { request.ViewId = consensus.viewID request.BlockNum = consensus.blockNum request.ShardId = consensus.ShardID // 32 byte block hash request.BlockHash = blockHash + return request +} + +// Populates the common basic fields for the consensus message and senders bitmap. +func (consensus *Consensus) populateMessageFieldsAndSendersBitmap( + request *msg_pb.ConsensusRequest, blockHash []byte, bitmap []byte, +) *msg_pb.ConsensusRequest { + consensus.populateMessageFields(request, blockHash) + // sender address + request.SenderPubkeyBitmap = bitmap + return request +} + +// Populates the common basic fields for the consensus message and single sender. +func (consensus *Consensus) populateMessageFieldsAndSender( + request *msg_pb.ConsensusRequest, blockHash []byte, pubKey bls.SerializedPublicKey, +) *msg_pb.ConsensusRequest { + consensus.populateMessageFields(request, blockHash) // sender address request.SenderPubkey = pubKey[:] return request @@ -38,8 +59,11 @@ func (consensus *Consensus) populateMessageFields( // construct is the single creation point of messages intended for the wire. func (consensus *Consensus) construct( - p msg_pb.MessageType, payloadForSign []byte, priKey *bls.PrivateKeyWrapper, + p msg_pb.MessageType, payloadForSign []byte, priKeys []*bls.PrivateKeyWrapper, ) (*NetworkMessage, error) { + if len(priKeys) == 0 { + return nil, errors.New("No private keys provided") + } message := &msg_pb.Message{ ServiceType: msg_pb.ServiceType_CONSENSUS, Type: p, @@ -52,11 +76,27 @@ func (consensus *Consensus) construct( aggSig *bls_core.Sign ) - consensusMsg = consensus.populateMessageFields( - message.GetConsensus(), consensus.blockHash[:], priKey.Pub.Bytes, - ) + if len(priKeys) == 1 { + consensusMsg = consensus.populateMessageFieldsAndSender( + message.GetConsensus(), consensus.blockHash[:], priKeys[0].Pub.Bytes, + ) + } else { + // TODO: use a persistent bitmap to report bitmap + mask, err := bls.NewMask(consensus.Decider.Participants(), nil) + if err != nil { + utils.Logger().Warn().Err(err).Msg("unable to setup mask for multi-sig message") + return nil, err + } + for _, key := range priKeys { + mask.SetKey(key.Pub.Bytes, true) + } + consensusMsg = consensus.populateMessageFieldsAndSendersBitmap( + message.GetConsensus(), consensus.blockHash[:], mask.Bitmap, + ) + } // Do the signing, 96 byte of bls signature + needMsgSig := true switch p { case msg_pb.MessageType_PREPARED: consensusMsg.Block = consensus.block @@ -69,13 +109,23 @@ func (consensus *Consensus) construct( buffer.Write(consensus.prepareBitmap.Bitmap) consensusMsg.Payload = buffer.Bytes() case msg_pb.MessageType_PREPARE: - if s := priKey.Pri.SignHash(consensusMsg.BlockHash); s != nil { - consensusMsg.Payload = s.Serialize() + needMsgSig = false + sig := bls_core.Sign{} + for _, priKey := range priKeys { + if s := priKey.Pri.SignHash(consensusMsg.BlockHash); s != nil { + sig.Add(s) + } } + consensusMsg.Payload = sig.Serialize() case msg_pb.MessageType_COMMIT: - if s := priKey.Pri.SignHash(payloadForSign); s != nil { - consensusMsg.Payload = s.Serialize() + needMsgSig = false + sig := bls_core.Sign{} + for _, priKey := range priKeys { + if s := priKey.Pri.SignHash(payloadForSign); s != nil { + sig.Add(s) + } } + consensusMsg.Payload = sig.Serialize() case msg_pb.MessageType_COMMITTED: buffer := bytes.Buffer{} // 96 bytes aggregated signature @@ -88,7 +138,16 @@ func (consensus *Consensus) construct( consensusMsg.Payload = consensus.blockHash[:] } - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey.Pri) + var marshaledMessage []byte + var err error + if needMsgSig { + // The message that needs signing only needs to be signed with a single key + marshaledMessage, err = consensus.signAndMarshalConsensusMessage(message, priKeys[0].Pri) + } else { + // Skip message (potentially multi-sig) signing for validator consensus messages (prepare and commit) + // as signature is already signed on the block data. + marshaledMessage, err = protobuf.Marshal(message) + } if err != nil { utils.Logger().Error().Err(err). Str("phase", p.String()). @@ -96,7 +155,7 @@ func (consensus *Consensus) construct( return nil, err } - FBFTMsg, err2 := ParseFBFTMessage(message) + FBFTMsg, err2 := consensus.ParseFBFTMessage(message) if err2 != nil { utils.Logger().Error().Err(err). diff --git a/consensus/construct_test.go b/consensus/construct_test.go index 04c548f49..660b2971d 100644 --- a/consensus/construct_test.go +++ b/consensus/construct_test.go @@ -35,7 +35,7 @@ func TestConstructAnnounceMessage(test *testing.T) { pubKeyWrapper := bls.PublicKeyWrapper{Object: blsPriKey.GetPublicKey()} pubKeyWrapper.Bytes.FromLibBLSPublicKey(pubKeyWrapper.Object) priKeyWrapper := bls.PrivateKeyWrapper{blsPriKey, &pubKeyWrapper} - if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, &priKeyWrapper); err != nil { + if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, []*bls.PrivateKeyWrapper{&priKeyWrapper}); err != nil { test.Fatalf("could not construct announce: %v", err) } } @@ -73,7 +73,7 @@ func TestConstructPreparedMessage(test *testing.T) { validatorKey.FromLibBLSPublicKey(validatorPubKey) consensus.Decider.SubmitVote( quorum.Prepare, - leaderKey, + []bls.SerializedPublicKey{leaderKey}, leaderPriKey.Sign(message), common.BytesToHash(consensus.blockHash[:]), consensus.blockNum, @@ -81,7 +81,7 @@ func TestConstructPreparedMessage(test *testing.T) { ) if _, err := consensus.Decider.SubmitVote( quorum.Prepare, - validatorKey, + []bls.SerializedPublicKey{validatorKey}, validatorPriKey.Sign(message), common.BytesToHash(consensus.blockHash[:]), consensus.blockNum, @@ -101,7 +101,7 @@ func TestConstructPreparedMessage(test *testing.T) { pubKeyWrapper := bls.PublicKeyWrapper{Object: blsPriKey.GetPublicKey()} pubKeyWrapper.Bytes.FromLibBLSPublicKey(pubKeyWrapper.Object) priKeyWrapper := bls.PrivateKeyWrapper{blsPriKey, &pubKeyWrapper} - network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, &priKeyWrapper) + network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, []*bls.PrivateKeyWrapper{&priKeyWrapper}) if err != nil { test.Errorf("Error when creating prepared message") } diff --git a/consensus/double_sign.go b/consensus/double_sign.go index cceb2a729..d11044afd 100644 --- a/consensus/double_sign.go +++ b/consensus/double_sign.go @@ -1,6 +1,9 @@ package consensus import ( + "bytes" + "sort" + "github.com/ethereum/go-ethereum/common" bls_core "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/consensus/quorum" @@ -12,96 +15,120 @@ import ( // Returns true when it is a double-sign or there is error, otherwise, false. func (consensus *Consensus) checkDoubleSign(recvMsg *FBFTMessage) bool { if consensus.couldThisBeADoubleSigner(recvMsg) { - if alreadyCastBallot := consensus.Decider.ReadBallot( - quorum.Commit, recvMsg.SenderPubkey.Bytes, - ); alreadyCastBallot != nil { - firstPubKey, err := bls.BytesToBLSPublicKey(alreadyCastBallot.SignerPubKey[:]) - if err != nil { - return false - } - if recvMsg.SenderPubkey.Object.IsEqual(firstPubKey) { - for _, blk := range consensus.FBFTLog.GetBlocksByNumber(recvMsg.BlockNum) { - firstSignedBlock := blk.Header() - areHeightsEqual := firstSignedBlock.Number().Uint64() == recvMsg.BlockNum - areViewIDsEqual := firstSignedBlock.ViewID().Uint64() == recvMsg.ViewID - areHeadersEqual := firstSignedBlock.Hash() == recvMsg.BlockHash + addrSet := map[common.Address]struct{}{} + for _, pubKey2 := range recvMsg.SenderPubkeys { + if alreadyCastBallot := consensus.Decider.ReadBallot( + quorum.Commit, pubKey2.Bytes, + ); alreadyCastBallot != nil { + for _, pubKey1 := range alreadyCastBallot.SignerPubKeys { + if bytes.Compare(pubKey2.Bytes[:], pubKey1[:]) == 0 { + for _, blk := range consensus.FBFTLog.GetBlocksByNumber(recvMsg.BlockNum) { + firstSignedHeader := blk.Header() + areHeightsEqual := firstSignedHeader.Number().Uint64() == recvMsg.BlockNum + areViewIDsEqual := firstSignedHeader.ViewID().Uint64() == recvMsg.ViewID + areHeadersEqual := firstSignedHeader.Hash() == recvMsg.BlockHash - // If signer already firstSignedBlock, and the block height is the same - // and the viewID is the same, then we need to verify the block - // hash, and if block hash is different, then that is a clear - // case of double signing - if areHeightsEqual && areViewIDsEqual && !areHeadersEqual { - var doubleSign bls_core.Sign - if err := doubleSign.Deserialize(recvMsg.Payload); err != nil { - consensus.getLogger().Err(err).Str("msg", recvMsg.String()). - Msg("could not deserialize potential double signer") - return true - } + // If signer already firstSignedHeader, and the block height is the same + // and the viewID is the same, then we need to verify the block + // hash, and if block hash is different, then that is a clear + // case of double signing + if areHeightsEqual && areViewIDsEqual && !areHeadersEqual { + var doubleSign bls_core.Sign + if err := doubleSign.Deserialize(recvMsg.Payload); err != nil { + consensus.getLogger().Err(err).Str("msg", recvMsg.String()). + Msg("could not deserialize potential double signer") + return true + } - curHeader := consensus.ChainReader.CurrentHeader() - committee, err := consensus.ChainReader.ReadShardState(curHeader.Epoch()) - if err != nil { - consensus.getLogger().Err(err). - Uint32("shard", consensus.ShardID). - Uint64("epoch", curHeader.Epoch().Uint64()). - Msg("could not read shard state") - return true - } + curHeader := consensus.ChainReader.CurrentHeader() + committee, err := consensus.ChainReader.ReadShardState(curHeader.Epoch()) + if err != nil { + consensus.getLogger().Err(err). + Uint32("shard", consensus.ShardID). + Uint64("epoch", curHeader.Epoch().Uint64()). + Msg("could not read shard state") + return true + } - subComm, err := committee.FindCommitteeByID( - consensus.ShardID, - ) - if err != nil { - consensus.getLogger().Err(err). - Str("msg", recvMsg.String()). - Msg("could not find subcommittee for bls key") - return true - } + subComm, err := committee.FindCommitteeByID( + consensus.ShardID, + ) + if err != nil { + consensus.getLogger().Err(err). + Str("msg", recvMsg.String()). + Msg("could not find subcommittee for bls key") + return true + } - addr, err := subComm.AddressForBLSKey(recvMsg.SenderPubkey.Bytes) - if err != nil { - consensus.getLogger().Err(err).Str("msg", recvMsg.String()). - Msg("could not find address for bls key") - return true - } + addr, err := subComm.AddressForBLSKey(pubKey2.Bytes) + if err != nil { + consensus.getLogger().Err(err).Str("msg", recvMsg.String()). + Msg("could not find address for bls key") + return true + } + if _, ok := addrSet[*addr]; ok { + // Address already slashed + break + } - leaderAddr, err := subComm.AddressForBLSKey(consensus.LeaderPubKey.Bytes) - if err != nil { - consensus.getLogger().Err(err).Str("msg", recvMsg.String()). - Msg("could not find address for leader bls key") - return true - } + leaderAddr, err := subComm.AddressForBLSKey(consensus.LeaderPubKey.Bytes) + if err != nil { + consensus.getLogger().Err(err).Str("msg", recvMsg.String()). + Msg("could not find address for leader bls key") + return true + } - go func(reporter common.Address) { - evid := slash.Evidence{ - ConflictingVotes: slash.ConflictingVotes{ - FirstVote: slash.Vote{ - alreadyCastBallot.SignerPubKey, - alreadyCastBallot.BlockHeaderHash, - alreadyCastBallot.Signature, - }, - SecondVote: slash.Vote{ - recvMsg.SenderPubkey.Bytes, - recvMsg.BlockHash, - common.Hex2Bytes(doubleSign.SerializeToHexStr()), - }}, - Moment: slash.Moment{ - Epoch: curHeader.Epoch(), - ShardID: consensus.ShardID, - }, - Offender: *addr, - } - proof := slash.Record{ - Evidence: evid, - Reporter: reporter, + go func(reporter common.Address) { + secondKeys := make([]bls.SerializedPublicKey, len(recvMsg.SenderPubkeys)) + for i, pubKey := range recvMsg.SenderPubkeys { + secondKeys[i] = pubKey.Bytes + } + evid := slash.Evidence{ + ConflictingVotes: slash.ConflictingVotes{ + FirstVote: slash.Vote{ + alreadyCastBallot.SignerPubKeys, + alreadyCastBallot.BlockHeaderHash, + alreadyCastBallot.Signature, + }, + SecondVote: slash.Vote{ + secondKeys, + recvMsg.BlockHash, + common.Hex2Bytes(doubleSign.SerializeToHexStr()), + }}, + Moment: slash.Moment{ + Epoch: curHeader.Epoch(), + ShardID: consensus.ShardID, + Height: recvMsg.BlockNum, + ViewID: recvMsg.ViewID, + }, + Offender: *addr, + } + + sort.SliceStable(evid.ConflictingVotes.FirstVote.SignerPubKeys, func(i, j int) bool { + return bytes.Compare( + evid.ConflictingVotes.FirstVote.SignerPubKeys[i][:], + evid.ConflictingVotes.FirstVote.SignerPubKeys[j][:]) < 0 + }) + sort.SliceStable(evid.ConflictingVotes.SecondVote.SignerPubKeys, func(i, j int) bool { + return bytes.Compare( + evid.ConflictingVotes.SecondVote.SignerPubKeys[i][:], + evid.ConflictingVotes.SecondVote.SignerPubKeys[j][:]) < 0 + }) + proof := slash.Record{ + Evidence: evid, + Reporter: reporter, + } + consensus.SlashChan <- proof + }(*leaderAddr) + addrSet[*addr] = struct{}{} + break } - consensus.SlashChan <- proof - }(*leaderAddr) - return true + } } } } } + return true } return false diff --git a/consensus/fbft_log.go b/consensus/fbft_log.go index 23e1d87b4..e8e664924 100644 --- a/consensus/fbft_log.go +++ b/consensus/fbft_log.go @@ -23,27 +23,32 @@ type FBFTLog struct { // FBFTMessage is the record of pbft messages received by a node during FBFT process type FBFTMessage struct { - MessageType msg_pb.MessageType - ViewID uint64 - BlockNum uint64 - BlockHash common.Hash - Block []byte - SenderPubkey *bls.PublicKeyWrapper - LeaderPubkey *bls.PublicKeyWrapper - Payload []byte - ViewchangeSig *bls_core.Sign - ViewidSig *bls_core.Sign - M2AggSig *bls_core.Sign - M2Bitmap *bls_cosi.Mask - M3AggSig *bls_core.Sign - M3Bitmap *bls_cosi.Mask + MessageType msg_pb.MessageType + ViewID uint64 + BlockNum uint64 + BlockHash common.Hash + Block []byte + SenderPubkeys []*bls.PublicKeyWrapper + SenderPubkeyBitmap []byte + LeaderPubkey *bls.PublicKeyWrapper + Payload []byte + ViewchangeSig *bls_core.Sign + ViewidSig *bls_core.Sign + M2AggSig *bls_core.Sign + M2Bitmap *bls_cosi.Mask + M3AggSig *bls_core.Sign + M3Bitmap *bls_cosi.Mask } // String .. func (m *FBFTMessage) String() string { sender := "" - if m.SenderPubkey != nil { - sender = m.SenderPubkey.Bytes.Hex() + for _, key := range m.SenderPubkeys { + if sender == "" { + sender = key.Bytes.Hex() + } else { + sender = sender + ";" + key.Bytes.Hex() + } } leader := "" if m.LeaderPubkey != nil { @@ -240,7 +245,7 @@ func (log *FBFTLog) FindMessageByMaxViewID(msgs []*FBFTMessage) *FBFTMessage { } // ParseFBFTMessage parses FBFT message into FBFTMessage structure -func ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { +func (consensus *Consensus) ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { // TODO Have this do sanity checks on the message please pbftMsg := FBFTMessage{} pbftMsg.MessageType = msg.GetType() @@ -252,13 +257,27 @@ func ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { copy(pbftMsg.Payload[:], consensusMsg.Payload[:]) pbftMsg.Block = make([]byte, len(consensusMsg.Block)) copy(pbftMsg.Block[:], consensusMsg.Block[:]) + pbftMsg.SenderPubkeyBitmap = make([]byte, len(consensusMsg.SenderPubkeyBitmap)) + copy(pbftMsg.SenderPubkeyBitmap[:], consensusMsg.SenderPubkeyBitmap[:]) - pubKey, err := bls_cosi.BytesToBLSPublicKey(consensusMsg.SenderPubkey) - if err != nil { - return nil, err + if len(consensusMsg.SenderPubkey) != 0 { + // If SenderPubKey is populated, treat it as a single key message + pubKey, err := bls_cosi.BytesToBLSPublicKey(consensusMsg.SenderPubkey) + if err != nil { + return nil, err + } + pbftMsg.SenderPubkeys = []*bls.PublicKeyWrapper{{Object: pubKey}} + copy(pbftMsg.SenderPubkeys[0].Bytes[:], consensusMsg.SenderPubkey[:]) + } else { + // else, it should be a multi-key message where the bitmap is populated + consensus.multiSigMutex.RLock() + pubKeys, err := consensus.multiSigBitmap.GetSignedPubKeysFromBitmap(pbftMsg.SenderPubkeyBitmap) + consensus.multiSigMutex.RUnlock() + if err != nil { + return nil, err + } + pbftMsg.SenderPubkeys = pubKeys } - pbftMsg.SenderPubkey = &bls.PublicKeyWrapper{Object: pubKey} - copy(pbftMsg.SenderPubkey.Bytes[:], consensusMsg.SenderPubkey[:]) return &pbftMsg, nil } @@ -304,8 +323,8 @@ func ParseViewChangeMessage(msg *msg_pb.Message) (*FBFTMessage, error) { return nil, err } - pbftMsg.SenderPubkey = &bls.PublicKeyWrapper{Object: pubKey} - copy(pbftMsg.SenderPubkey.Bytes[:], vcMsg.SenderPubkey[:]) + pbftMsg.SenderPubkeys = []*bls.PublicKeyWrapper{{Object: pubKey}} + copy(pbftMsg.SenderPubkeys[0].Bytes[:], vcMsg.SenderPubkey[:]) pbftMsg.LeaderPubkey = &bls.PublicKeyWrapper{Object: leaderKey} copy(pbftMsg.LeaderPubkey.Bytes[:], vcMsg.LeaderPubkey[:]) pbftMsg.ViewchangeSig = &vcSig @@ -336,8 +355,8 @@ func (consensus *Consensus) ParseNewViewMessage(msg *msg_pb.Message) (*FBFTMessa return nil, err } - FBFTMsg.SenderPubkey = &bls.PublicKeyWrapper{Object: pubKey} - copy(FBFTMsg.SenderPubkey.Bytes[:], vcMsg.SenderPubkey[:]) + FBFTMsg.SenderPubkeys = []*bls.PublicKeyWrapper{{Object: pubKey}} + copy(FBFTMsg.SenderPubkeys[0].Bytes[:], vcMsg.SenderPubkey[:]) members := consensus.Decider.Participants() if len(vcMsg.M3Aggsigs) > 0 { diff --git a/consensus/leader.go b/consensus/leader.go index b2a22605b..5f5fa81f5 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -3,11 +3,13 @@ package consensus import ( "time" - "github.com/harmony-one/harmony/consensus/signature" + "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/consensus/signature" + "github.com/ethereum/go-ethereum/rlp" - "github.com/harmony-one/bls/ffi/go/bls" + bls_core "github.com/harmony-one/bls/ffi/go/bls" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core/types" @@ -37,7 +39,7 @@ func (consensus *Consensus) announce(block *types.Block) { return } - networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, key) + networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, []*bls.PrivateKeyWrapper{key}) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_ANNOUNCE.String()). @@ -66,7 +68,7 @@ func (consensus *Consensus) announce(block *types.Block) { if _, err := consensus.Decider.AddNewVote( quorum.Prepare, - key.Pub.Bytes, + []*bls.PublicKeyWrapper{key.Pub}, key.Pri.SignHash(consensus.blockHash[:]), block.Hash(), block.NumberU64(), @@ -100,7 +102,7 @@ func (consensus *Consensus) announce(block *types.Block) { } func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { - recvMsg, err := ParseFBFTMessage(msg) + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error().Err(err).Msg("[OnPrepare] Unparseable validator message") return @@ -128,18 +130,20 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { blockHash := consensus.blockHash[:] prepareBitmap := consensus.prepareBitmap // proceed only when the message is not received before - signed := consensus.Decider.ReadBallot(quorum.Prepare, recvMsg.SenderPubkey.Bytes) - if signed != nil { - consensus.getLogger().Debug(). - Str("validatorPubKey", recvMsg.SenderPubkey.Bytes.Hex()). - Msg("[OnPrepare] Already Received prepare message from the validator") - return + for _, signer := range recvMsg.SenderPubkeys { + signed := consensus.Decider.ReadBallot(quorum.Prepare, signer.Bytes) + if signed != nil { + consensus.getLogger().Debug(). + Str("validatorPubKey", signer.Bytes.Hex()). + Msg("[OnPrepare] Already Received prepare message from the validator") + return + } } if consensus.Decider.IsQuorumAchieved(quorum.Prepare) { // already have enough signatures consensus.getLogger().Debug(). - Str("validatorPubKey", recvMsg.SenderPubkey.Bytes.Hex()). + Interface("validatorPubKeys", recvMsg.SenderPubkeys). Msg("[OnPrepare] Received Additional Prepare Message") return } @@ -148,14 +152,22 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { // Check BLS signature for the multi-sig prepareSig := recvMsg.Payload - var sign bls.Sign + var sign bls_core.Sign err = sign.Deserialize(prepareSig) if err != nil { consensus.getLogger().Error().Err(err). Msg("[OnPrepare] Failed to deserialize bls signature") return } - if !sign.VerifyHash(recvMsg.SenderPubkey.Object, blockHash) { + signerPubKey := &bls_core.PublicKey{} + if len(recvMsg.SenderPubkeys) == 1 { + signerPubKey = recvMsg.SenderPubkeys[0].Object + } else { + for _, pubKey := range recvMsg.SenderPubkeys { + signerPubKey.Add(pubKey.Object) + } + } + if !sign.VerifyHash(signerPubKey, blockHash) { consensus.getLogger().Error().Msg("[OnPrepare] Received invalid BLS signature") return } @@ -167,7 +179,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { //// Write - Start if _, err := consensus.Decider.AddNewVote( - quorum.Prepare, recvMsg.SenderPubkey.Bytes, + quorum.Prepare, recvMsg.SenderPubkeys, &sign, recvMsg.BlockHash, recvMsg.BlockNum, recvMsg.ViewID, ); err != nil { @@ -175,7 +187,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { return } // Set the bitmap indicating that this validator signed. - if err := prepareBitmap.SetKey(recvMsg.SenderPubkey.Bytes, true); err != nil { + if err := prepareBitmap.SetKeysAtomic(recvMsg.SenderPubkeys, true); err != nil { consensus.getLogger().Warn().Err(err).Msg("[OnPrepare] prepareBitmap.SetKey failed") return } @@ -193,7 +205,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { } func (consensus *Consensus) onCommit(msg *msg_pb.Message) { - recvMsg, err := ParseFBFTMessage(msg) + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Debug().Err(err).Msg("[OnCommit] Parse pbft message failed") return @@ -214,14 +226,13 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { //// Read - End // Verify the signature on commitPayload is correct - validatorPubKey, commitSig := recvMsg.SenderPubkey, recvMsg.Payload logger := consensus.getLogger().With(). - Str("validatorPubKey", validatorPubKey.Bytes.Hex()). + Interface("validatorPubKey", recvMsg.SenderPubkeys). Int64("numReceivedSoFar", signerCount).Logger() logger.Debug().Msg("[OnCommit] Received new commit message") - var sign bls.Sign - if err := sign.Deserialize(commitSig); err != nil { + var sign bls_core.Sign + if err := sign.Deserialize(recvMsg.Payload); err != nil { logger.Debug().Msg("[OnCommit] Failed to deserialize bls signature") return } @@ -243,7 +254,15 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { Uint64("MsgBlockNum", recvMsg.BlockNum). Logger() - if !sign.VerifyHash(recvMsg.SenderPubkey.Object, commitPayload) { + signerPubKey := &bls_core.PublicKey{} + if len(recvMsg.SenderPubkeys) == 1 { + signerPubKey = recvMsg.SenderPubkeys[0].Object + } else { + for _, pubKey := range recvMsg.SenderPubkeys { + signerPubKey.Add(pubKey.Object) + } + } + if !sign.VerifyHash(signerPubKey, commitPayload) { logger.Error().Msg("[OnCommit] Cannot verify commit message") return } @@ -254,14 +273,14 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { return } if _, err := consensus.Decider.AddNewVote( - quorum.Commit, recvMsg.SenderPubkey.Bytes, + quorum.Commit, recvMsg.SenderPubkeys, &sign, recvMsg.BlockHash, recvMsg.BlockNum, recvMsg.ViewID, ); err != nil { return } // Set the bitmap indicating that this validator signed. - if err := commitBitmap.SetKey(recvMsg.SenderPubkey.Bytes, true); err != nil { + if err := commitBitmap.SetKeysAtomic(recvMsg.SenderPubkeys, true); err != nil { consensus.getLogger().Warn().Err(err). Msg("[OnCommit] commitBitmap.SetKey failed") return diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index 7477ed976..45db9bcd3 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -29,11 +29,14 @@ func (v *uniformVoteWeight) Policy() Policy { // AddNewVote .. func (v *uniformVoteWeight) AddNewVote( - p Phase, pubKeyBytes bls.SerializedPublicKey, + p Phase, pubKeys []*bls_cosi.PublicKeyWrapper, sig *bls_core.Sign, headerHash common.Hash, height, viewID uint64) (*votepower.Ballot, error) { - - return v.SubmitVote(p, pubKeyBytes, sig, headerHash, height, viewID) + pubKeysBytes := make([]bls.SerializedPublicKey, len(pubKeys)) + for i, pubKey := range pubKeys { + pubKeysBytes[i] = pubKey.Bytes + } + return v.SubmitVote(p, pubKeysBytes, sig, headerHash, height, viewID) } // IsQuorumAchieved .. diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 8370ff052..65c6edfdf 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -1,6 +1,7 @@ package quorum import ( + "bytes" "encoding/json" "math/big" @@ -57,19 +58,40 @@ func (v *stakedVoteWeight) Policy() Policy { // AddNewVote .. func (v *stakedVoteWeight) AddNewVote( - p Phase, pubKeyBytes bls.SerializedPublicKey, + p Phase, pubKeys []*bls_cosi.PublicKeyWrapper, sig *bls_core.Sign, headerHash common.Hash, height, viewID uint64) (*votepower.Ballot, error) { - // TODO(audit): pass in sig as byte[] too, so no need to serialize - ballet, err := v.SubmitVote(p, pubKeyBytes, sig, headerHash, height, viewID) + pubKeysBytes := make([]bls.SerializedPublicKey, len(pubKeys)) + signerAddr := common.Address{} + for i, pubKey := range pubKeys { + voter, ok := v.roster.Voters[pubKey.Bytes] + if !ok { + return nil, errors.Errorf("Signer not in committee: %x", pubKey.Bytes) + } + if i == 0 { + signerAddr = voter.EarningAccount + } else { + if bytes.Compare(signerAddr.Bytes(), voter.EarningAccount[:]) != 0 && !voter.IsHarmonyNode { + return nil, errors.Errorf("Multiple signer accounts used in multi-sig: %x, %x", signerAddr.Bytes(), voter.EarningAccount) + } + } + pubKeysBytes[i] = pubKey.Bytes + } + + ballet, err := v.SubmitVote(p, pubKeysBytes, sig, headerHash, height, viewID) if err != nil { return ballet, err } // Accumulate total voting power - additionalVotePower := v.roster.Voters[pubKeyBytes].OverallPercent + additionalVotePower := numeric.NewDec(0) + + for _, pubKeyBytes := range pubKeysBytes { + additionalVotePower = additionalVotePower.Add(v.roster.Voters[pubKeyBytes].OverallPercent) + } + tallyQuorum := func() *tallyAndQuorum { switch p { case Prepare: @@ -83,7 +105,6 @@ func (v *stakedVoteWeight) AddNewVote( return nil } }() - tallyQuorum.tally = tallyQuorum.tally.Add(additionalVotePower) t := v.QuorumThreshold() diff --git a/consensus/quorum/one-node-staked-vote_test.go b/consensus/quorum/one-node-staked-vote_test.go index f13072f90..3529ccaa0 100644 --- a/consensus/quorum/one-node-staked-vote_test.go +++ b/consensus/quorum/one-node-staked-vote_test.go @@ -114,7 +114,7 @@ func sign(d Decider, k secretKeyMap, p Phase) { for k, v := range k { sig := v.Sign(msg) // TODO Make upstream test provide meaningful test values - d.AddNewVote(p, k, sig, common.Hash{}, 0, 0) + d.AddNewVote(p, []*bls.PublicKeyWrapper{{Bytes: k}}, sig, common.Hash{}, 0, 0) } } diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 17b0a5361..c9c734553 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -80,7 +80,7 @@ type ParticipantTracker interface { type SignatoryTracker interface { ParticipantTracker SubmitVote( - p Phase, pubkey bls.SerializedPublicKey, + p Phase, pubkeys []bls.SerializedPublicKey, sig *bls_core.Sign, headerHash common.Hash, height, viewID uint64, ) (*votepower.Ballot, error) @@ -117,7 +117,7 @@ type Decider interface { SetVoters(subCommittee *shard.Committee, epoch *big.Int) (*TallyResult, error) Policy() Policy AddNewVote( - p Phase, pubkey bls.SerializedPublicKey, + p Phase, pubkeys []*bls_cosi.PublicKeyWrapper, sig *bls_core.Sign, headerHash common.Hash, height, viewID uint64, ) (*votepower.Ballot, error) @@ -168,10 +168,29 @@ type depInject struct { func (s *cIdentities) AggregateVotes(p Phase) *bls_core.Sign { ballots := s.ReadAllBallots(p) sigs := make([]*bls_core.Sign, 0, len(ballots)) + collectedKeys := map[bls_cosi.SerializedPublicKey]struct{}{} for _, ballot := range ballots { sig := &bls_core.Sign{} // NOTE invariant that shouldn't happen by now // but pointers are pointers + + // If the multisig from any of the signers in this ballot are already collected, + // we need to skip this ballot as its multisig is a duplicate. + alreadyCollected := false + for _, key := range ballot.SignerPubKeys { + if _, ok := collectedKeys[key]; ok { + alreadyCollected = true + break + } + } + if alreadyCollected { + continue + } + + for _, key := range ballot.SignerPubKeys { + collectedKeys[key] = struct{}{} + } + if ballot != nil { sig.DeserializeHexStr(common.Bytes2Hex(ballot.Signature)) sigs = append(sigs, sig) @@ -230,30 +249,37 @@ func (s *cIdentities) SignersCount(p Phase) int64 { } func (s *cIdentities) SubmitVote( - p Phase, pubkey bls.SerializedPublicKey, + p Phase, pubkeys []bls.SerializedPublicKey, sig *bls_core.Sign, headerHash common.Hash, height, viewID uint64, ) (*votepower.Ballot, error) { - if ballet := s.ReadBallot(p, pubkey); ballet != nil { - return nil, errors.Errorf("vote is already submitted %x", pubkey) + for _, pubKey := range pubkeys { + if ballet := s.ReadBallot(p, pubKey); ballet != nil { + return nil, errors.Errorf("vote is already submitted %x", pubKey) + } } ballot := &votepower.Ballot{ - SignerPubKey: pubkey, + SignerPubKeys: pubkeys, BlockHeaderHash: headerHash, Signature: common.Hex2Bytes(sig.SerializeToHexStr()), Height: height, ViewID: viewID, } - switch p { - case Prepare: - s.prepare.BallotBox[pubkey] = ballot - case Commit: - s.commit.BallotBox[pubkey] = ballot - case ViewChange: - s.viewChange.BallotBox[pubkey] = ballot - default: - return nil, errors.Wrapf(errPhaseUnknown, "given: %s", p.String()) + + // For each of the keys signed in the multi-sig, a separate ballot with the same multisig is recorded + // This way it's easier to check if a specific key already signed or not. + for _, pubKey := range pubkeys { + switch p { + case Prepare: + s.prepare.BallotBox[pubKey] = ballot + case Commit: + s.commit.BallotBox[pubKey] = ballot + case ViewChange: + s.viewChange.BallotBox[pubKey] = ballot + default: + return nil, errors.Wrapf(errPhaseUnknown, "given: %s", p.String()) + } } return ballot, nil } diff --git a/consensus/threshold.go b/consensus/threshold.go index 27f60ed66..c1ca8c6f1 100644 --- a/consensus/threshold.go +++ b/consensus/threshold.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/consensus/signature" "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" @@ -21,7 +22,7 @@ func (consensus *Consensus) didReachPrepareQuorum() error { } // Construct and broadcast prepared message networkMessage, err := consensus.construct( - msg_pb.MessageType_PREPARED, nil, leaderPriKey, + msg_pb.MessageType_PREPARED, nil, []*bls.PrivateKeyWrapper{leaderPriKey}, ) if err != nil { consensus.getLogger().Err(err). @@ -58,7 +59,7 @@ func (consensus *Consensus) didReachPrepareQuorum() error { if _, err := consensus.Decider.AddNewVote( quorum.Commit, - key.Pub.Bytes, + []*bls.PublicKeyWrapper{key.Pub}, key.Pri.SignHash(commitPayload), blockObj.Hash(), blockObj.NumberU64(), diff --git a/consensus/validator.go b/consensus/validator.go index fde5ab253..1e2c4c78a 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -5,17 +5,19 @@ import ( "encoding/hex" "time" + "github.com/harmony-one/harmony/crypto/bls" + nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/signature" "github.com/harmony-one/harmony/core/types" - nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/p2p" ) func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { - recvMsg, err := ParseFBFTMessage(msg) + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Error(). Err(err). @@ -59,11 +61,27 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { func (consensus *Consensus) prepare() { groupID := []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} - for _, key := range consensus.priKey { + priKeys := []*bls.PrivateKeyWrapper{} + p2pMsgs := []*NetworkMessage{} + for i, key := range consensus.priKey { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } - networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil, &key) + priKeys = append(priKeys, &consensus.priKey[i]) + if !consensus.MultiSig { + networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil, []*bls.PrivateKeyWrapper{&key}) + if err != nil { + consensus.getLogger().Err(err). + Str("message-type", msg_pb.MessageType_PREPARE.String()). + Msg("could not construct message") + return + } + + p2pMsgs = append(p2pMsgs, networkMessage) + } + } + if consensus.MultiSig { + networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil, priKeys) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_PREPARE.String()). @@ -71,11 +89,15 @@ func (consensus *Consensus) prepare() { return } + p2pMsgs = append(p2pMsgs, networkMessage) + } + + for _, p2pMsg := range p2pMsgs { // TODO: this will not return immediately, may block if consensus.current.Mode() != Listening { if err := consensus.msgSender.SendWithoutRetry( groupID, - p2p.ConstructMessage(networkMessage.Bytes), + p2p.ConstructMessage(p2pMsg.Bytes), ); err != nil { consensus.getLogger().Warn().Err(err).Msg("[OnAnnounce] Cannot send prepare message") } else { @@ -95,7 +117,7 @@ func (consensus *Consensus) prepare() { // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { - recvMsg, err := ParseFBFTMessage(msg) + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Debug().Err(err).Msg("[OnPrepared] Unparseable validator message") return @@ -190,7 +212,6 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { return } - // TODO: genesis account node delay for 1 second, // this is a temp fix for allows FN nodes to earning reward if consensus.delayCommit > 0 { time.Sleep(consensus.delayCommit) @@ -212,21 +233,47 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { groupID := []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), } - for _, key := range consensus.priKey { + + priKeys := []*bls.PrivateKeyWrapper{} + p2pMsgs := []*NetworkMessage{} + for i, key := range consensus.priKey { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } + priKeys = append(priKeys, &consensus.priKey[i]) + if !consensus.MultiSig { + networkMessage, err := consensus.construct(msg_pb.MessageType_COMMIT, + commitPayload, []*bls.PrivateKeyWrapper{&key}) + if err != nil { + consensus.getLogger().Err(err). + Str("message-type", msg_pb.MessageType_COMMIT.String()). + Msg("could not construct message") + return + } + + p2pMsgs = append(p2pMsgs, networkMessage) + } + } - networkMessage, _ := consensus.construct( - msg_pb.MessageType_COMMIT, - commitPayload, - &key, - ) + if consensus.MultiSig { + networkMessage, err := consensus.construct(msg_pb.MessageType_COMMIT, + commitPayload, priKeys) + if err != nil { + consensus.getLogger().Err(err). + Str("message-type", msg_pb.MessageType_COMMIT.String()). + Msg("could not construct message") + return + } + p2pMsgs = append(p2pMsgs, networkMessage) + } + + for _, p2pMsg := range p2pMsgs { + // TODO: this will not return immediately, may block if consensus.current.Mode() != Listening { if err := consensus.msgSender.SendWithoutRetry( groupID, - p2p.ConstructMessage(networkMessage.Bytes), + p2p.ConstructMessage(p2pMsg.Bytes), ); err != nil { consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!") } else { @@ -245,7 +292,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { } func (consensus *Consensus) onCommitted(msg *msg_pb.Message) { - recvMsg, err := ParseFBFTMessage(msg) + recvMsg, err := consensus.ParseFBFTMessage(msg) if err != nil { consensus.getLogger().Warn().Msg("[OnCommitted] unable to parse msg") return diff --git a/consensus/view_change.go b/consensus/view_change.go index fb07a8390..f31ad6447 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -173,7 +173,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { consensus.getLogger().Info(). Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). Int64("need", consensus.Decider.TwoThirdsSignersCount()). - Str("validatorPubKey", recvMsg.SenderPubkey.Bytes.Hex()). + Interface("validatorPubKeys", recvMsg.SenderPubkeys). Msg("[onViewChange] Received Enough View Change Messages") return } @@ -182,7 +182,11 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { return } - senderKey := recvMsg.SenderPubkey + if len(recvMsg.SenderPubkeys) != 1 { + consensus.getLogger().Error().Msg("[onViewChange] multiple signers in view change message.") + return + } + senderKey := recvMsg.SenderPubkeys[0] consensus.vcLock.Lock() defer consensus.vcLock.Unlock() @@ -283,7 +287,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { Str("validatorPubKey", senderKey.Bytes.Hex()). Msg("[onViewChange] Add M2 (NIL) type message") consensus.nilSigs[recvMsg.ViewID][senderKey.Bytes.Hex()] = recvMsg.ViewchangeSig - consensus.nilBitmap[recvMsg.ViewID].SetKey(recvMsg.SenderPubkey.Bytes, true) // Set the bitmap indicating that this validator signed. + consensus.nilBitmap[recvMsg.ViewID].SetKey(senderKey.Bytes, true) // Set the bitmap indicating that this validator signed. } else { // m1 type message if consensus.BlockVerifier(preparedBlock); err != nil { consensus.getLogger().Error().Err(err).Msg("[onViewChange] Prepared block verification failed") @@ -296,7 +300,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { Msg("[onViewChange] Already Received M1 Message From the Validator") return } - if !recvMsg.ViewchangeSig.VerifyHash(recvMsg.SenderPubkey.Object, recvMsg.Payload) { + if !recvMsg.ViewchangeSig.VerifyHash(senderKey.Object, recvMsg.Payload) { consensus.getLogger().Warn().Msg("[onViewChange] Failed to Verify Signature for M1 Type Viewchange Message") return } @@ -343,7 +347,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { copy(preparedMsg.BlockHash[:], recvMsg.Payload[:32]) preparedMsg.Payload = make([]byte, len(recvMsg.Payload)-32) copy(preparedMsg.Payload[:], recvMsg.Payload[32:]) - preparedMsg.SenderPubkey = newLeaderKey + preparedMsg.SenderPubkeys = []*bls.PublicKeyWrapper{newLeaderKey} consensus.getLogger().Info().Msg("[onViewChange] New Leader Prepared Message Added") consensus.FBFTLog.AddMessage(&preparedMsg) @@ -354,7 +358,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { Str("validatorPubKey", senderKey.Bytes.Hex()). Msg("[onViewChange] Add M1 (prepared) type message") consensus.bhpSigs[recvMsg.ViewID][senderKey.Bytes.Hex()] = recvMsg.ViewchangeSig - consensus.bhpBitmap[recvMsg.ViewID].SetKey(recvMsg.SenderPubkey.Bytes, true) // Set the bitmap indicating that this validator signed. + consensus.bhpBitmap[recvMsg.ViewID].SetKey(senderKey.Bytes, true) // Set the bitmap indicating that this validator signed. } // check and add viewID (m3 type) message signature @@ -366,7 +370,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { } viewIDHash := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDHash, recvMsg.ViewID) - if !recvMsg.ViewidSig.VerifyHash(recvMsg.SenderPubkey.Object, viewIDHash) { + if !recvMsg.ViewidSig.VerifyHash(senderKey.Object, viewIDHash) { consensus.getLogger().Warn(). Uint64("MsgViewID", recvMsg.ViewID). Msg("[onViewChange] Failed to Verify M3 Message Signature") @@ -378,7 +382,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { consensus.viewIDSigs[recvMsg.ViewID][senderKey.Bytes.Hex()] = recvMsg.ViewidSig // Set the bitmap indicating that this validator signed. - consensus.viewIDBitmap[recvMsg.ViewID].SetKey(recvMsg.SenderPubkey.Bytes, true) + consensus.viewIDBitmap[recvMsg.ViewID].SetKey(senderKey.Bytes, true) consensus.getLogger().Info(). Int("have", len(consensus.viewIDSigs[recvMsg.ViewID])). Int64("total", consensus.Decider.ParticipantsCount()). @@ -430,7 +434,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { if _, err := consensus.Decider.SubmitVote( quorum.Commit, - key.Pub.Bytes, + []bls.SerializedPublicKey{key.Pub.Bytes}, key.Pri.SignHash(commitPayload), common.BytesToHash(consensus.blockHash[:]), block.NumberU64(), @@ -491,7 +495,12 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { return } - senderKey := recvMsg.SenderPubkey + if len(recvMsg.SenderPubkeys) != 1 { + consensus.getLogger().Error().Msg("[onNewView] multiple signers in view change message.") + return + } + senderKey := recvMsg.SenderPubkeys[0] + consensus.vcLock.Lock() defer consensus.vcLock.Unlock() @@ -594,7 +603,7 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { copy(preparedMsg.BlockHash[:], blockHash[:]) preparedMsg.Payload = make([]byte, len(recvMsg.Payload)-32) copy(preparedMsg.Payload[:], recvMsg.Payload[32:]) - preparedMsg.SenderPubkey = senderKey + preparedMsg.SenderPubkeys = []*bls.PublicKeyWrapper{senderKey} consensus.FBFTLog.AddMessage(&preparedMsg) if hasBlock { @@ -624,24 +633,46 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { preparedBlock.Epoch(), preparedBlock.Hash(), preparedBlock.NumberU64(), preparedBlock.Header().ViewID().Uint64()) groupID := []nodeconfig.GroupID{ nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} - for _, key := range consensus.priKey { + + priKeys := []*bls.PrivateKeyWrapper{} + p2pMsgs := []*NetworkMessage{} + for i, key := range consensus.priKey { if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { continue } + priKeys = append(priKeys, &consensus.priKey[i]) + if !consensus.MultiSig { + network, err := consensus.construct( + msg_pb.MessageType_COMMIT, + commitPayload, + []*bls.PrivateKeyWrapper{&key}, + ) + if err != nil { + consensus.getLogger().Err(err).Msg("could not create commit message") + return + } + p2pMsgs = append(p2pMsgs, network) + } + } + + if consensus.MultiSig { network, err := consensus.construct( msg_pb.MessageType_COMMIT, commitPayload, - &key, + priKeys, ) if err != nil { consensus.getLogger().Err(err).Msg("could not create commit message") return } - msgToSend := network.Bytes + p2pMsgs = append(p2pMsgs, network) + } + + for _, p2pMsg := range p2pMsgs { consensus.getLogger().Info().Msg("onNewView === commit") consensus.host.SendMessageToGroups( groupID, - p2p.ConstructMessage(msgToSend), + p2p.ConstructMessage(p2pMsg.Bytes), ) } consensus.getLogger().Info(). diff --git a/consensus/votepower/roster.go b/consensus/votepower/roster.go index 1dd43174f..0788e620b 100644 --- a/consensus/votepower/roster.go +++ b/consensus/votepower/roster.go @@ -3,6 +3,7 @@ package votepower import ( "encoding/hex" "encoding/json" + "fmt" "math/big" "sort" @@ -24,23 +25,23 @@ var ( // Ballot is a vote cast by a validator type Ballot struct { - SignerPubKey bls.SerializedPublicKey `json:"bls-public-key"` - BlockHeaderHash common.Hash `json:"block-header-hash"` - Signature []byte `json:"bls-signature"` - Height uint64 `json:"block-height"` - ViewID uint64 `json:"view-id"` + SignerPubKeys []bls.SerializedPublicKey `json:"bls-public-keys"` + BlockHeaderHash common.Hash `json:"block-header-hash"` + Signature []byte `json:"bls-signature"` + Height uint64 `json:"block-height"` + ViewID uint64 `json:"view-id"` } // MarshalJSON .. func (b Ballot) MarshalJSON() ([]byte, error) { return json.Marshal(struct { - A string `json:"bls-public-key"` + A string `json:"bls-public-keys"` B string `json:"block-header-hash"` C string `json:"bls-signature"` E uint64 `json:"block-height"` F uint64 `json:"view-id"` }{ - b.SignerPubKey.Hex(), + fmt.Sprint(b.SignerPubKeys), b.BlockHeaderHash.Hex(), hex.EncodeToString(b.Signature), b.Height, diff --git a/crypto/bls/mask.go b/crypto/bls/mask.go index 5e1297ca7..7bac04db5 100644 --- a/crypto/bls/mask.go +++ b/crypto/bls/mask.go @@ -66,7 +66,7 @@ func AggregateSig(sigs []*bls.Sign) *bls.Sign { // Mask represents a cosigning participation bitmask. type Mask struct { Bitmap []byte - Publics []*bls.PublicKey + Publics []*PublicKeyWrapper PublicsIndex map[SerializedPublicKey]int AggregatePublic *bls.PublicKey } @@ -77,9 +77,9 @@ type Mask struct { // bitmask to 1 (enabled). func NewMask(publics []PublicKeyWrapper, myKey *PublicKeyWrapper) (*Mask, error) { index := map[SerializedPublicKey]int{} - publicKeys := make([]*bls.PublicKey, len(publics)) + publicKeys := make([]*PublicKeyWrapper, len(publics)) for i, key := range publics { - publicKeys[i] = key.Object + publicKeys[i] = &publics[i] index[key.Bytes] = i } m := &Mask{ @@ -135,11 +135,11 @@ func (m *Mask) SetMask(mask []byte) error { msk := byte(1) << uint(i&7) if ((m.Bitmap[byt] & msk) == 0) && ((mask[byt] & msk) != 0) { m.Bitmap[byt] ^= msk // flip bit in Bitmap from 0 to 1 - m.AggregatePublic.Add(m.Publics[i]) + m.AggregatePublic.Add(m.Publics[i].Object) } if ((m.Bitmap[byt] & msk) != 0) && ((mask[byt] & msk) == 0) { m.Bitmap[byt] ^= msk // flip bit in Bitmap from 1 to 0 - m.AggregatePublic.Sub(m.Publics[i]) + m.AggregatePublic.Sub(m.Publics[i].Object) } } return nil @@ -155,11 +155,11 @@ func (m *Mask) SetBit(i int, enable bool) error { msk := byte(1) << uint(i&7) if ((m.Bitmap[byt] & msk) == 0) && enable { m.Bitmap[byt] ^= msk // flip bit in Bitmap from 0 to 1 - m.AggregatePublic.Add(m.Publics[i]) + m.AggregatePublic.Add(m.Publics[i].Object) } if ((m.Bitmap[byt] & msk) != 0) && !enable { m.Bitmap[byt] ^= msk // flip bit in Bitmap from 1 to 0 - m.AggregatePublic.Sub(m.Publics[i]) + m.AggregatePublic.Sub(m.Publics[i].Object) } return nil } @@ -173,17 +173,37 @@ func (m *Mask) GetPubKeyFromMask(flag bool) []*bls.PublicKey { msk := byte(1) << uint(i&7) if flag { if (m.Bitmap[byt] & msk) != 0 { - pubKeys = append(pubKeys, m.Publics[i]) + pubKeys = append(pubKeys, m.Publics[i].Object) } } else { if (m.Bitmap[byt] & msk) == 0 { - pubKeys = append(pubKeys, m.Publics[i]) + pubKeys = append(pubKeys, m.Publics[i].Object) } } } return pubKeys } +// GetSignedPubKeysFromBitmap will return pubkeys that are signed based on the specified bitmap. +func (m *Mask) GetSignedPubKeysFromBitmap(bitmap []byte) ([]*PublicKeyWrapper, error) { + if m.Len() != len(bitmap) { + return nil, errors.Errorf( + "mismatching bitmap lengths expectedBitmapLength %d providedBitmapLength %d", + m.Len(), + len(bitmap), + ) + } + pubKeys := []*PublicKeyWrapper{} + for i := range m.Publics { + byt := i >> 3 + msk := byte(1) << uint(i&7) + if (bitmap[byt] & msk) != 0 { + pubKeys = append(pubKeys, m.Publics[i]) + } + } + return pubKeys, nil +} + // IndexEnabled checks whether the given index is enabled in the Bitmap or not. func (m *Mask) IndexEnabled(i int) (bool, error) { if i >= len(m.Publics) { @@ -213,6 +233,25 @@ func (m *Mask) SetKey(public SerializedPublicKey, enable bool) error { return errors.New("key not found") } +// SetKeysAtomic set the bit in the Bitmap for the given cosigners only when all the cosigners are present in the map. +func (m *Mask) SetKeysAtomic(publics []*PublicKeyWrapper, enable bool) error { + indexes := make([]int, len(publics)) + for i, key := range publics { + index, found := m.PublicsIndex[key.Bytes] + if !found { + return errors.New("key not found") + } + indexes[i] = index + } + for _, index := range indexes { + err := m.SetBit(index, enable) + if err != nil { + return err + } + } + return nil +} + // CountEnabled returns the number of enabled nodes in the CoSi participation // Bitmap. func (m *Mask) CountEnabled() int { diff --git a/crypto/bls/mask_test.go b/crypto/bls/mask_test.go index aa25dbed5..c198f9e9e 100644 --- a/crypto/bls/mask_test.go +++ b/crypto/bls/mask_test.go @@ -226,6 +226,12 @@ func TestEnableKeyFunctions(test *testing.T) { if err := mask.SetKey(pubKey4.Bytes, true); err == nil { test.Error("Expected key nout found error") } + + enabledKeysFromBitmap, _ := mask.GetSignedPubKeysFromBitmap(mask.Bitmap) + + if len(enabledKeysFromBitmap) != 1 { + test.Error("Count of enabled keys from bitmap doesn't match") + } } func TestCopyParticipatingMask(test *testing.T) { diff --git a/crypto/pki/utils.go b/crypto/pki/utils.go deleted file mode 100644 index ade116b46..000000000 --- a/crypto/pki/utils.go +++ /dev/null @@ -1,20 +0,0 @@ -package pki - -import ( - "encoding/binary" - - "github.com/harmony-one/bls/ffi/go/bls" -) - -func init() { - bls.Init(bls.BLS12_381) -} - -// GetBLSPrivateKeyFromInt returns bls private key -func GetBLSPrivateKeyFromInt(value int) *bls.SecretKey { - priKey := [32]byte{} - binary.LittleEndian.PutUint32(priKey[:], uint32(value)) - var privateKey bls.SecretKey - privateKey.SetLittleEndian(priKey[:]) - return &privateKey -} diff --git a/crypto/pki/utils_test.go b/crypto/pki/utils_test.go deleted file mode 100644 index 365bb4a6a..000000000 --- a/crypto/pki/utils_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package pki - -import ( - "encoding/binary" - "testing" - "time" - - "github.com/harmony-one/bls/ffi/go/bls" -) - -func TestGetBLSPrivateKeyFromInt(test *testing.T) { - t := time.Now().UnixNano() - privateKey1 := GetBLSPrivateKeyFromInt(int(t)) - priKey := [32]byte{} - binary.LittleEndian.PutUint32(priKey[:], uint32(t)) - var privateKey2 bls.SecretKey - privateKey2.SetLittleEndian(priKey[:]) - if !privateKey1.IsEqual(&privateKey2) { - test.Error("two public address should be equal") - } -} diff --git a/node/node.go b/node/node.go index 9fc2c5315..665a01de3 100644 --- a/node/node.go +++ b/node/node.go @@ -350,11 +350,12 @@ type withError struct { } var ( - errNotRightKeySize = errors.New("key received over wire is wrong size") - errNoSenderPubKey = errors.New("no sender public BLS key in message") - errWrongShardID = errors.New("wrong shard id") - errInvalidNodeMsg = errors.New("invalid node message") - errIgnoreBeaconMsg = errors.New("ignore beacon sync block") + errNotRightKeySize = errors.New("key received over wire is wrong size") + errNoSenderPubKey = errors.New("no sender public BLS key in message") + errWrongSizeOfBitmap = errors.New("wrong size of sender bitmap") + errWrongShardID = errors.New("wrong shard id") + errInvalidNodeMsg = errors.New("invalid node message") + errIgnoreBeaconMsg = errors.New("ignore beacon sync block") ) // validateNodeMessage validate node message @@ -467,29 +468,30 @@ func (node *Node) validateShardBoundMessage( } maybeCon, maybeVC := m.GetConsensus(), m.GetViewchange() - senderKey := bls.SerializedPublicKey{} + senderKey := []byte{} + senderBitmap := []byte{} if maybeCon != nil { if maybeCon.ShardId != node.Consensus.ShardID { atomic.AddUint32(&node.NumInvalidMessages, 1) return nil, nil, true, errors.WithStack(errWrongShardID) } - copy(senderKey[:], maybeCon.SenderPubkey[:]) + senderKey = maybeCon.SenderPubkey + + if len(maybeCon.SenderPubkeyBitmap) > 0 { + senderBitmap = maybeCon.SenderPubkeyBitmap + } } else if maybeVC != nil { if maybeVC.ShardId != node.Consensus.ShardID { atomic.AddUint32(&node.NumInvalidMessages, 1) return nil, nil, true, errors.WithStack(errWrongShardID) } - copy(senderKey[:], maybeVC.SenderPubkey) + senderKey = maybeVC.SenderPubkey } else { atomic.AddUint32(&node.NumInvalidMessages, 1) return nil, nil, true, errors.WithStack(errNoSenderPubKey) } - if len(senderKey) != bls.PublicKeySizeInBytes { - atomic.AddUint32(&node.NumInvalidMessages, 1) - return nil, nil, true, errors.WithStack(errNotRightKeySize) - } // ignore mesage not intended for validator // but still forward them to the network if !node.Consensus.IsLeader() { @@ -500,13 +502,29 @@ func (node *Node) validateShardBoundMessage( } } - if !node.Consensus.IsValidatorInCommittee(senderKey) { - atomic.AddUint32(&node.NumSlotMessages, 1) - return nil, nil, true, errors.WithStack(shard.ErrValidNotInCommittee) + serializedKey := bls.SerializedPublicKey{} + if len(senderKey) > 0 { + if len(senderKey) != bls.PublicKeySizeInBytes { + atomic.AddUint32(&node.NumInvalidMessages, 1) + return nil, nil, true, errors.WithStack(errNotRightKeySize) + } + + copy(serializedKey[:], senderKey) + if !node.Consensus.IsValidatorInCommittee(serializedKey) { + atomic.AddUint32(&node.NumSlotMessages, 1) + return nil, nil, true, errors.WithStack(shard.ErrValidNotInCommittee) + } + } else { + count := node.Consensus.Decider.ParticipantsCount() + if (count+7)>>3 != int64(len(senderBitmap)) { + return nil, nil, true, errors.WithStack(errWrongSizeOfBitmap) + } } atomic.AddUint32(&node.NumValidMessages, 1) - return &m, &senderKey, false, nil + + // serializedKey will be empty for multiSig sender + return &m, &serializedKey, false, nil } var ( diff --git a/node/node_explorer.go b/node/node_explorer.go index c0c42b2ed..38fdd6887 100644 --- a/node/node_explorer.go +++ b/node/node_explorer.go @@ -33,7 +33,7 @@ var ( // explorerMessageHandler passes received message in node_handler to explorer service func (node *Node) explorerMessageHandler(ctx context.Context, msg *msg_pb.Message) error { if msg.Type == msg_pb.MessageType_COMMITTED { - recvMsg, err := consensus.ParseFBFTMessage(msg) + recvMsg, err := node.Consensus.ParseFBFTMessage(msg) if err != nil { utils.Logger().Error().Err(err). Msg("[Explorer] onCommitted unable to parse msg") @@ -79,7 +79,7 @@ func (node *Node) explorerMessageHandler(ctx context.Context, msg *msg_pb.Messag node.commitBlockForExplorer(block) } else if msg.Type == msg_pb.MessageType_PREPARED { - recvMsg, err := consensus.ParseFBFTMessage(msg) + recvMsg, err := node.Consensus.ParseFBFTMessage(msg) if err != nil { utils.Logger().Error().Err(err).Msg("[Explorer] Unable to parse Prepared msg") return err diff --git a/staking/slash/double-sign.go b/staking/slash/double-sign.go index d3ceae58a..b31d4d89a 100644 --- a/staking/slash/double-sign.go +++ b/staking/slash/double-sign.go @@ -1,6 +1,7 @@ package slash import ( + "bytes" "encoding/hex" "encoding/json" "math/big" @@ -73,9 +74,9 @@ type ConflictingVotes struct { // Vote is the vote of the double signer type Vote struct { - SignerPubKey bls.SerializedPublicKey `json:"bls-public-key"` - BlockHeaderHash common.Hash `json:"block-header-hash"` - Signature []byte `json:"bls-signature"` + SignerPubKeys []bls.SerializedPublicKey `json:"bls-public-keys"` + BlockHeaderHash common.Hash `json:"block-header-hash"` + Signature []byte `json:"bls-signature"` } // Record is an proof of a slashing made by a witness of a double-signing event @@ -113,13 +114,13 @@ func (r Records) String() string { } var ( - errBallotSignerKeysNotSame = errors.New("conflicting ballots must have same signer key") - errReporterAndOffenderSame = errors.New("reporter and offender cannot be same") - errAlreadyBannedValidator = errors.New("cannot slash on already banned validator") - errSignerKeyNotRightSize = errors.New("bls keys from slash candidate not right side") - errSlashFromFutureEpoch = errors.New("cannot have slash from future epoch") - errSlashBeforeStakingEpoch = errors.New("cannot have slash before staking epoch") - errSlashBlockNoConflict = errors.New("cannot slash for signing on non-conflicting blocks") + errNoMatchingDoubleSignKeys = errors.New("no matching double sign keys") + errReporterAndOffenderSame = errors.New("reporter and offender cannot be same") + errAlreadyBannedValidator = errors.New("cannot slash on already banned validator") + errSignerKeyNotRightSize = errors.New("bls keys from slash candidate not right side") + errSlashFromFutureEpoch = errors.New("cannot have slash from future epoch") + errSlashBeforeStakingEpoch = errors.New("cannot have slash before staking epoch") + errSlashBlockNoConflict = errors.New("cannot slash for signing on non-conflicting blocks") ) // MarshalJSON .. @@ -170,23 +171,31 @@ func Verify( first, second := candidate.Evidence.FirstVote, candidate.Evidence.SecondVote - k1, k2 := len(first.SignerPubKey), len(second.SignerPubKey) - if k1 != bls.PublicKeySizeInBytes || - k2 != bls.PublicKeySizeInBytes { - return errors.Wrapf( - errSignerKeyNotRightSize, "cast key %d double-signed key %d", k1, k2, - ) + + for _, pubKey := range append(first.SignerPubKeys, second.SignerPubKeys...) { + if len(pubKey) != bls.PublicKeySizeInBytes { + return errors.Wrapf( + errSignerKeyNotRightSize, "double-signed key %x", pubKey, + ) + } } if first.BlockHeaderHash == second.BlockHeaderHash { return errors.Wrapf(errSlashBlockNoConflict, "first %v+ second %v+", first, second) } - if shard.CompareBLSPublicKey(first.SignerPubKey, second.SignerPubKey) != 0 { - k1, k2 := first.SignerPubKey.Hex(), second.SignerPubKey.Hex() - return errors.Wrapf( - errBallotSignerKeysNotSame, "%s %s", k1, k2, - ) + doubleSignKeys := []bls.SerializedPublicKey{} + for _, pubKey1 := range first.SignerPubKeys { + for _, pubKey2 := range second.SignerPubKeys { + if shard.CompareBLSPublicKey(pubKey1, pubKey2) == 0 { + doubleSignKeys = append(doubleSignKeys, pubKey1) + break + } + } + } + + if len(doubleSignKeys) == 0 { + return errNoMatchingDoubleSignKeys } currentEpoch := chain.CurrentBlock().Epoch() // the slash can't come from the future (shard chain's epoch can't be larger than beacon chain's) @@ -212,14 +221,25 @@ func Verify( ) } - if addr, err := subCommittee.AddressForBLSKey( - second.SignerPubKey, - ); err != nil { - return err - } else if *addr != candidate.Evidence.Offender { - return errors.Errorf("offender address (%x) does not match the signer's address (%x)", candidate.Evidence.Offender, addr) + signerFound := false + addrs := []common.Address{} + for _, pubKey := range doubleSignKeys { + addr, err := subCommittee.AddressForBLSKey( + pubKey, + ) + if err != nil { + return err + } + + if *addr == candidate.Evidence.Offender { + signerFound = true + } + addrs = append(addrs, *addr) } + if !signerFound { + return errors.Errorf("offender address (%x) does not match the signer's address (%x)", candidate.Evidence.Offender, addrs) + } // last ditch check if hash.FromRLPNew256( candidate.Evidence.FirstVote, @@ -229,8 +249,8 @@ func Verify( return errors.Wrapf( errBallotsNotDiff, "%s %s", - candidate.Evidence.FirstVote.SignerPubKey.Hex(), - candidate.Evidence.SecondVote.SignerPubKey.Hex(), + candidate.Evidence.FirstVote.SignerPubKeys, + candidate.Evidence.SecondVote.SignerPubKeys, ) } @@ -246,8 +266,13 @@ func Verify( return err } - if publicKey, err = bls.BytesToBLSPublicKey(ballot.SignerPubKey[:]); err != nil { - return err + for _, pubKey := range ballot.SignerPubKeys { + publicKeyObj, err := bls.BytesToBLSPublicKey(pubKey[:]) + + if err != nil { + return err + } + publicKey.Add(publicKeyObj) } // slash verification only happens in staking era, therefore want commit payload for staking epoch commitPayload := consensus_sig.ConstructCommitPayload(chain, @@ -504,15 +529,28 @@ func Rate(votingPower *votepower.Roster, records Records) numeric.Dec { rate := numeric.ZeroDec() for i := range records { - key := records[i].Evidence.SecondVote.SignerPubKey - if card, exists := votingPower.Voters[key]; exists { - rate = rate.Add(card.GroupPercent) - } else { - utils.Logger().Debug(). - RawJSON("roster", []byte(votingPower.String())). - RawJSON("double-sign-record", []byte(records[i].String())). - Msg("did not have offenders voter card in roster as expected") + doubleSignKeys := []bls.SerializedPublicKey{} + for _, pubKey1 := range records[i].Evidence.FirstVote.SignerPubKeys { + for _, pubKey2 := range records[i].Evidence.SecondVote.SignerPubKeys { + if shard.CompareBLSPublicKey(pubKey1, pubKey2) == 0 { + doubleSignKeys = append(doubleSignKeys, pubKey1) + break + } + } } + + for _, key := range doubleSignKeys { + if card, exists := votingPower.Voters[key]; exists && + bytes.Equal(card.EarningAccount[:], records[i].Evidence.Offender[:]) { + rate = rate.Add(card.GroupPercent) + } else { + utils.Logger().Debug(). + RawJSON("roster", []byte(votingPower.String())). + RawJSON("double-sign-record", []byte(records[i].String())). + Msg("did not have offenders voter card in roster as expected") + } + } + } if rate.LT(oneDoubleSignerRate) { diff --git a/staking/slash/double-sign_test.go b/staking/slash/double-sign_test.go index 5a4edf338..6c6e64c3c 100644 --- a/staking/slash/double-sign_test.go +++ b/staking/slash/double-sign_test.go @@ -145,7 +145,7 @@ func TestVerify(t *testing.T) { sdb: defaultTestStateDB(), chain: defaultFakeBlockChain(), - expErr: errBallotSignerKeysNotSame, + expErr: errNoMatchingDoubleSignKeys, }, { // block is in the future @@ -692,9 +692,9 @@ func TestRate(t *testing.T) { keyPairs[2].Pub(): numeric.NewDecWithPrec(3, 2), }), records: Records{ - makeEmptyRecordWithSecondSignerKey(keyPairs[0].Pub()), - makeEmptyRecordWithSecondSignerKey(keyPairs[1].Pub()), - makeEmptyRecordWithSecondSignerKey(keyPairs[2].Pub()), + makeEmptyRecordWithSignerKey(keyPairs[0].Pub()), + makeEmptyRecordWithSignerKey(keyPairs[1].Pub()), + makeEmptyRecordWithSignerKey(keyPairs[2].Pub()), }, expRate: numeric.NewDecWithPrec(6, 2), }, @@ -703,7 +703,7 @@ func TestRate(t *testing.T) { keyPairs[0].Pub(): numeric.NewDecWithPrec(1, 2), }), records: Records{ - makeEmptyRecordWithSecondSignerKey(keyPairs[0].Pub()), + makeEmptyRecordWithSignerKey(keyPairs[0].Pub()), }, expRate: oneDoubleSignerRate, }, @@ -719,9 +719,9 @@ func TestRate(t *testing.T) { keyPairs[3].Pub(): numeric.NewDecWithPrec(3, 2), }), records: Records{ - makeEmptyRecordWithSecondSignerKey(keyPairs[0].Pub()), - makeEmptyRecordWithSecondSignerKey(keyPairs[1].Pub()), - makeEmptyRecordWithSecondSignerKey(keyPairs[2].Pub()), + makeEmptyRecordWithSignerKey(keyPairs[0].Pub()), + makeEmptyRecordWithSignerKey(keyPairs[1].Pub()), + makeEmptyRecordWithSignerKey(keyPairs[2].Pub()), }, expRate: numeric.NewDecWithPrec(3, 2), }, @@ -735,9 +735,10 @@ func TestRate(t *testing.T) { } -func makeEmptyRecordWithSecondSignerKey(pub bls.SerializedPublicKey) Record { +func makeEmptyRecordWithSignerKey(pub bls.SerializedPublicKey) Record { var r Record - r.Evidence.SecondVote.SignerPubKey = pub + r.Evidence.SecondVote.SignerPubKeys = []bls.SerializedPublicKey{pub} + r.Evidence.FirstVote.SignerPubKeys = []bls.SerializedPublicKey{pub} return r } @@ -774,7 +775,7 @@ func defaultSlashRecord() Record { func makeVoteData(kp blsKeyPair, block *types.Block) Vote { return Vote{ - SignerPubKey: kp.Pub(), + SignerPubKeys: []bls.SerializedPublicKey{kp.Pub()}, BlockHeaderHash: block.Hash(), Signature: kp.Sign(block), } @@ -1061,7 +1062,7 @@ func copyConflictingVotes(cv ConflictingVotes) ConflictingVotes { // copyVote makes a deep copy of slash.Vote func copyVote(v Vote) Vote { cp := Vote{ - SignerPubKey: v.SignerPubKey, + SignerPubKeys: v.SignerPubKeys, BlockHeaderHash: v.BlockHeaderHash, } if v.Signature != nil { diff --git a/staking/slash/test/copy.go b/staking/slash/test/copy.go index 3964b3b86..3218ffacb 100644 --- a/staking/slash/test/copy.go +++ b/staking/slash/test/copy.go @@ -47,7 +47,7 @@ func CopyConflictingVotes(cv slash.ConflictingVotes) slash.ConflictingVotes { // CopyVote makes a deep copy of slash.Vote func CopyVote(v slash.Vote) slash.Vote { cp := slash.Vote{ - SignerPubKey: v.SignerPubKey, + SignerPubKeys: v.SignerPubKeys, BlockHeaderHash: v.BlockHeaderHash, } if v.Signature != nil { diff --git a/staking/slash/test/copy_test.go b/staking/slash/test/copy_test.go index 9f6e23600..d8314c674 100644 --- a/staking/slash/test/copy_test.go +++ b/staking/slash/test/copy_test.go @@ -89,13 +89,13 @@ var ( } nonZeroVote1 = slash.Vote{ - SignerPubKey: bls.SerializedPublicKey{1}, + SignerPubKeys: []bls.SerializedPublicKey{{1}}, BlockHeaderHash: common.Hash{2}, Signature: []byte{1, 2, 3}, } nonZeroVote2 = slash.Vote{ - SignerPubKey: bls.SerializedPublicKey{3}, + SignerPubKeys: []bls.SerializedPublicKey{{3}}, BlockHeaderHash: common.Hash{4}, Signature: []byte{4, 5, 6}, }