From 23cdabc182f68d264feb37b005916affafbffb55 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Fri, 14 Jun 2019 16:40:17 -0700 Subject: [PATCH 1/3] Fix consensus stability --- api/proto/message/message.pb.go | 173 ++++++++++++++++++-------------- api/proto/message/message.proto | 33 +++--- api/service/syncing/syncing.go | 3 + consensus/consensus_service.go | 1 + consensus/consensus_v2.go | 18 ++-- drand/drand_leader.go | 6 ++ drand/drand_leader_msg.go | 1 + drand/drand_validator_msg.go | 2 +- node/node_handler.go | 7 +- node/node_newblock.go | 22 ++-- 10 files changed, 156 insertions(+), 110 deletions(-) diff --git a/api/proto/message/message.pb.go b/api/proto/message/message.pb.go index 853a34e46..a399cf042 100644 --- a/api/proto/message/message.pb.go +++ b/api/proto/message/message.pb.go @@ -523,9 +523,10 @@ func (m *StakingRequest) GetNodeId() string { type ConsensusRequest struct { ViewId uint32 `protobuf:"varint,1,opt,name=view_id,json=viewId,proto3" json:"view_id,omitempty"` BlockNum uint64 `protobuf:"varint,2,opt,name=block_num,json=blockNum,proto3" json:"block_num,omitempty"` - BlockHash []byte `protobuf:"bytes,3,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` - SenderPubkey []byte `protobuf:"bytes,4,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` - Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` + ShardId uint32 `protobuf:"varint,3,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` + BlockHash []byte `protobuf:"bytes,4,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` + SenderPubkey []byte `protobuf:"bytes,5,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` + Payload []byte `protobuf:"bytes,6,opt,name=payload,proto3" json:"payload,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -570,6 +571,13 @@ func (m *ConsensusRequest) GetBlockNum() uint64 { return 0 } +func (m *ConsensusRequest) GetShardId() uint32 { + if m != nil { + return m.ShardId + } + return 0 +} + func (m *ConsensusRequest) GetBlockHash() []byte { if m != nil { return m.BlockHash @@ -592,9 +600,10 @@ func (m *ConsensusRequest) GetPayload() []byte { } type DrandRequest struct { - SenderPubkey []byte `protobuf:"bytes,1,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` - BlockHash []byte `protobuf:"bytes,2,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` - Payload []byte `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + ShardId uint32 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` + SenderPubkey []byte `protobuf:"bytes,2,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` + BlockHash []byte `protobuf:"bytes,3,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"` + Payload []byte `protobuf:"bytes,4,opt,name=payload,proto3" json:"payload,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -625,6 +634,13 @@ func (m *DrandRequest) XXX_DiscardUnknown() { var xxx_messageInfo_DrandRequest proto.InternalMessageInfo +func (m *DrandRequest) GetShardId() uint32 { + if m != nil { + return m.ShardId + } + return 0 +} + func (m *DrandRequest) GetSenderPubkey() []byte { if m != nil { return m.SenderPubkey @@ -649,17 +665,18 @@ func (m *DrandRequest) GetPayload() []byte { type ViewChangeRequest struct { ViewId uint32 `protobuf:"varint,1,opt,name=view_id,json=viewId,proto3" json:"view_id,omitempty"` BlockNum uint64 `protobuf:"varint,2,opt,name=block_num,json=blockNum,proto3" json:"block_num,omitempty"` - SenderPubkey []byte `protobuf:"bytes,3,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` - LeaderPubkey []byte `protobuf:"bytes,4,opt,name=leader_pubkey,json=leaderPubkey,proto3" json:"leader_pubkey,omitempty"` - Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` - ViewchangeSig []byte `protobuf:"bytes,6,opt,name=viewchange_sig,json=viewchangeSig,proto3" json:"viewchange_sig,omitempty"` - ViewidSig []byte `protobuf:"bytes,7,opt,name=viewid_sig,json=viewidSig,proto3" json:"viewid_sig,omitempty"` + ShardId uint32 `protobuf:"varint,3,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"` + SenderPubkey []byte `protobuf:"bytes,4,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"` + LeaderPubkey []byte `protobuf:"bytes,5,opt,name=leader_pubkey,json=leaderPubkey,proto3" json:"leader_pubkey,omitempty"` + Payload []byte `protobuf:"bytes,6,opt,name=payload,proto3" json:"payload,omitempty"` + ViewchangeSig []byte `protobuf:"bytes,7,opt,name=viewchange_sig,json=viewchangeSig,proto3" json:"viewchange_sig,omitempty"` + ViewidSig []byte `protobuf:"bytes,8,opt,name=viewid_sig,json=viewidSig,proto3" json:"viewid_sig,omitempty"` // below is for newview message only // only need 1 valid m1 type message which is in payload - M2Aggsigs []byte `protobuf:"bytes,8,opt,name=m2_aggsigs,json=m2Aggsigs,proto3" json:"m2_aggsigs,omitempty"` - M2Bitmap []byte `protobuf:"bytes,9,opt,name=m2_bitmap,json=m2Bitmap,proto3" json:"m2_bitmap,omitempty"` - M3Aggsigs []byte `protobuf:"bytes,10,opt,name=m3_aggsigs,json=m3Aggsigs,proto3" json:"m3_aggsigs,omitempty"` - M3Bitmap []byte `protobuf:"bytes,11,opt,name=m3_bitmap,json=m3Bitmap,proto3" json:"m3_bitmap,omitempty"` + M2Aggsigs []byte `protobuf:"bytes,9,opt,name=m2_aggsigs,json=m2Aggsigs,proto3" json:"m2_aggsigs,omitempty"` + M2Bitmap []byte `protobuf:"bytes,10,opt,name=m2_bitmap,json=m2Bitmap,proto3" json:"m2_bitmap,omitempty"` + M3Aggsigs []byte `protobuf:"bytes,11,opt,name=m3_aggsigs,json=m3Aggsigs,proto3" json:"m3_aggsigs,omitempty"` + M3Bitmap []byte `protobuf:"bytes,12,opt,name=m3_bitmap,json=m3Bitmap,proto3" json:"m3_bitmap,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -704,6 +721,13 @@ func (m *ViewChangeRequest) GetBlockNum() uint64 { return 0 } +func (m *ViewChangeRequest) GetShardId() uint32 { + if m != nil { + return m.ShardId + } + return 0 +} + func (m *ViewChangeRequest) GetSenderPubkey() []byte { if m != nil { return m.SenderPubkey @@ -784,65 +808,66 @@ func init() { func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 914 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x95, 0xcd, 0x8e, 0xe2, 0x46, - 0x10, 0xc7, 0x31, 0x30, 0x18, 0x97, 0x0d, 0xd3, 0xdb, 0x49, 0x76, 0x9d, 0xc9, 0x46, 0x19, 0xb1, - 0x8a, 0x34, 0x5a, 0x29, 0xa3, 0x15, 0x1c, 0xa2, 0x48, 0xb9, 0x30, 0xa6, 0xb5, 0x58, 0x33, 0x63, - 0x48, 0x63, 0x76, 0x94, 0x93, 0x65, 0xa0, 0xc5, 0x58, 0x63, 0x6c, 0xe2, 0x36, 0xb3, 0xe2, 0x85, - 0x72, 0xc9, 0x3d, 0xe7, 0xe4, 0x79, 0xf2, 0x12, 0x51, 0xb7, 0x6d, 0xcc, 0xc7, 0x46, 0x91, 0x72, - 0xc8, 0x8d, 0xfa, 0x57, 0xfd, 0xaa, 0xab, 0xcb, 0x5d, 0x05, 0xb4, 0x56, 0x8c, 0x73, 0x7f, 0xc9, - 0xae, 0xd7, 0x49, 0x9c, 0xc6, 0x58, 0xcd, 0xcd, 0xce, 0xef, 0x35, 0x50, 0xef, 0xb3, 0xdf, 0xf8, - 0x7b, 0x30, 0x38, 0x4b, 0x9e, 0x83, 0x39, 0xf3, 0xd2, 0xed, 0x9a, 0x99, 0xca, 0xa5, 0x72, 0xd5, - 0xee, 0x7e, 0x7e, 0x5d, 0xa0, 0x93, 0xcc, 0xe9, 0x6e, 0xd7, 0x8c, 0xea, 0xbc, 0x34, 0xf0, 0x15, - 0xd4, 0x25, 0x50, 0x3d, 0x02, 0xf2, 0xc4, 0x12, 0x90, 0x11, 0xf8, 0x35, 0x68, 0x3c, 0x58, 0x46, - 0x7e, 0xba, 0x49, 0x98, 0x59, 0xbb, 0x54, 0xae, 0x0c, 0x5a, 0x0a, 0xb8, 0x07, 0x2a, 0x4f, 0xfd, - 0xa7, 0x20, 0x5a, 0x9a, 0xf5, 0x4b, 0xe5, 0x4a, 0xef, 0xbe, 0x2a, 0xcf, 0xce, 0x74, 0xca, 0x7e, - 0xd9, 0x30, 0x9e, 0x0e, 0x2b, 0xb4, 0x88, 0xc4, 0x3f, 0x80, 0x36, 0x8f, 0x23, 0xce, 0x22, 0xbe, - 0xe1, 0xe6, 0x99, 0xc4, 0xbe, 0xdc, 0x61, 0x56, 0xe1, 0x29, 0xc1, 0x32, 0x1a, 0x7f, 0x07, 0x67, - 0x8b, 0xc4, 0x8f, 0x16, 0x66, 0x43, 0x62, 0x5f, 0xec, 0xb0, 0x81, 0x50, 0x4b, 0x24, 0x8b, 0xc2, - 0x3f, 0x02, 0x3c, 0x07, 0xec, 0xe3, 0xfc, 0xd1, 0x8f, 0x96, 0xcc, 0x54, 0x25, 0x73, 0xb1, 0x63, - 0x3e, 0x04, 0xec, 0xa3, 0x25, 0x5d, 0x25, 0xb8, 0x17, 0x8f, 0x6f, 0xe0, 0x3c, 0x8c, 0xd3, 0x94, - 0x25, 0x5b, 0x2f, 0xc9, 0x02, 0xcc, 0xe6, 0xd1, 0x25, 0xef, 0x32, 0x7f, 0xc9, 0xb7, 0xc3, 0x03, - 0xe5, 0x46, 0x03, 0x35, 0x67, 0x3b, 0x7f, 0x28, 0xd0, 0xa4, 0x8c, 0xaf, 0xc5, 0x65, 0xfe, 0x8f, - 0x2f, 0x47, 0x00, 0x95, 0xe5, 0x67, 0xc7, 0xca, 0x0f, 0xa8, 0x77, 0xcd, 0xd3, 0xfa, 0x33, 0xff, - 0xb0, 0x42, 0xcf, 0xc3, 0x43, 0xe9, 0x06, 0xa0, 0x59, 0xe0, 0x9d, 0xf7, 0x70, 0x7e, 0x44, 0x60, - 0x13, 0xd4, 0x75, 0xe8, 0x6f, 0x59, 0xc2, 0xcd, 0xea, 0x65, 0xed, 0x4a, 0xa3, 0x85, 0x89, 0x2f, - 0xa0, 0x39, 0xf3, 0x43, 0x3f, 0x9a, 0x33, 0x6e, 0xd6, 0xa4, 0x6b, 0x67, 0x77, 0x7e, 0x53, 0xa0, - 0x7d, 0xd8, 0x3b, 0xfc, 0x2e, 0xbf, 0x58, 0xd6, 0x89, 0xd7, 0xff, 0xd0, 0xe2, 0xeb, 0xbd, 0x0b, - 0x7e, 0x03, 0xfa, 0x3a, 0x09, 0x9e, 0xfd, 0x94, 0x79, 0x4f, 0x6c, 0x2b, 0x3b, 0xa2, 0x51, 0xc8, - 0xa5, 0x5b, 0xb6, 0xc5, 0x2f, 0xa1, 0xe1, 0xaf, 0xe2, 0x4d, 0x94, 0xca, 0x7b, 0xd7, 0x68, 0x6e, - 0x75, 0xae, 0xa1, 0x2e, 0x7b, 0xa9, 0xc1, 0x19, 0x71, 0x5c, 0x42, 0x51, 0x05, 0x03, 0x34, 0x28, - 0x99, 0x4c, 0xef, 0x5c, 0xa4, 0xe0, 0x73, 0xd0, 0xc7, 0xb6, 0x75, 0xeb, 0x3d, 0xd8, 0x8e, 0x43, - 0x28, 0xaa, 0x76, 0x6e, 0xa1, 0x7d, 0xf8, 0x9a, 0xf1, 0x25, 0xe8, 0x69, 0xe2, 0x47, 0xdc, 0x9f, - 0xa7, 0x41, 0x1c, 0xc9, 0x9a, 0x0d, 0xba, 0x2f, 0xe1, 0x57, 0xa0, 0x46, 0xf1, 0x82, 0x79, 0xc1, - 0x22, 0x2f, 0xac, 0x21, 0x4c, 0x7b, 0xd1, 0xf9, 0x55, 0x01, 0x74, 0xfc, 0xc8, 0x45, 0xb4, 0x78, - 0x78, 0x22, 0x5a, 0xe4, 0x6a, 0xd1, 0x86, 0x30, 0xed, 0x05, 0xfe, 0x0a, 0xb4, 0x59, 0x18, 0xcf, - 0x9f, 0xbc, 0x68, 0xb3, 0x92, 0x89, 0xea, 0xb4, 0x29, 0x05, 0x67, 0xb3, 0xc2, 0x5f, 0x03, 0x64, - 0xce, 0x47, 0x9f, 0x3f, 0x16, 0xc3, 0x29, 0x95, 0xa1, 0xcf, 0x1f, 0xf1, 0x1b, 0x68, 0x71, 0x16, - 0x2d, 0x58, 0xe2, 0xad, 0x37, 0x33, 0xd1, 0xa1, 0xba, 0x8c, 0x30, 0x32, 0x71, 0x2c, 0x35, 0xf9, - 0xfd, 0xfc, 0x6d, 0x18, 0xfb, 0x0b, 0x39, 0x8a, 0x06, 0x2d, 0xcc, 0x4e, 0x08, 0xc6, 0xfe, 0x54, - 0x9d, 0xa6, 0x53, 0x3e, 0x91, 0xee, 0xb0, 0xa4, 0xea, 0x71, 0x49, 0x7b, 0xa7, 0xd5, 0x0e, 0x4f, - 0xfb, 0xab, 0x0a, 0x2f, 0x4e, 0x06, 0xf2, 0x3f, 0xf6, 0xe5, 0xa4, 0xd2, 0xda, 0x27, 0x2a, 0x7d, - 0x03, 0xad, 0x90, 0xf9, 0xa7, 0xdd, 0xc9, 0xc4, 0x7f, 0xeb, 0x0e, 0xfe, 0x16, 0xda, 0xe5, 0xaa, - 0xf0, 0x78, 0xb0, 0x94, 0x2b, 0xc9, 0xa0, 0xad, 0x52, 0x9d, 0x04, 0x4b, 0xd1, 0x0f, 0x21, 0x04, - 0x0b, 0x19, 0xa2, 0x66, 0xfd, 0xc8, 0x94, 0xdc, 0xbd, 0xea, 0x7a, 0xfe, 0x72, 0xc9, 0x83, 0x25, - 0x97, 0xdb, 0xc5, 0xa0, 0xda, 0xaa, 0xdb, 0xcf, 0x04, 0x71, 0xcb, 0x55, 0xd7, 0x9b, 0x05, 0xe9, - 0xca, 0x5f, 0x9b, 0x9a, 0xf4, 0x36, 0x57, 0xdd, 0x1b, 0x69, 0x4b, 0xb6, 0xb7, 0x63, 0x21, 0x67, - 0x7b, 0xfb, 0x6c, 0xaf, 0x60, 0xf5, 0x9c, 0xed, 0x65, 0xec, 0xdb, 0x21, 0xe8, 0x7b, 0x1b, 0x06, - 0xb7, 0x40, 0xb3, 0x46, 0xce, 0x84, 0x38, 0x93, 0xe9, 0x04, 0x55, 0xb0, 0x0e, 0xea, 0xc4, 0xed, - 0xdf, 0xda, 0xce, 0x7b, 0xa4, 0x88, 0x21, 0x19, 0xd0, 0xbe, 0x33, 0x40, 0x55, 0x8c, 0xa1, 0x6d, - 0xdd, 0xd9, 0xc4, 0x71, 0xbd, 0xc9, 0x74, 0x3c, 0x1e, 0x51, 0x17, 0xd5, 0xde, 0xfe, 0xa9, 0x80, - 0xbe, 0xb7, 0x7b, 0xf0, 0x05, 0xbc, 0x74, 0xc8, 0x83, 0x33, 0x1a, 0x10, 0xef, 0x86, 0xf4, 0xad, - 0x91, 0xe3, 0x15, 0xa9, 0x2a, 0xd8, 0x80, 0x66, 0xdf, 0x71, 0x46, 0x53, 0xc7, 0x22, 0x48, 0x11, - 0xa7, 0x8c, 0x29, 0x19, 0xf7, 0x29, 0x41, 0x55, 0xe1, 0xca, 0x8d, 0x01, 0xaa, 0x89, 0x69, 0xb4, - 0x46, 0xf7, 0xf7, 0xb6, 0x8b, 0xea, 0x59, 0x6d, 0xe2, 0xb7, 0x4b, 0x06, 0xe8, 0x0c, 0xb7, 0x01, - 0x3e, 0xd8, 0xe4, 0xc1, 0x1a, 0xf6, 0x9d, 0xf7, 0x04, 0x35, 0x44, 0x16, 0x87, 0x3c, 0x08, 0x09, - 0xa9, 0xc2, 0x29, 0x6b, 0xf5, 0x6c, 0xc7, 0x76, 0x11, 0x60, 0x04, 0x46, 0x66, 0xe7, 0xd9, 0x74, - 0xfc, 0x19, 0x9c, 0xdf, 0x8d, 0x5c, 0x97, 0xd0, 0x9f, 0x3d, 0x4a, 0x7e, 0x9a, 0x92, 0x89, 0x8b, - 0x8c, 0x6e, 0x1f, 0x5a, 0x56, 0x18, 0xb0, 0x28, 0xcd, 0x7b, 0x82, 0xdf, 0x81, 0x3a, 0x4e, 0xe2, - 0x39, 0xe3, 0x1c, 0xa3, 0xe3, 0x0d, 0x7b, 0xf1, 0x62, 0xa7, 0x14, 0x4b, 0xb0, 0x53, 0x99, 0x35, - 0xe4, 0xbf, 0x74, 0xef, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x27, 0x52, 0x45, 0x7b, 0xb6, 0x07, - 0x00, 0x00, + // 934 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x95, 0xdf, 0x6e, 0xe2, 0x46, + 0x14, 0xc6, 0x31, 0x10, 0x0c, 0xc7, 0x86, 0xcc, 0x4e, 0xdb, 0x5d, 0x6f, 0xba, 0x55, 0x23, 0x56, + 0x95, 0xa2, 0x95, 0x1a, 0xad, 0xe0, 0xa2, 0xaa, 0xd4, 0x1b, 0x02, 0xa3, 0xc4, 0x4a, 0x62, 0xe8, + 0xe0, 0x6c, 0xd4, 0x2b, 0x6b, 0x82, 0x47, 0xc4, 0x0a, 0xd8, 0xd4, 0x63, 0xb2, 0xe2, 0x05, 0xda, + 0x87, 0xe9, 0x7d, 0xaf, 0xbb, 0x6f, 0x56, 0xcd, 0x8c, 0xc1, 0xfc, 0xd9, 0xaa, 0x37, 0x55, 0xef, + 0x38, 0xdf, 0x39, 0xbf, 0x99, 0x73, 0x3e, 0xcf, 0x0c, 0xd0, 0x9c, 0x73, 0x21, 0xd8, 0x94, 0x9f, + 0x2f, 0xd2, 0x24, 0x4b, 0xb0, 0x99, 0x87, 0xed, 0x3f, 0x2b, 0x60, 0xde, 0xea, 0xdf, 0xf8, 0x07, + 0xb0, 0x05, 0x4f, 0x9f, 0xa3, 0x09, 0x0f, 0xb2, 0xd5, 0x82, 0x3b, 0xc6, 0xa9, 0x71, 0xd6, 0xea, + 0x7c, 0x79, 0xbe, 0x46, 0xc7, 0x3a, 0xe9, 0xaf, 0x16, 0x9c, 0x5a, 0xa2, 0x08, 0xf0, 0x19, 0x54, + 0x15, 0x50, 0xde, 0x03, 0xf2, 0x85, 0x15, 0xa0, 0x2a, 0xf0, 0x1b, 0x68, 0x88, 0x68, 0x1a, 0xb3, + 0x6c, 0x99, 0x72, 0xa7, 0x72, 0x6a, 0x9c, 0xd9, 0xb4, 0x10, 0x70, 0x17, 0x4c, 0x91, 0xb1, 0xa7, + 0x28, 0x9e, 0x3a, 0xd5, 0x53, 0xe3, 0xcc, 0xea, 0xbc, 0x2a, 0xf6, 0xd6, 0x3a, 0xe5, 0xbf, 0x2e, + 0xb9, 0xc8, 0xae, 0x4a, 0x74, 0x5d, 0x89, 0x7f, 0x84, 0xc6, 0x24, 0x89, 0x05, 0x8f, 0xc5, 0x52, + 0x38, 0x47, 0x0a, 0x7b, 0xbd, 0xc1, 0xfa, 0xeb, 0x4c, 0x01, 0x16, 0xd5, 0xf8, 0x7b, 0x38, 0x0a, + 0x53, 0x16, 0x87, 0x4e, 0x4d, 0x61, 0x5f, 0x6d, 0xb0, 0x81, 0x54, 0x0b, 0x44, 0x57, 0xe1, 0x9f, + 0x00, 0x9e, 0x23, 0xfe, 0x71, 0xf2, 0xc8, 0xe2, 0x29, 0x77, 0x4c, 0xc5, 0x9c, 0x6c, 0x98, 0x0f, + 0x11, 0xff, 0xd8, 0x57, 0xa9, 0x02, 0xdc, 0xaa, 0xc7, 0x17, 0x70, 0x3c, 0x4b, 0xb2, 0x8c, 0xa7, + 0xab, 0x20, 0xd5, 0x05, 0x4e, 0x7d, 0x6f, 0xc8, 0x1b, 0x9d, 0x2f, 0xf8, 0xd6, 0x6c, 0x47, 0xb9, + 0x68, 0x80, 0x99, 0xb3, 0xed, 0xbf, 0x0c, 0xa8, 0x53, 0x2e, 0x16, 0x72, 0x98, 0xff, 0xe3, 0xcb, + 0x11, 0x40, 0x45, 0xfb, 0x7a, 0x5b, 0xf5, 0x01, 0xad, 0x8e, 0x73, 0xd8, 0xbf, 0xce, 0x5f, 0x95, + 0xe8, 0xf1, 0x6c, 0x57, 0xba, 0x00, 0xa8, 0xaf, 0xf1, 0xf6, 0x25, 0x1c, 0xef, 0x11, 0xd8, 0x01, + 0x73, 0x31, 0x63, 0x2b, 0x9e, 0x0a, 0xa7, 0x7c, 0x5a, 0x39, 0x6b, 0xd0, 0x75, 0x88, 0x4f, 0xa0, + 0xfe, 0xc0, 0x66, 0x2c, 0x9e, 0x70, 0xe1, 0x54, 0x54, 0x6a, 0x13, 0xb7, 0xff, 0x30, 0xa0, 0xb5, + 0xeb, 0x1d, 0x7e, 0x9f, 0x0f, 0xa6, 0x9d, 0x78, 0xf3, 0x0f, 0x16, 0x9f, 0x6f, 0x0d, 0xf8, 0x2d, + 0x58, 0x8b, 0x34, 0x7a, 0x66, 0x19, 0x0f, 0x9e, 0xf8, 0x4a, 0x39, 0xd2, 0xa0, 0x90, 0x4b, 0xd7, + 0x7c, 0x85, 0x5f, 0x42, 0x8d, 0xcd, 0x93, 0x65, 0x9c, 0xa9, 0xb9, 0x2b, 0x34, 0x8f, 0xda, 0xe7, + 0x50, 0x55, 0x5e, 0x36, 0xe0, 0x88, 0x78, 0x3e, 0xa1, 0xa8, 0x84, 0x01, 0x6a, 0x94, 0x8c, 0xef, + 0x6e, 0x7c, 0x64, 0xe0, 0x63, 0xb0, 0x46, 0x6e, 0xff, 0x3a, 0xb8, 0x77, 0x3d, 0x8f, 0x50, 0x54, + 0x6e, 0x5f, 0x43, 0x6b, 0xf7, 0x34, 0xe3, 0x53, 0xb0, 0xb2, 0x94, 0xc5, 0x82, 0x4d, 0xb2, 0x28, + 0x89, 0x55, 0xcf, 0x36, 0xdd, 0x96, 0xf0, 0x2b, 0x30, 0xe3, 0x24, 0xe4, 0x41, 0x14, 0xe6, 0x8d, + 0xd5, 0x64, 0xe8, 0x86, 0xed, 0x4f, 0x06, 0xa0, 0xfd, 0x43, 0x2e, 0xab, 0xe5, 0xc1, 0x93, 0xd5, + 0x72, 0xad, 0x26, 0xad, 0xc9, 0xd0, 0x0d, 0xf1, 0xd7, 0xd0, 0x78, 0x98, 0x25, 0x93, 0xa7, 0x20, + 0x5e, 0xce, 0xd5, 0x42, 0x55, 0x5a, 0x57, 0x82, 0xb7, 0x9c, 0xe3, 0xd7, 0x50, 0x17, 0x8f, 0x2c, + 0x0d, 0x25, 0x56, 0x51, 0x98, 0xa9, 0x62, 0x37, 0xc4, 0xdf, 0x00, 0x68, 0xee, 0x91, 0x89, 0x47, + 0x75, 0x37, 0x6d, 0xaa, 0x57, 0xba, 0x62, 0xe2, 0x11, 0xbf, 0x85, 0xa6, 0xe0, 0x71, 0xc8, 0xd3, + 0x60, 0xb1, 0x7c, 0x90, 0xe6, 0x1d, 0xa9, 0x0a, 0x5b, 0x8b, 0x23, 0xa5, 0xa9, 0x4f, 0xcb, 0x56, + 0xb3, 0x84, 0xe9, 0xeb, 0x66, 0xd3, 0x75, 0xd8, 0xfe, 0xdd, 0x00, 0x7b, 0xfb, 0xc6, 0xed, 0x74, + 0x62, 0xec, 0x76, 0x72, 0xb0, 0x55, 0xf9, 0x33, 0x5b, 0xed, 0xb6, 0x5b, 0xd9, 0x6f, 0x77, 0xab, + 0x93, 0xea, 0x6e, 0x27, 0xbf, 0x55, 0xe0, 0xc5, 0xc1, 0x3d, 0xfe, 0xef, 0xed, 0x3c, 0x18, 0xa2, + 0xfa, 0x99, 0x21, 0xde, 0x42, 0x73, 0xc6, 0xd9, 0xa1, 0xa9, 0x5a, 0xfc, 0x37, 0x53, 0xf1, 0x77, + 0xd0, 0x2a, 0x1e, 0x9f, 0x40, 0x44, 0x53, 0xf5, 0x60, 0xd9, 0xb4, 0x59, 0xa8, 0xe3, 0x68, 0x2a, + 0xad, 0x92, 0x42, 0x14, 0xaa, 0x92, 0xba, 0xb6, 0x4a, 0x2b, 0x79, 0x7a, 0xde, 0x09, 0xd8, 0x74, + 0x2a, 0xa2, 0xa9, 0x70, 0x1a, 0x3a, 0x3d, 0xef, 0xf4, 0xb4, 0x20, 0x0d, 0x98, 0x77, 0x82, 0x87, + 0x28, 0x9b, 0xb3, 0x85, 0x03, 0x2a, 0x5b, 0x9f, 0x77, 0x2e, 0x54, 0xac, 0xd8, 0xee, 0x86, 0xb5, + 0x72, 0xb6, 0xbb, 0xcd, 0x76, 0xd7, 0xac, 0x9d, 0xb3, 0x5d, 0xcd, 0xbe, 0xbb, 0x02, 0x6b, 0xeb, + 0xcd, 0xc2, 0x4d, 0x68, 0xf4, 0x87, 0xde, 0x98, 0x78, 0xe3, 0xbb, 0x31, 0x2a, 0x61, 0x0b, 0xcc, + 0xb1, 0xdf, 0xbb, 0x76, 0xbd, 0x4b, 0x64, 0xc8, 0x6b, 0x37, 0xa0, 0x3d, 0x6f, 0x80, 0xca, 0x18, + 0x43, 0xab, 0x7f, 0xe3, 0x12, 0xcf, 0x0f, 0xc6, 0x77, 0xa3, 0xd1, 0x90, 0xfa, 0xa8, 0xf2, 0xee, + 0x93, 0x01, 0xd6, 0xd6, 0x6b, 0x86, 0x4f, 0xe0, 0xa5, 0x47, 0xee, 0xbd, 0xe1, 0x80, 0x04, 0x17, + 0xa4, 0xd7, 0x1f, 0x7a, 0xc1, 0x7a, 0xa9, 0x12, 0xb6, 0xa1, 0xde, 0xf3, 0xbc, 0xe1, 0x9d, 0xd7, + 0x27, 0xc8, 0x90, 0xbb, 0x8c, 0x28, 0x19, 0xf5, 0x28, 0x41, 0x65, 0x99, 0xca, 0x83, 0x01, 0xaa, + 0xc8, 0xfb, 0xdd, 0x1f, 0xde, 0xde, 0xba, 0x3e, 0xaa, 0xea, 0xde, 0xe4, 0x6f, 0x9f, 0x0c, 0xd0, + 0x11, 0x6e, 0x01, 0x7c, 0x70, 0xc9, 0x7d, 0xff, 0xaa, 0xe7, 0x5d, 0x12, 0x54, 0x93, 0xab, 0x78, + 0xe4, 0x5e, 0x4a, 0xc8, 0x94, 0x49, 0xd5, 0x6b, 0xe0, 0x7a, 0xae, 0x8f, 0x00, 0x23, 0xb0, 0x75, + 0x9c, 0xaf, 0x66, 0xe1, 0x2f, 0xe0, 0xf8, 0x66, 0xe8, 0xfb, 0x84, 0xfe, 0x12, 0x50, 0xf2, 0xf3, + 0x1d, 0x19, 0xfb, 0xc8, 0xee, 0xf4, 0xa0, 0xd9, 0x9f, 0x45, 0x3c, 0xce, 0x72, 0x4f, 0xf0, 0x7b, + 0x30, 0x47, 0x69, 0x32, 0xe1, 0x42, 0x60, 0xb4, 0xff, 0x66, 0x9f, 0xbc, 0xd8, 0x28, 0xeb, 0x67, + 0xb5, 0x5d, 0x7a, 0xa8, 0xa9, 0xff, 0xfd, 0xee, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe3, 0x60, + 0x9a, 0xe1, 0x08, 0x08, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/api/proto/message/message.proto b/api/proto/message/message.proto index 4b2d79ae5..893a06a94 100644 --- a/api/proto/message/message.proto +++ b/api/proto/message/message.proto @@ -82,31 +82,34 @@ message StakingRequest { message ConsensusRequest { uint32 view_id = 1; uint64 block_num = 2; - bytes block_hash = 3; - bytes sender_pubkey = 4; - bytes payload = 5; + uint32 shard_id = 3; + bytes block_hash = 4; + bytes sender_pubkey = 5; + bytes payload = 6; } message DrandRequest { - bytes sender_pubkey = 1; - bytes block_hash = 2; - bytes payload = 3; + uint32 shard_id = 1; + bytes sender_pubkey = 2; + bytes block_hash = 3; + bytes payload = 4; } message ViewChangeRequest { uint32 view_id = 1; uint64 block_num = 2; - bytes sender_pubkey = 3; - bytes leader_pubkey = 4; - bytes payload = 5; // message payload: either m1 type or m2 type - bytes viewchange_sig = 6; // signature on payload - bytes viewid_sig = 7; // signature on view_id + uint32 shard_id = 3; + bytes sender_pubkey = 4; + bytes leader_pubkey = 5; + bytes payload = 6; // message payload: either m1 type or m2 type + bytes viewchange_sig = 7; // signature on payload + bytes viewid_sig = 8; // signature on view_id // below is for newview message only // only need 1 valid m1 type message which is in payload - bytes m2_aggsigs = 8; // m2: |nil| - bytes m2_bitmap = 9; - bytes m3_aggsigs = 10; // m3: |viewID| - bytes m3_bitmap= 11; + bytes m2_aggsigs = 9; // m2: |nil| + bytes m2_bitmap = 10; + bytes m3_aggsigs = 11; // m3: |viewID| + bytes m3_bitmap= 12; } diff --git a/api/service/syncing/syncing.go b/api/service/syncing/syncing.go index 4f81ad537..4d0f0d62e 100644 --- a/api/service/syncing/syncing.go +++ b/api/service/syncing/syncing.go @@ -476,6 +476,9 @@ func (ss *StateSync) updateBlockAndStatus(block *types.Block, bc *core.BlockChai _, err := bc.InsertChain([]*types.Block{block}) if err != nil { utils.GetLogInstance().Debug("Error adding new block to blockchain", "Error", err) + + utils.GetLogInstance().Debug("Rolling back current block!", "block", bc.CurrentBlock()) + bc.Rollback([]common.Hash{bc.CurrentBlock().Hash()}) return false } ss.syncMux.Lock() diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 4c110b748..a8cb6ae5b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -101,6 +101,7 @@ func (consensus *Consensus) Prepare(chain consensus_engine.ChainReader, header * func (consensus *Consensus) populateMessageFields(request *msg_pb.ConsensusRequest) { request.ViewId = consensus.viewID request.BlockNum = consensus.blockNum + request.ShardId = consensus.ShardID // 32 byte block hash request.BlockHash = consensus.blockHash[:] diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 2e1f1c62b..91cdc53d2 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -37,6 +37,11 @@ func (consensus *Consensus) handleMessageUpdate(payload []byte) { return } + if msg.GetConsensus().ShardId != consensus.ShardID { + consensus.getLogger().Warn("Received consensus message from different shard", + "myShardId", consensus.ShardID, "receivedShardId", msg.GetConsensus().ShardId) + return + } switch msg.Type { case msg_pb.MessageType_ANNOUNCE: consensus.onAnnounce(msg) @@ -57,14 +62,14 @@ func (consensus *Consensus) handleMessageUpdate(payload []byte) { } // TODO: move to consensus_leader.go later -func (consensus *Consensus) tryAnnounce(block *types.Block) { +func (consensus *Consensus) announce(block *types.Block) { blockHash := block.Hash() copy(consensus.blockHash[:], blockHash[:]) // prepare message and broadcast to validators encodedBlock, err := rlp.EncodeToBytes(block) if err != nil { - consensus.getLogger().Debug("tryAnnounce Failed encoding block") + consensus.getLogger().Debug("announce Failed encoding block") return } consensus.block = encodedBlock @@ -77,7 +82,7 @@ func (consensus *Consensus) tryAnnounce(block *types.Block) { _ = protobuf.Unmarshal(msgPayload, msg) pbftMsg, err := ParsePbftMessage(msg) if err != nil { - consensus.getLogger().Warn("tryAnnounce unable to parse pbft message", "error", err) + consensus.getLogger().Warn("announce unable to parse pbft message", "error", err) return } @@ -174,13 +179,13 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { consensus.getLogger().Debug("viewID check failed", "msgViewID", recvMsg.ViewID, "msgBlockNum", recvMsg.BlockNum) return } - consensus.tryPrepare(&blockObj) + consensus.prepare(&blockObj) return } // tryPrepare will try to send prepare message -func (consensus *Consensus) tryPrepare(block *types.Block) { +func (consensus *Consensus) prepare(block *types.Block) { // if consensus.blockNum != block.NumberU64() || !consensus.pbftLog.HasMatchingViewAnnounce(consensus.blockNum, consensus.viewID, hash) { // consensus.getLogger().Debug("blockNum or announce message not match") // return @@ -780,7 +785,6 @@ func (consensus *Consensus) Start(blockChannel chan *types.Block, stopChan chan break } } - case <-consensus.syncReadyChan: consensus.SetBlockNum(consensus.ChainReader.CurrentHeader().Number.Uint64() + 1) consensus.ignoreViewIDCheck = true @@ -817,7 +821,7 @@ func (consensus *Consensus) Start(blockChannel chan *types.Block, stopChan chan startTime = time.Now() consensus.getLogger().Debug("STARTING CONSENSUS", "numTxs", len(newBlock.Transactions()), "consensus", consensus, "startTime", startTime, "publicKeys", len(consensus.PublicKeys)) - consensus.tryAnnounce(newBlock) + consensus.announce(newBlock) case msg := <-consensus.MsgChan: consensus.handleMessageUpdate(msg) diff --git a/drand/drand_leader.go b/drand/drand_leader.go index 88dfbe778..d80d879a7 100644 --- a/drand/drand_leader.go +++ b/drand/drand_leader.go @@ -89,6 +89,12 @@ func (dRand *DRand) ProcessMessageLeader(payload []byte) { utils.GetLogInstance().Error("Failed to unmarshal message payload.", "err", err, "dRand", dRand) } + if message.GetDrand().ShardId != dRand.ShardID { + utils.GetLogInstance().Warn("Received drand message from different shard", + "myShardId", dRand.ShardID, "receivedShardId", message.GetDrand().ShardId) + return + } + switch message.Type { case msg_pb.MessageType_DRAND_COMMIT: dRand.processCommitMessage(message) diff --git a/drand/drand_leader_msg.go b/drand/drand_leader_msg.go index 1335b707e..64479f362 100644 --- a/drand/drand_leader_msg.go +++ b/drand/drand_leader_msg.go @@ -19,6 +19,7 @@ func (dRand *DRand) constructInitMessage() []byte { drandMsg := message.GetDrand() drandMsg.SenderPubkey = dRand.pubKey.Serialize() drandMsg.BlockHash = dRand.blockHash[:] + drandMsg.ShardId = dRand.ShardID // Don't need the payload in init message marshaledMessage, err := dRand.signAndMarshalDRandMessage(message) if err != nil { diff --git a/drand/drand_validator_msg.go b/drand/drand_validator_msg.go index bcfd7c683..f1790cdf7 100644 --- a/drand/drand_validator_msg.go +++ b/drand/drand_validator_msg.go @@ -19,7 +19,7 @@ func (dRand *DRand) constructCommitMessage(vrf [32]byte, proof []byte) []byte { drandMsg := message.GetDrand() drandMsg.SenderPubkey = dRand.pubKey.Serialize() drandMsg.BlockHash = dRand.blockHash[:] - drandMsg.BlockHash = dRand.blockHash[:] + drandMsg.ShardId = dRand.ShardID drandMsg.Payload = append(vrf[:], proof...) // Adding the public key into payload so leader can verify the vrf // TODO: change the curve to follow the same curve with consensus, so the public key doesn't need to be attached. diff --git a/node/node_handler.go b/node/node_handler.go index 79648d30f..2913a6a49 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -805,10 +805,5 @@ func getBinaryPath() (argv0 string, err error) { // ConsensusMessageHandler passes received message in node_handler to consensus func (node *Node) ConsensusMessageHandler(msgPayload []byte) { - select { - case node.Consensus.MsgChan <- msgPayload: - case <-time.After(consensusTimeout): - //utils.GetLogInstance().Debug("[Consensus] ConsensusMessageHandler timeout", "duration", consensusTimeout, "msgPayload", len(msgPayload)) - } - return + node.Consensus.MsgChan <- msgPayload } diff --git a/node/node_newblock.go b/node/node_newblock.go index 579747571..3e611a6d7 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -17,7 +17,7 @@ import ( const ( DefaultThreshold = 1 FirstTimeThreshold = 2 - ConsensusTimeOut = 10 + ConsensusTimeOut = 30 PeriodicBlock = 1 * time.Second BlockPeriod = 10 * time.Second ) @@ -34,13 +34,22 @@ func (node *Node) WaitForConsensusReadyv2(readySignal chan struct{}, stopChan ch time.Sleep(30 * time.Second) // Wait for other nodes to be ready (test-only) firstTime := true + timeoutCount := 0 + var newBlock *types.Block for { // keep waiting for Consensus ready select { case <-stopChan: - utils.GetLogInstance().Debug("Consensus propose new block: STOPPED!") + utils.GetLogInstance().Debug("Consensus new block proposal: STOPPED!") return - + case <-time.After(ConsensusTimeOut * time.Second): + node.Consensus.ResetState() + timeoutCount++ + if newBlock != nil { + utils.GetLogInstance().Debug("Consensus timeout, retry!", "count", timeoutCount) + // Send the new block to Consensus so it can be confirmed. + node.BlockChannel <- newBlock + } case <-readySignal: firstTry := true deadline := time.Now().Add(BlockPeriod) @@ -68,19 +77,18 @@ func (node *Node) WaitForConsensusReadyv2(readySignal chan struct{}, stopChan ch ctxerror.New("cannot commit transactions"). WithCause(err)) } - block, err := node.Worker.Commit() + newBlock, err := node.Worker.Commit() if err != nil { ctxerror.Log15(utils.GetLogger().Error, ctxerror.New("cannot commit new block"). WithCause(err)) continue - } else if err := node.proposeShardState(block); err != nil { + } else if err := node.proposeShardState(newBlock); err != nil { ctxerror.Log15(utils.GetLogger().Error, ctxerror.New("cannot add shard state"). WithCause(err)) } else { - newBlock := block - utils.GetLogInstance().Debug("Successfully proposed new block", "blockNum", block.NumberU64(), "numTxs", block.Transactions().Len()) + utils.GetLogInstance().Debug("Successfully proposed new block", "blockNum", newBlock.NumberU64(), "numTxs", newBlock.Transactions().Len()) // Send the new block to Consensus so it can be confirmed. node.BlockChannel <- newBlock From 2c8cf585dcd206afa1d1a2f227e5b1f456ed2543 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Fri, 14 Jun 2019 16:58:49 -0700 Subject: [PATCH 2/3] Add shard id check for view change --- consensus/consensus_v2.go | 17 +++++++++++++---- consensus/consensus_viewchange_msg.go | 2 ++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 91cdc53d2..64b541ba5 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -37,11 +37,20 @@ func (consensus *Consensus) handleMessageUpdate(payload []byte) { return } - if msg.GetConsensus().ShardId != consensus.ShardID { - consensus.getLogger().Warn("Received consensus message from different shard", - "myShardId", consensus.ShardID, "receivedShardId", msg.GetConsensus().ShardId) - return + if msg.Type == msg_pb.MessageType_VIEWCHANGE || msg.Type == msg_pb.MessageType_NEWVIEW { + if msg.GetViewchange().ShardId != consensus.ShardID { + consensus.getLogger().Warn("Received view change message from different shard", + "myShardId", consensus.ShardID, "receivedShardId", msg.GetConsensus().ShardId) + return + } + } else { + if msg.GetConsensus().ShardId != consensus.ShardID { + consensus.getLogger().Warn("Received consensus message from different shard", + "myShardId", consensus.ShardID, "receivedShardId", msg.GetConsensus().ShardId) + return + } } + switch msg.Type { case msg_pb.MessageType_ANNOUNCE: consensus.onAnnounce(msg) diff --git a/consensus/consensus_viewchange_msg.go b/consensus/consensus_viewchange_msg.go index e1188f801..0d0848f25 100644 --- a/consensus/consensus_viewchange_msg.go +++ b/consensus/consensus_viewchange_msg.go @@ -22,6 +22,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { vcMsg := message.GetViewchange() vcMsg.ViewId = consensus.mode.GetViewID() vcMsg.BlockNum = consensus.blockNum + vcMsg.ShardId = consensus.ShardID // sender address vcMsg.SenderPubkey = consensus.PubKey.Serialize() @@ -77,6 +78,7 @@ func (consensus *Consensus) constructNewViewMessage() []byte { vcMsg := message.GetViewchange() vcMsg.ViewId = consensus.mode.GetViewID() vcMsg.BlockNum = consensus.blockNum + vcMsg.ShardId = consensus.ShardID // sender address vcMsg.SenderPubkey = consensus.PubKey.Serialize() vcMsg.Payload = consensus.m1Payload From 6d573169b1d9101883b2cc5c6c788d18574aae94 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Fri, 14 Jun 2019 17:00:57 -0700 Subject: [PATCH 3/3] Add nil check --- consensus/consensus_v2.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 64b541ba5..ea4ce7767 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -38,13 +38,13 @@ func (consensus *Consensus) handleMessageUpdate(payload []byte) { } if msg.Type == msg_pb.MessageType_VIEWCHANGE || msg.Type == msg_pb.MessageType_NEWVIEW { - if msg.GetViewchange().ShardId != consensus.ShardID { + if msg.GetViewchange() != nil && msg.GetViewchange().ShardId != consensus.ShardID { consensus.getLogger().Warn("Received view change message from different shard", "myShardId", consensus.ShardID, "receivedShardId", msg.GetConsensus().ShardId) return } } else { - if msg.GetConsensus().ShardId != consensus.ShardID { + if msg.GetConsensus() != nil && msg.GetConsensus().ShardId != consensus.ShardID { consensus.getLogger().Warn("Received consensus message from different shard", "myShardId", consensus.ShardID, "receivedShardId", msg.GetConsensus().ShardId) return