From 92130a2ba34f733849aeac0652826f9a54935eab Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 13 Oct 2020 23:13:10 +0000 Subject: [PATCH 01/14] [viewchange] retry sending view change messages retry is needed, as the next leader may not enter view change mode immediately when other validators already sent out view change messages. Then the new leader may not collect enough signatures for the new view before timeout. This can speed up the view change process. Signed-off-by: Leo Chen --- consensus/view_change.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index ade4bbe7c..0e1b82ec6 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -157,11 +157,16 @@ func (consensus *Consensus) startViewChange(viewID uint64) { continue } msgToSend := consensus.constructViewChangeMessage(&key) - consensus.host.SendMessageToGroups([]nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), - }, + if err := consensus.msgSender.SendWithRetry( + consensus.blockNum, + msg_pb.MessageType_VIEWCHANGE, + []nodeconfig.GroupID{ + nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, p2p.ConstructMessage(msgToSend), - ) + ); err != nil { + consensus.getLogger().Err(err). + Msg("could not send out the ViewChange message") + } } } From cadec98dd96e6b626094636c50304cf812ba8ee8 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 14 Oct 2020 00:02:48 +0000 Subject: [PATCH 02/14] [viewchange] ignore viewchange/newview messages if not in viewchanging mode Signed-off-by: Leo Chen --- consensus/consensus_v2.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 10f52a16a..a5b88a993 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -43,6 +43,13 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb return nil } + // when node is not in ViewChanging mode, ignore viewchange / newview messages + if !consensus.IsViewChangingMode() && + (msg.Type == msg_pb.MessageType_VIEWCHANGE || + msg.Type == msg_pb.MessageType_NEWVIEW) { + return nil + } + intendedForValidator, intendedForLeader := !consensus.IsLeader(), consensus.IsLeader() From b6deec9a5a745562a6ec8ffbaf8bab1f83833bcb Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 14 Oct 2020 01:04:29 +0000 Subject: [PATCH 03/14] [viewchange] add RLock for m1Payload functions Signed-off-by: Leo Chen --- consensus/view_change.go | 2 +- consensus/view_change_construct.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 0e1b82ec6..8e5fa759e 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -230,8 +230,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { // received enough view change messages, change state to normal consensus if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { - consensus.getLogger().Info().Msg("[onViewChange] View Change Message Quorum Reached") consensus.current.SetMode(Normal) + consensus.getLogger().Info().Msg("[onViewChange] View Change Message Quorum Reached") consensus.LeaderPubKey = newLeaderKey consensus.ResetState() if consensus.vc.IsM1PayloadEmpty() { diff --git a/consensus/view_change_construct.go b/consensus/view_change_construct.go index 7329d4c67..234433f9c 100644 --- a/consensus/view_change_construct.go +++ b/consensus/view_change_construct.go @@ -438,6 +438,7 @@ func (vc *viewChange) InitPayload( } // isM1PayloadEmpty returns true if m1Payload is not set +// this is an unlocked internal function call func (vc *viewChange) isM1PayloadEmpty() bool { return len(vc.m1Payload) == 0 } @@ -458,6 +459,8 @@ func (vc *viewChange) GetViewIDBitmap(viewID uint64) *bls_cosi.Mask { // GetM1Payload returns the m1Payload func (vc *viewChange) GetM1Payload() []byte { + vc.vcLock.RLock() + defer vc.vcLock.RUnlock() return vc.m1Payload } From 72735d3a374a64d24a78e5e3bc1c10ef949b1871 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 14 Oct 2020 06:13:51 +0000 Subject: [PATCH 04/14] [viewchange] be consistent when construct NewView message Signed-off-by: Leo Chen --- consensus/view_change_msg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/view_change_msg.go b/consensus/view_change_msg.go index 15ecc7b5e..331d81cc8 100644 --- a/consensus/view_change_msg.go +++ b/consensus/view_change_msg.go @@ -106,7 +106,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64, priKey *bls.P Type: msg_pb.MessageType_NEWVIEW, Request: &msg_pb.Message_Viewchange{ Viewchange: &msg_pb.ViewChangeRequest{ - ViewId: consensus.GetViewChangingID(), + ViewId: viewID, BlockNum: consensus.blockNum, ShardId: consensus.ShardID, SenderPubkey: priKey.Pub.Bytes[:], From 537ec786366c6a68ee68dbf4e3c2a9d324410375 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 14 Oct 2020 05:47:00 +0000 Subject: [PATCH 05/14] [viewchange] simplify GetPreparedBlock function Signed-off-by: Leo Chen --- consensus/view_change_construct.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/consensus/view_change_construct.go b/consensus/view_change_construct.go index 234433f9c..358f091b7 100644 --- a/consensus/view_change_construct.go +++ b/consensus/view_change_construct.go @@ -113,18 +113,19 @@ func (vc *viewChange) GetPreparedBlock(fbftlog *FBFTLog, hash [32]byte) ([]byte, vc.vcLock.RLock() defer vc.vcLock.RUnlock() - if !vc.isM1PayloadEmpty() { - block := fbftlog.GetBlockByHash(hash) - if block != nil { - encodedBlock, err := rlp.EncodeToBytes(block) - if err != nil || len(encodedBlock) == 0 { - vc.getLogger().Err(err).Msg("[GetPreparedBlock] Failed encoding prepared block") - return vc.m1Payload, nil - } - return vc.m1Payload, encodedBlock + if vc.isM1PayloadEmpty() { + return nil, nil + } + + if block := fbftlog.GetBlockByHash(hash); block != nil { + encodedBlock, err := rlp.EncodeToBytes(block) + if err != nil || len(encodedBlock) == 0 { + return nil, nil } + return vc.m1Payload, encodedBlock } - return vc.m1Payload, nil + + return nil, nil } // GetM2Bitmap returns the nilBitmap as M2Bitmap From f8e2fd71201e74f1a8f4c6c8bc0f23d3e7b4b33c Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 14 Oct 2020 07:29:53 +0000 Subject: [PATCH 06/14] [viewchange] lock to process view change Signed-off-by: Leo Chen --- consensus/consensus_service.go | 64 ++++++++++++++++ consensus/view_change.go | 133 ++++++++++++++------------------- 2 files changed, 122 insertions(+), 75 deletions(-) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index 6a6da898e..ff9a07bd2 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -14,6 +14,7 @@ import ( "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/quorum" + "github.com/harmony-one/harmony/consensus/signature" bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/hash" "github.com/harmony-one/harmony/internal/chain" @@ -504,6 +505,69 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) { return } +var ( + errGetPreparedBlock = errors.New("failed to get prepared block for self commit") + errReadBitmapPayload = errors.New("failed to read signature bitmap payload") +) + +// selfCommit will create a commit message and commit it locally +// it is used by the new leadder of the view change routine +// when view change is succeeded and the new leader +// received prepared payload from other validators or from local +func (consensus *Consensus) selfCommit(payload []byte) error { + var blockHash [32]byte + copy(blockHash[:], payload[:32]) + + // Leader sign and add commit message + block := consensus.FBFTLog.GetBlockByHash(blockHash) + if block == nil { + return errGetPreparedBlock + } + + aggSig, mask, err := consensus.ReadSignatureBitmapPayload(payload, 32) + if err != nil { + return errReadBitmapPayload + } + + // update consensus structure when succeeded + // protect consensus data update logic + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + + copy(consensus.blockHash[:], blockHash[:]) + consensus.switchPhase("selfCommit", FBFTCommit) + consensus.aggregatedPrepareSig = aggSig + consensus.prepareBitmap = mask + commitPayload := signature.ConstructCommitPayload(consensus.ChainReader, + block.Epoch(), block.Hash(), block.NumberU64(), block.Header().ViewID().Uint64()) + for i, key := range consensus.priKey { + if err := consensus.commitBitmap.SetKey(key.Pub.Bytes, true); err != nil { + consensus.getLogger().Error(). + Err(err). + Int("Index", i). + Str("Key", key.Pub.Bytes.Hex()). + Msg("[selfCommit] New Leader commit bitmap set failed") + continue + } + + if _, err := consensus.Decider.SubmitVote( + quorum.Commit, + key.Pub.Bytes, + key.Pri.SignHash(commitPayload), + common.BytesToHash(consensus.blockHash[:]), + block.NumberU64(), + block.Header().ViewID().Uint64(), + ); err != nil { + consensus.getLogger().Warn(). + Err(err). + Int("Index", i). + Str("Key", key.Pub.Bytes.Hex()). + Msg("[selfCommit] submit vote on viewchange commit failed") + } + } + return nil +} + // getLogger returns logger for consensus contexts added func (consensus *Consensus) getLogger() *zerolog.Logger { logger := utils.Logger().With(). diff --git a/consensus/view_change.go b/consensus/view_change.go index 8e5fa759e..9553e31a4 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -13,6 +13,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" + "github.com/pkg/errors" ) // MaxViewIDDiff limits the received view ID to only 249 further from the current view ID @@ -170,6 +171,39 @@ func (consensus *Consensus) startViewChange(viewID uint64) { } } +// stopViewChange stops the current view change +func (consensus *Consensus) stopViewChange(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper) error { + msgToSend := consensus.constructNewViewMessage( + viewID, newLeaderPriKey, + ) + if err := consensus.msgSender.SendWithRetry( + consensus.blockNum, + msg_pb.MessageType_NEWVIEW, + []nodeconfig.GroupID{ + nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, + p2p.ConstructMessage(msgToSend), + ); err != nil { + return errors.New("failed to send out the NewView message") + } + consensus.getLogger().Info(). + Str("myKey", newLeaderPriKey.Pub.Bytes.Hex()). + Hex("M1Payload", consensus.vc.GetM1Payload()). + Msg("[stopViewChange] Sent NewView Messge") + + consensus.current.SetMode(Normal) + consensus.consensusTimeout[timeoutViewChange].Stop() + consensus.SetViewIDs(viewID) + consensus.ResetViewChangeState() + consensus.consensusTimeout[timeoutConsensus].Start() + + consensus.getLogger().Info(). + Uint64("viewID", viewID). + Str("myKey", newLeaderPriKey.Pub.Bytes.Hex()). + Msg("[stopViewChange] viewChange stopped. I am the New Leader") + + return nil +} + // onViewChange is called when the view change message is received. func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { consensus.getLogger().Info().Msg("[onViewChange] Received ViewChange Message") @@ -229,88 +263,34 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { } // received enough view change messages, change state to normal consensus - if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) { - consensus.current.SetMode(Normal) - consensus.getLogger().Info().Msg("[onViewChange] View Change Message Quorum Reached") - consensus.LeaderPubKey = newLeaderKey - consensus.ResetState() + if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.IsViewChangingMode() { + // no previous prepared message, go straight to normal mode + // and start proposing new block if consensus.vc.IsM1PayloadEmpty() { - // TODO(Chao): explain why ReadySignal is sent only in this case but not the other case. - // Make sure the newly proposed block have the correct view ID - consensus.SetCurBlockViewID(recvMsg.ViewID) - go func() { - consensus.ReadySignal <- struct{}{} - }() - } else { - consensus.switchPhase("onViewChange", FBFTCommit) - payload := consensus.vc.GetM1Payload() - copy(consensus.blockHash[:], payload[:32]) - aggSig, mask, err := consensus.ReadSignatureBitmapPayload(payload, 32) - - if err != nil { - consensus.getLogger().Error().Err(err). - Msg("[onViewChange] ReadSignatureBitmapPayload Fail") - return - } - - consensus.aggregatedPrepareSig = aggSig - consensus.prepareBitmap = mask - // Leader sign and add commit message - block := consensus.FBFTLog.GetBlockByHash(consensus.blockHash) - if block == nil { - consensus.getLogger().Warn().Msg("[onViewChange] failed to get prepared block for self commit") + if err := consensus.stopViewChange(recvMsg.ViewID, newLeaderPriKey); err != nil { + consensus.getLogger().Error().Err(err).Msg("[onViewChange] stopViewChange failed") return } - commitPayload := signature.ConstructCommitPayload(consensus.ChainReader, - block.Epoch(), block.Hash(), block.NumberU64(), block.Header().ViewID().Uint64()) - for i, key := range consensus.priKey { - if err := consensus.commitBitmap.SetKey(key.Pub.Bytes, true); err != nil { - consensus.getLogger().Warn().Err(err). - Msgf("[OnViewChange] New Leader commit bitmap set failed for key at index %d", i) - continue - } - - if _, err := consensus.Decider.SubmitVote( - quorum.Commit, - key.Pub.Bytes, - key.Pri.SignHash(commitPayload), - common.BytesToHash(consensus.blockHash[:]), - block.NumberU64(), - block.Header().ViewID().Uint64(), - ); err != nil { - consensus.getLogger().Warn().Err(err).Msg("submit vote on viewchange commit failed") - return - } + consensus.ResetState() + consensus.LeaderPubKey = newLeaderKey - } + go func() { + consensus.ReadySignal <- struct{}{} + }() + return } - consensus.SetViewIDs(recvMsg.ViewID) - msgToSend := consensus.constructNewViewMessage( - recvMsg.ViewID, newLeaderPriKey, - ) - - if err := consensus.msgSender.SendWithRetry( - consensus.blockNum, - msg_pb.MessageType_NEWVIEW, - []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, - p2p.ConstructMessage(msgToSend), - ); err != nil { - consensus.getLogger().Err(err). - Msg("could not send out the NEWVIEW message") + payload := consensus.vc.GetM1Payload() + if err := consensus.selfCommit(payload); err != nil { + consensus.getLogger().Error().Err(err).Msg("[onViewChange] self commit failed") + return } - consensus.getLogger().Info(). - Str("myKey", newLeaderKey.Bytes.Hex()). - Hex("M1Payload", consensus.vc.GetM1Payload()). - Msg("[onViewChange] Sent NewView Messge") - - consensus.ResetViewChangeState() - consensus.consensusTimeout[timeoutViewChange].Stop() - consensus.consensusTimeout[timeoutConsensus].Start() - consensus.getLogger().Info(). - Str("myKey", newLeaderKey.Bytes.Hex()). - Msg("[onViewChange] I am the New Leader") + if err := consensus.stopViewChange(recvMsg.ViewID, newLeaderPriKey); err != nil { + consensus.getLogger().Error().Err(err).Msg("[onViewChange] stopViewChange failed") + return + } + consensus.ResetState() + consensus.LeaderPubKey = newLeaderKey } } @@ -372,9 +352,12 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { Msg("[onNewView] Failed to Verify Signature for M1 (prepare) message") return } + consensus.mutex.Lock() copy(consensus.blockHash[:], blockHash) consensus.aggregatedPrepareSig = aggSig consensus.prepareBitmap = mask + consensus.mutex.Unlock() + // create prepared message from newview preparedMsg := FBFTMessage{ MessageType: msg_pb.MessageType_PREPARED, From 37f6a38b99e329b648636c574a10b843792ad1e6 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 14 Oct 2020 08:03:57 +0000 Subject: [PATCH 07/14] [consensus] add sendCommitMessages function Signed-off-by: Leo Chen --- consensus/validator.go | 68 ++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/consensus/validator.go b/consensus/validator.go index d57bd765f..f502a871f 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -89,6 +89,41 @@ func (consensus *Consensus) prepare() { consensus.switchPhase("Announce", FBFTPrepare) } +// sendCommitMessages send out commit messages to leader +func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { + // Sign commit signature on the received block + commitPayload := signature.ConstructCommitPayload(consensus.ChainReader, + blockObj.Epoch(), blockObj.Hash(), blockObj.NumberU64(), blockObj.Header().ViewID().Uint64()) + groupID := []nodeconfig.GroupID{ + nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), + } + for _, key := range consensus.priKey { + if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { + continue + } + + networkMessage, _ := consensus.construct( + msg_pb.MessageType_COMMIT, + commitPayload, + &key, + ) + + if consensus.current.Mode() != Listening { + if err := consensus.msgSender.SendWithoutRetry( + groupID, + p2p.ConstructMessage(networkMessage.Bytes), + ); err != nil { + consensus.getLogger().Warn().Msg("[sendCommitMessages] Cannot send commit message!!") + } else { + consensus.getLogger().Info(). + Uint64("blockNum", consensus.blockNum). + Hex("blockHash", consensus.blockHash[:]). + Msg("[sendCommitMessages] Sent Commit Message!!") + } + } + } +} + // if onPrepared accepts the prepared message from the leader, then // it will send a COMMIT message for the leader to receive on the network. func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { @@ -202,38 +237,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { if bytes.Equal(consensus.blockHash[:], emptyHash[:]) { copy(consensus.blockHash[:], blockHash[:]) } - - // Sign commit signature on the received block - commitPayload := signature.ConstructCommitPayload(consensus.ChainReader, - blockObj.Epoch(), blockObj.Hash(), blockObj.NumberU64(), blockObj.Header().ViewID().Uint64()) - groupID := []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), - } - for _, key := range consensus.priKey { - if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { - continue - } - - networkMessage, _ := consensus.construct( - msg_pb.MessageType_COMMIT, - commitPayload, - &key, - ) - - if consensus.current.Mode() != Listening { - if err := consensus.msgSender.SendWithoutRetry( - groupID, - p2p.ConstructMessage(networkMessage.Bytes), - ); err != nil { - consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!") - } else { - consensus.getLogger().Info(). - Uint64("blockNum", consensus.blockNum). - Hex("blockHash", consensus.blockHash[:]). - Msg("[OnPrepared] Sent Commit Message!!") - } - } - } + consensus.sendCommitMessages(&blockObj) consensus.switchPhase("onPrepared", FBFTCommit) } From 3784d6a56404a81831db043dacd47df07a69355d Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 14 Oct 2020 08:06:49 +0000 Subject: [PATCH 08/14] [viewchange] simplify onNewView function Signed-off-by: Leo Chen --- consensus/view_change.go | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 9553e31a4..d2d8ffc50 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -9,7 +9,6 @@ import ( "github.com/ethereum/go-ethereum/common" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/consensus/signature" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" @@ -381,6 +380,8 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { return } + consensus.consensusTimeout[timeoutViewChange].Stop() + // newView message verified success, override my state consensus.SetViewIDs(recvMsg.ViewID) consensus.LeaderPubKey = recvMsg.SenderPubkey @@ -388,31 +389,7 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { // NewView message is verified, change state to normal consensus if preparedBlock != nil { - // Construct and send the commit message - commitPayload := signature.ConstructCommitPayload(consensus.ChainReader, - preparedBlock.Epoch(), preparedBlock.Hash(), preparedBlock.NumberU64(), preparedBlock.Header().ViewID().Uint64()) - groupID := []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} - for _, key := range consensus.priKey { - if !consensus.IsValidatorInCommittee(key.Pub.Bytes) { - continue - } - network, err := consensus.construct( - msg_pb.MessageType_COMMIT, - commitPayload, - &key, - ) - if err != nil { - consensus.getLogger().Err(err).Msg("could not create commit message") - continue - } - msgToSend := network.Bytes - consensus.getLogger().Info().Msg("onNewView === commit") - consensus.host.SendMessageToGroups( - groupID, - p2p.ConstructMessage(msgToSend), - ) - } + consensus.sendCommitMessages(preparedBlock) consensus.switchPhase("onNewView", FBFTCommit) } else { consensus.ResetState() @@ -422,7 +399,6 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { Str("newLeaderKey", consensus.LeaderPubKey.Bytes.Hex()). Msg("new leader changed") consensus.consensusTimeout[timeoutConsensus].Start() - consensus.consensusTimeout[timeoutViewChange].Stop() } // ResetViewChangeState resets the view change structure From bdd717edfbbbe68f62d641f6ed2289038d821479 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 14 Oct 2020 08:10:19 +0000 Subject: [PATCH 09/14] [viewchange] protect consensus structure using mutex Signed-off-by: Leo Chen --- consensus/view_change.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/consensus/view_change.go b/consensus/view_change.go index d2d8ffc50..dbc6c0662 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -172,6 +172,9 @@ func (consensus *Consensus) startViewChange(viewID uint64) { // stopViewChange stops the current view change func (consensus *Consensus) stopViewChange(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper) error { + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + msgToSend := consensus.constructNewViewMessage( viewID, newLeaderPriKey, ) @@ -380,6 +383,9 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { return } + consensus.mutex.Lock() + defer consensus.mutex.Unlock() + consensus.consensusTimeout[timeoutViewChange].Stop() // newView message verified success, override my state From 793529ee245d825c40a6380d81081ae561fd791d Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Thu, 15 Oct 2020 06:45:42 +0000 Subject: [PATCH 10/14] [viewchange] rename stopViewChange to startNewView Signed-off-by: Leo Chen --- consensus/view_change.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index dbc6c0662..8fe1e0092 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -170,8 +170,8 @@ func (consensus *Consensus) startViewChange(viewID uint64) { } } -// stopViewChange stops the current view change -func (consensus *Consensus) stopViewChange(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper) error { +// startNewView stops the current view change +func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper) error { consensus.mutex.Lock() defer consensus.mutex.Unlock() @@ -190,7 +190,7 @@ func (consensus *Consensus) stopViewChange(viewID uint64, newLeaderPriKey *bls.P consensus.getLogger().Info(). Str("myKey", newLeaderPriKey.Pub.Bytes.Hex()). Hex("M1Payload", consensus.vc.GetM1Payload()). - Msg("[stopViewChange] Sent NewView Messge") + Msg("[startNewView] Sent NewView Messge") consensus.current.SetMode(Normal) consensus.consensusTimeout[timeoutViewChange].Stop() @@ -201,7 +201,7 @@ func (consensus *Consensus) stopViewChange(viewID uint64, newLeaderPriKey *bls.P consensus.getLogger().Info(). Uint64("viewID", viewID). Str("myKey", newLeaderPriKey.Pub.Bytes.Hex()). - Msg("[stopViewChange] viewChange stopped. I am the New Leader") + Msg("[startNewView] viewChange stopped. I am the New Leader") return nil } @@ -269,8 +269,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { // no previous prepared message, go straight to normal mode // and start proposing new block if consensus.vc.IsM1PayloadEmpty() { - if err := consensus.stopViewChange(recvMsg.ViewID, newLeaderPriKey); err != nil { - consensus.getLogger().Error().Err(err).Msg("[onViewChange] stopViewChange failed") + if err := consensus.startNewView(recvMsg.ViewID, newLeaderPriKey); err != nil { + consensus.getLogger().Error().Err(err).Msg("[onViewChange] startNewView failed") return } consensus.ResetState() @@ -287,8 +287,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { consensus.getLogger().Error().Err(err).Msg("[onViewChange] self commit failed") return } - if err := consensus.stopViewChange(recvMsg.ViewID, newLeaderPriKey); err != nil { - consensus.getLogger().Error().Err(err).Msg("[onViewChange] stopViewChange failed") + if err := consensus.startNewView(recvMsg.ViewID, newLeaderPriKey); err != nil { + consensus.getLogger().Error().Err(err).Msg("[onViewChange] startNewView failed") return } consensus.ResetState() From 67a167eb3a6f2af08cc78e40981b30374ab5e471 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Thu, 15 Oct 2020 06:49:49 +0000 Subject: [PATCH 11/14] [viewchange] filter out viewchange message in p2p validation filter out viewchange/newview message when the validator is not in viewchanging mode. Signed-off-by: Leo Chen --- consensus/consensus_v2.go | 7 ------- node/node.go | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index a5b88a993..10f52a16a 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -43,13 +43,6 @@ func (consensus *Consensus) HandleMessageUpdate(ctx context.Context, msg *msg_pb return nil } - // when node is not in ViewChanging mode, ignore viewchange / newview messages - if !consensus.IsViewChangingMode() && - (msg.Type == msg_pb.MessageType_VIEWCHANGE || - msg.Type == msg_pb.MessageType_NEWVIEW) { - return nil - } - intendedForValidator, intendedForLeader := !consensus.IsLeader(), consensus.IsLeader() diff --git a/node/node.go b/node/node.go index 1a1abe72c..bbf42e3fb 100644 --- a/node/node.go +++ b/node/node.go @@ -461,9 +461,9 @@ func (node *Node) validateShardBoundMessage( return nil, nil, true, nil } } else { - // ignore newview message if the node is not in viewchanging mode + // ignore viewchange/newview message if the node is not in viewchanging mode switch m.Type { - case msg_pb.MessageType_NEWVIEW: + case msg_pb.MessageType_NEWVIEW, msg_pb.MessageType_VIEWCHANGE: return nil, nil, true, nil } } From fd84f847e0a1221fdb77e4a32eb72f1efa47b0a6 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Thu, 15 Oct 2020 06:52:44 +0000 Subject: [PATCH 12/14] [viewchange] handle conseneus message construct errors Signed-off-by: Leo Chen --- consensus/validator.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/consensus/validator.go b/consensus/validator.go index f502a871f..c39297265 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -102,11 +102,15 @@ func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) { continue } - networkMessage, _ := consensus.construct( + networkMessage, err := consensus.construct( msg_pb.MessageType_COMMIT, commitPayload, &key, ) + if err != nil { + consensus.getLogger().Warn().Msg("[sendCommitMessages] cannot construct network message") + continue + } if consensus.current.Mode() != Listening { if err := consensus.msgSender.SendWithoutRetry( From 95df383defbad60a712f01ea49612311e14a5db9 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 15 Oct 2020 22:15:00 -0700 Subject: [PATCH 13/14] do not create new account for cx txn (#3391) --- core/state/statedb.go | 8 +++++--- core/vm/evm.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index f3eb6e47e..296c5e929 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -739,9 +739,11 @@ func (db *DB) Prepare(thash, bhash common.Hash, ti int) { } func (db *DB) clearJournalAndRefund() { - db.journal = newJournal() - db.validRevisions = db.validRevisions[:0] - db.refund = 0 + if len(db.journal.entries) > 0 { + db.journal = newJournal() + db.refund = 0 + } + db.validRevisions = db.validRevisions[:0] // Snapshots can be created without journal entires } // Commit writes the state to the underlying in-memory trie database. diff --git a/core/vm/evm.go b/core/vm/evm.go index d94ded8c8..fef4e4d74 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -217,7 +217,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas to = AccountRef(addr) snapshot = evm.StateDB.Snapshot() ) - if !evm.StateDB.Exist(addr) { + if !evm.StateDB.Exist(addr) && txType != types.SubtractionOnly { precompiles := PrecompiledContractsHomestead if evm.ChainConfig().IsS3(evm.EpochNumber) { precompiles = PrecompiledContractsByzantium From 1fbe2bf54eaf216c32d8a49e514626a0605711ab Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Fri, 16 Oct 2020 05:47:51 +0000 Subject: [PATCH 14/14] [viewchange] protect additional consensus state change using mutext Signed-off-by: Leo Chen --- consensus/view_change.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/consensus/view_change.go b/consensus/view_change.go index 8fe1e0092..4a00e306a 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -203,6 +203,9 @@ func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.Pri Str("myKey", newLeaderPriKey.Pub.Bytes.Hex()). Msg("[startNewView] viewChange stopped. I am the New Leader") + consensus.ResetState() + consensus.LeaderPubKey = newLeaderPriKey.Pub + return nil } @@ -250,7 +253,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { recvMsg.ViewID, recvMsg.BlockNum, consensus.priKey); err != nil { - consensus.getLogger().Error().Err(err).Msg("Init Payload Error") + consensus.getLogger().Error().Err(err).Msg("[onViewChange] Init Payload Error") return } @@ -260,7 +263,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { Uint64("viewID", recvMsg.ViewID). Uint64("blockNum", recvMsg.BlockNum). Str("msgSender", recvMsg.SenderPubkey.Bytes.Hex()). - Msg("Verify View Change Message Error") + Msg("[onViewChange] process View Change message error") return } @@ -273,8 +276,6 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { consensus.getLogger().Error().Err(err).Msg("[onViewChange] startNewView failed") return } - consensus.ResetState() - consensus.LeaderPubKey = newLeaderKey go func() { consensus.ReadySignal <- struct{}{} @@ -291,8 +292,6 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { consensus.getLogger().Error().Err(err).Msg("[onViewChange] startNewView failed") return } - consensus.ResetState() - consensus.LeaderPubKey = newLeaderKey } }