merge with main

consensus/consensus_service.go

@@ -16,6 +16,7 @@ import (
 	"github.com/harmony-one/harmony/block"
 	consensus_engine "github.com/harmony-one/harmony/consensus/engine"
 	"github.com/harmony-one/harmony/consensus/quorum"
+	"github.com/harmony-one/harmony/consensus/signature"
 	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/crypto/hash"
 	"github.com/harmony-one/harmony/internal/chain"
@@ -522,6 +523,69 @@ func (consensus *Consensus) switchPhase(subject string, desired FBFTPhase) {
 	return
 }
 
+var (
+	errGetPreparedBlock  = errors.New("failed to get prepared block for self commit")
+	errReadBitmapPayload = errors.New("failed to read signature bitmap payload")
+)
+
+// selfCommit will create a commit message and commit it locally
+// it is used by the new leadder of the view change routine
+// when view change is succeeded and the new leader
+// received prepared payload from other validators or from local
+func (consensus *Consensus) selfCommit(payload []byte) error {
+	var blockHash [32]byte
+	copy(blockHash[:], payload[:32])
+
+	// Leader sign and add commit message
+	block := consensus.FBFTLog.GetBlockByHash(blockHash)
+	if block == nil {
+		return errGetPreparedBlock
+	}
+
+	aggSig, mask, err := consensus.ReadSignatureBitmapPayload(payload, 32)
+	if err != nil {
+		return errReadBitmapPayload
+	}
+
+	// update consensus structure when succeeded
+	// protect consensus data update logic
+	consensus.mutex.Lock()
+	defer consensus.mutex.Unlock()
+
+	copy(consensus.blockHash[:], blockHash[:])
+	consensus.switchPhase("selfCommit", FBFTCommit)
+	consensus.aggregatedPrepareSig = aggSig
+	consensus.prepareBitmap = mask
+	commitPayload := signature.ConstructCommitPayload(consensus.ChainReader,
+		block.Epoch(), block.Hash(), block.NumberU64(), block.Header().ViewID().Uint64())
+	for i, key := range consensus.priKey {
+		if err := consensus.commitBitmap.SetKey(key.Pub.Bytes, true); err != nil {
+			consensus.getLogger().Error().
+				Err(err).
+				Int("Index", i).
+				Str("Key", key.Pub.Bytes.Hex()).
+				Msg("[selfCommit] New Leader commit bitmap set failed")
+			continue
+		}
+
+		if _, err := consensus.Decider.SubmitVote(
+			quorum.Commit,
+			[]bls.SerializedPublicKey{key.Pub.Bytes},
+			key.Pri.SignHash(commitPayload),
+			common.BytesToHash(consensus.blockHash[:]),
+			block.NumberU64(),
+			block.Header().ViewID().Uint64(),
+		); err != nil {
+			consensus.getLogger().Warn().
+				Err(err).
+				Int("Index", i).
+				Str("Key", key.Pub.Bytes.Hex()).
+				Msg("[selfCommit] submit vote on viewchange commit failed")
+		}
+	}
+	return nil
+}
+
 // getLogger returns logger for consensus contexts added
 func (consensus *Consensus) getLogger() *zerolog.Logger {
 	logger := utils.Logger().With().

consensus/validator.go

@@ -76,6 +76,26 @@ func (consensus *Consensus) prepare() {
 	consensus.switchPhase("Announce", FBFTPrepare)
 }
 
+// sendCommitMessages send out commit messages to leader
+func (consensus *Consensus) sendCommitMessages(blockObj *types.Block) {
+	priKeys := consensus.getPriKeysInCommittee()
+
+	// Sign commit signature on the received block and construct the p2p messages
+	commitPayload := signature.ConstructCommitPayload(consensus.ChainReader,
+		blockObj.Epoch(), blockObj.Hash(), blockObj.NumberU64(), blockObj.Header().ViewID().Uint64())
+
+	p2pMsgs := consensus.constructP2pMessages(msg_pb.MessageType_COMMIT, commitPayload, priKeys)
+
+	if err := consensus.broadcastConsensusP2pMessages(p2pMsgs); err != nil {
+		consensus.getLogger().Warn().Err(err).Msg("[sendCommitMessages] Cannot send commit message!!")
+	} else {
+		consensus.getLogger().Info().
+			Uint64("blockNum", consensus.blockNum).
+			Hex("blockHash", consensus.blockHash[:]).
+			Msg("[sendCommitMessages] Sent Commit Message!!")
+	}
+}
+
 // if onPrepared accepts the prepared message from the leader, then
 // it will send a COMMIT message for the leader to receive on the network.
 func (consensus *Consensus) onPrepared(msg *msg_pb.Message) {
@@ -189,23 +209,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) {
 		copy(consensus.blockHash[:], blockHash[:])
 	}
 
-	priKeys := consensus.getPriKeysInCommittee()
+	consensus.sendCommitMessages(&blockObj)
-
-	// Sign commit signature on the received block and construct the p2p messages
-	commitPayload := signature.ConstructCommitPayload(consensus.ChainReader,
-		blockObj.Epoch(), blockObj.Hash(), blockObj.NumberU64(), blockObj.Header().ViewID().Uint64())
-
-	p2pMsgs := consensus.constructP2pMessages(msg_pb.MessageType_COMMIT, commitPayload, priKeys)
-
-	if err := consensus.broadcastConsensusP2pMessages(p2pMsgs); err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[OnPrepared] Cannot send commit message!!")
-	} else {
-		consensus.getLogger().Info().
-			Uint64("blockNum", consensus.blockNum).
-			Hex("blockHash", consensus.blockHash[:]).
-			Msg("[OnPrepared] Sent Commit Message!!")
-	}
-
 	consensus.switchPhase("onPrepared", FBFTCommit)
 }
 

consensus/view_change.go

@@ -9,10 +9,10 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/consensus/quorum"
-	"github.com/harmony-one/harmony/consensus/signature"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/p2p"
+	"github.com/pkg/errors"
 )
 
 // MaxViewIDDiff limits the received view ID to only 249 further from the current view ID
@@ -157,12 +157,56 @@ func (consensus *Consensus) startViewChange(viewID uint64) {
 			continue
 		}
 		msgToSend := consensus.constructViewChangeMessage(&key)
-		consensus.host.SendMessageToGroups([]nodeconfig.GroupID{
+		if err := consensus.msgSender.SendWithRetry(
-			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)),
+			consensus.blockNum,
-		},
+			msg_pb.MessageType_VIEWCHANGE,
+			[]nodeconfig.GroupID{
+				nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))},
 			p2p.ConstructMessage(msgToSend),
+		); err != nil {
+			consensus.getLogger().Err(err).
+				Msg("could not send out the ViewChange message")
+		}
+	}
+}
+
+// startNewView stops the current view change
+func (consensus *Consensus) startNewView(viewID uint64, newLeaderPriKey *bls.PrivateKeyWrapper) error {
+	consensus.mutex.Lock()
+	defer consensus.mutex.Unlock()
+
+	msgToSend := consensus.constructNewViewMessage(
+		viewID, newLeaderPriKey,
 	)
+	if err := consensus.msgSender.SendWithRetry(
+		consensus.blockNum,
+		msg_pb.MessageType_NEWVIEW,
+		[]nodeconfig.GroupID{
+			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))},
+		p2p.ConstructMessage(msgToSend),
+	); err != nil {
+		return errors.New("failed to send out the NewView message")
 	}
+	consensus.getLogger().Info().
+		Str("myKey", newLeaderPriKey.Pub.Bytes.Hex()).
+		Hex("M1Payload", consensus.vc.GetM1Payload()).
+		Msg("[startNewView] Sent NewView Messge")
+
+	consensus.current.SetMode(Normal)
+	consensus.consensusTimeout[timeoutViewChange].Stop()
+	consensus.SetViewIDs(viewID)
+	consensus.ResetViewChangeState()
+	consensus.consensusTimeout[timeoutConsensus].Start()
+
+	consensus.getLogger().Info().
+		Uint64("viewID", viewID).
+		Str("myKey", newLeaderPriKey.Pub.Bytes.Hex()).
+		Msg("[startNewView] viewChange stopped. I am the New Leader")
+
+	consensus.ResetState()
+	consensus.LeaderPubKey = newLeaderPriKey.Pub
+
+	return nil
 }
 
 // onViewChange is called when the view change message is received.
@@ -212,7 +256,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 		recvMsg.ViewID,
 		recvMsg.BlockNum,
 		consensus.priKey); err != nil {
-		consensus.getLogger().Error().Err(err).Msg("Init Payload Error")
+		consensus.getLogger().Error().Err(err).Msg("[onViewChange] Init Payload Error")
 		return
 	}
 
@@ -222,93 +266,35 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 			Uint64("viewID", recvMsg.ViewID).
 			Uint64("blockNum", recvMsg.BlockNum).
 			Str("msgSender", senderKey.Bytes.Hex()).
-			Msg("Verify View Change Message Error")
+			Msg("[onViewChange] process View Change message error")
 		return
 	}
 
 	// received enough view change messages, change state to normal consensus
-	if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) {
+	if consensus.Decider.IsQuorumAchievedByMask(consensus.vc.GetViewIDBitmap(recvMsg.ViewID)) && consensus.IsViewChangingMode() {
-		consensus.getLogger().Info().Msg("[onViewChange] View Change Message Quorum Reached")
+		// no previous prepared message, go straight to normal mode
-		consensus.current.SetMode(Normal)
+		// and start proposing new block
-		consensus.LeaderPubKey = newLeaderKey
-		consensus.ResetState()
 		if consensus.vc.IsM1PayloadEmpty() {
-			// TODO(Chao): explain why ReadySignal is sent only in this case but not the other case.
+			if err := consensus.startNewView(recvMsg.ViewID, newLeaderPriKey); err != nil {
-			// Make sure the newly proposed block have the correct view ID
+				consensus.getLogger().Error().Err(err).Msg("[onViewChange] startNewView failed")
-			consensus.SetCurBlockViewID(recvMsg.ViewID)
+				return
+			}
+
 			go func() {
 				consensus.ReadySignal <- struct{}{}
 			}()
-		} else {
-			consensus.switchPhase("onViewChange", FBFTCommit)
-			payload := consensus.vc.GetM1Payload()
-			copy(consensus.blockHash[:], payload[:32])
-			aggSig, mask, err := consensus.ReadSignatureBitmapPayload(payload, 32)
-
-			if err != nil {
-				consensus.getLogger().Error().Err(err).
-					Msg("[onViewChange] ReadSignatureBitmapPayload Fail")
 			return
 		}
 
-			consensus.aggregatedPrepareSig = aggSig
+		payload := consensus.vc.GetM1Payload()
-			consensus.prepareBitmap = mask
+		if err := consensus.selfCommit(payload); err != nil {
-			// Leader sign and add commit message
+			consensus.getLogger().Error().Err(err).Msg("[onViewChange] self commit failed")
-			block := consensus.FBFTLog.GetBlockByHash(consensus.blockHash)
-			if block == nil {
-				consensus.getLogger().Warn().Msg("[onViewChange] failed to get prepared block for self commit")
 			return
 		}
-			commitPayload := signature.ConstructCommitPayload(consensus.ChainReader,
+		if err := consensus.startNewView(recvMsg.ViewID, newLeaderPriKey); err != nil {
-				block.Epoch(), block.Hash(), block.NumberU64(), block.Header().ViewID().Uint64())
+			consensus.getLogger().Error().Err(err).Msg("[onViewChange] startNewView failed")
-			for i, key := range consensus.priKey {
-				if err := consensus.commitBitmap.SetKey(key.Pub.Bytes, true); err != nil {
-					consensus.getLogger().Warn().Err(err).
-						Msgf("[OnViewChange] New Leader commit bitmap set failed for key at index %d", i)
-					continue
-				}
-
-				if _, err := consensus.Decider.SubmitVote(
-					quorum.Commit,
-					[]bls.SerializedPublicKey{key.Pub.Bytes},
-					key.Pri.SignHash(commitPayload),
-					common.BytesToHash(consensus.blockHash[:]),
-					block.NumberU64(),
-					block.Header().ViewID().Uint64(),
-				); err != nil {
-					consensus.getLogger().Warn().Err(err).Msg("submit vote on viewchange commit failed")
 			return
 		}
-
-			}
-		}
-
-		consensus.SetViewIDs(recvMsg.ViewID)
-		msgToSend := consensus.constructNewViewMessage(
-			recvMsg.ViewID, newLeaderPriKey,
-		)
-
-		if err := consensus.msgSender.SendWithRetry(
-			consensus.blockNum,
-			msg_pb.MessageType_NEWVIEW,
-			[]nodeconfig.GroupID{
-				nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))},
-			p2p.ConstructMessage(msgToSend),
-		); err != nil {
-			consensus.getLogger().Err(err).
-				Msg("could not send out the NEWVIEW message")
-		}
-		consensus.getLogger().Info().
-			Str("myKey", newLeaderKey.Bytes.Hex()).
-			Hex("M1Payload", consensus.vc.GetM1Payload()).
-			Msg("[onViewChange] Sent NewView Messge")
-
-		consensus.ResetViewChangeState()
-		consensus.consensusTimeout[timeoutViewChange].Stop()
-		consensus.consensusTimeout[timeoutConsensus].Start()
-		consensus.getLogger().Info().
-			Str("myKey", newLeaderKey.Bytes.Hex()).
-			Msg("[onViewChange] I am the New Leader")
 	}
 }
 
@@ -376,9 +362,12 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) {
 				Msg("[onNewView] Failed to Verify Signature for M1 (prepare) message")
 			return
 		}
+		consensus.mutex.Lock()
 		copy(consensus.blockHash[:], blockHash)
 		consensus.aggregatedPrepareSig = aggSig
 		consensus.prepareBitmap = mask
+		consensus.mutex.Unlock()
+
 		// create prepared message from newview
 		preparedMsg := FBFTMessage{
 			MessageType: msg_pb.MessageType_PREPARED,
@@ -403,6 +392,11 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) {
 		return
 	}
 
+	consensus.mutex.Lock()
+	defer consensus.mutex.Unlock()
+
+	consensus.consensusTimeout[timeoutViewChange].Stop()
+
 	// newView message verified success, override my state
 	consensus.SetViewIDs(recvMsg.ViewID)
 	consensus.LeaderPubKey = senderKey
@@ -410,32 +404,7 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) {
 
 	// NewView message is verified, change state to normal consensus
 	if preparedBlock != nil {
-		// Construct and send the commit message
+		consensus.sendCommitMessages(preparedBlock)
-		commitPayload := signature.ConstructCommitPayload(consensus.ChainReader,
-			preparedBlock.Epoch(), preparedBlock.Hash(), preparedBlock.NumberU64(), preparedBlock.Header().ViewID().Uint64())
-		groupID := []nodeconfig.GroupID{
-			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}
-
-		for _, key := range consensus.priKey {
-			if !consensus.IsValidatorInCommittee(key.Pub.Bytes) {
-				continue
-			}
-			p2pMsg, err := consensus.construct(
-				msg_pb.MessageType_COMMIT,
-				commitPayload,
-				[]*bls.PrivateKeyWrapper{&key},
-			)
-			if err != nil {
-				consensus.getLogger().Err(err).Msg("could not create commit message")
-				continue
-			}
-
-			consensus.getLogger().Info().Msg("onNewView === commit")
-			consensus.host.SendMessageToGroups(
-				groupID,
-				p2p.ConstructMessage(p2pMsg.Bytes),
-			)
-		}
 		consensus.switchPhase("onNewView", FBFTCommit)
 	} else {
 		consensus.ResetState()
@@ -445,7 +414,6 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) {
 		Str("newLeaderKey", consensus.LeaderPubKey.Bytes.Hex()).
 		Msg("new leader changed")
 	consensus.consensusTimeout[timeoutConsensus].Start()
-	consensus.consensusTimeout[timeoutViewChange].Stop()
 }
 
 // ResetViewChangeState resets the view change structure

consensus/view_change_construct.go

@@ -113,18 +113,19 @@ func (vc *viewChange) GetPreparedBlock(fbftlog *FBFTLog, hash [32]byte) ([]byte,
 	vc.vcLock.RLock()
 	defer vc.vcLock.RUnlock()
 
-	if !vc.isM1PayloadEmpty() {
+	if vc.isM1PayloadEmpty() {
-		block := fbftlog.GetBlockByHash(hash)
+		return nil, nil
-		if block != nil {
+	}
+
+	if block := fbftlog.GetBlockByHash(hash); block != nil {
 		encodedBlock, err := rlp.EncodeToBytes(block)
 		if err != nil || len(encodedBlock) == 0 {
-				vc.getLogger().Err(err).Msg("[GetPreparedBlock] Failed encoding prepared block")
+			return nil, nil
-				return vc.m1Payload, nil
 		}
 		return vc.m1Payload, encodedBlock
 	}
-	}
+
-	return vc.m1Payload, nil
+	return nil, nil
 }
 
 // GetM2Bitmap returns the nilBitmap as M2Bitmap
@@ -443,6 +444,7 @@ func (vc *viewChange) InitPayload(
 }
 
 // isM1PayloadEmpty returns true if m1Payload is not set
+// this is an unlocked internal function call
 func (vc *viewChange) isM1PayloadEmpty() bool {
 	return len(vc.m1Payload) == 0
 }
@@ -463,6 +465,8 @@ func (vc *viewChange) GetViewIDBitmap(viewID uint64) *bls_cosi.Mask {
 
 // GetM1Payload returns the m1Payload
 func (vc *viewChange) GetM1Payload() []byte {
+	vc.vcLock.RLock()
+	defer vc.vcLock.RUnlock()
 	return vc.m1Payload
 }
 

consensus/view_change_msg.go

@@ -106,7 +106,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64, priKey *bls.P
 		Type:        msg_pb.MessageType_NEWVIEW,
 		Request: &msg_pb.Message_Viewchange{
 			Viewchange: &msg_pb.ViewChangeRequest{
-				ViewId:       consensus.GetViewChangingID(),
+				ViewId:       viewID,
 				BlockNum:     consensus.blockNum,
 				ShardId:      consensus.ShardID,
 				SenderPubkey: priKey.Pub.Bytes[:],

core/state/statedb.go

@@ -739,9 +739,11 @@ func (db *DB) Prepare(thash, bhash common.Hash, ti int) {
 }
 
 func (db *DB) clearJournalAndRefund() {
+	if len(db.journal.entries) > 0 {
 		db.journal = newJournal()
-	db.validRevisions = db.validRevisions[:0]
 		db.refund = 0
+	}
+	db.validRevisions = db.validRevisions[:0] // Snapshots can be created without journal entires
 }
 
 // Commit writes the state to the underlying in-memory trie database.

core/vm/evm.go

@@ -217,7 +217,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
 		to       = AccountRef(addr)
 		snapshot = evm.StateDB.Snapshot()
 	)
-	if !evm.StateDB.Exist(addr) {
+	if !evm.StateDB.Exist(addr) && txType != types.SubtractionOnly {
 		precompiles := PrecompiledContractsHomestead
 		if evm.ChainConfig().IsS3(evm.EpochNumber) {
 			precompiles = PrecompiledContractsByzantium

node/node.go

@@ -462,9 +462,9 @@ func (node *Node) validateShardBoundMessage(
 			return nil, nil, true, nil
 		}
 	} else {
-		// ignore newview message if the node is not in viewchanging mode
+		// ignore viewchange/newview message if the node is not in viewchanging mode
 		switch m.Type {
-		case msg_pb.MessageType_NEWVIEW:
+		case msg_pb.MessageType_NEWVIEW, msg_pb.MessageType_VIEWCHANGE:
 			return nil, nil, true, nil
 		}
 	}