WIP

consensus/consensus.go

@@ -58,14 +58,19 @@ type Consensus struct {
 	commitBitmap         *bls_cosi.Mask
 
 	// Commits collected from view change
+	// for each viewID, we need keep track of corresponding sigs and bitmap
+	// until one of the viewID has enough votes (>=2f+1)
+	// after one of viewID has enough votes, we can reset and clean the map
+	// honest nodes will never double votes on different viewID
 	// bhpSigs: blockHashPreparedSigs is the signature on m1 type message
-	bhpSigs map[string]*bls.Sign
+	bhpSigs map[uint64]map[string]*bls.Sign
 	// nilSigs: there is no prepared message when view change,
 	// it's signature on m2 type (i.e. nil) messages
-	nilSigs      map[string]*bls.Sign
+	nilSigs      map[uint64]map[string]*bls.Sign
-	bhpBitmap    *bls_cosi.Mask
+	viewIDSigs   map[uint64]map[string]*bls.Sign
-	nilBitmap    *bls_cosi.Mask
+	bhpBitmap    map[uint64]*bls_cosi.Mask
-	viewIDBitmap *bls_cosi.Mask
+	nilBitmap    map[uint64]*bls_cosi.Mask
+	viewIDBitmap map[uint64]*bls_cosi.Mask
 	m1Payload    []byte     // message payload for type m1 := |vcBlockHash|prepared_agg_sigs|prepared_bitmap|, new leader only need one
 	vcLock       sync.Mutex // mutex for view change
 

consensus/consensus_service.go

@@ -170,19 +170,19 @@ func (consensus *Consensus) GetValidatorPeers() []p2p.Peer {
 	return validatorPeers
 }
 
-// GetBhpSigsArray returns the signatures for prepared message in viewchange
+// GetViewIDSigsArray returns the signatures for viewID in viewchange
-func (consensus *Consensus) GetBhpSigsArray() []*bls.Sign {
+func (consensus *Consensus) GetViewIDSigsArray(viewID uint64) []*bls.Sign {
 	sigs := []*bls.Sign{}
-	for _, sig := range consensus.bhpSigs {
+	for _, sig := range consensus.viewIDSigs[viewID] {
 		sigs = append(sigs, sig)
 	}
 	return sigs
 }
 
 // GetNilSigsArray returns the signatures for nil prepared message in viewchange
-func (consensus *Consensus) GetNilSigsArray() []*bls.Sign {
+func (consensus *Consensus) GetNilSigsArray(viewID uint64) []*bls.Sign {
 	sigs := []*bls.Sign{}
-	for _, sig := range consensus.nilSigs {
+	for _, sig := range consensus.nilSigs[viewID] {
 		sigs = append(sigs, sig)
 	}
 	return sigs
@@ -624,3 +624,28 @@ func (consensus *Consensus) NeedsRandomNumberGeneration(epoch *big.Int) bool {
 
 	return false
 }
+
+func (consensus *Consensus) addViewIDKeyIfNotExist(viewID uint64) {
+	members := consensus.Decider.Participants()
+	if _, ok := consensus.bhpSigs[viewID]; !ok {
+		consensus.bhpSigs[viewID] = map[string]*bls.Sign{}
+	}
+	if _, ok := consensus.nilSigs[viewID]; !ok {
+		consensus.nilSigs[viewID] = map[string]*bls.Sign{}
+	}
+	if _, ok := consensus.viewIDSigs[viewID]; !ok {
+		consensus.viewIDSigs[viewID] = map[string]*bls.Sign{}
+	}
+	if _, ok := consensus.bhpBitmap[viewID]; !ok {
+		bhpBitmap, _ := bls_cosi.NewMask(members, nil)
+		consensus.bhpBitmap[viewID] = bhpBitmap
+	}
+	if _, ok := consensus.nilBitmap[viewID]; !ok {
+		nilBitmap, _ := bls_cosi.NewMask(members, nil)
+		consensus.nilBitmap[viewID] = nilBitmap
+	}
+	if _, ok := consensus.viewIDBitmap[viewID]; !ok {
+		viewIDBitmap, _ := bls_cosi.NewMask(members, nil)
+		consensus.viewIDBitmap[viewID] = viewIDBitmap
+	}
+}

consensus/consensus_viewchange_msg.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/harmony-one/harmony/api/proto"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
-	"github.com/harmony-one/harmony/consensus/quorum"
 	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/utils"
 )
@@ -75,7 +74,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte {
 }
 
 // new leader construct newview message
-func (consensus *Consensus) constructNewViewMessage() []byte {
+func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte {
 	message := &msg_pb.Message{
 		ServiceType: msg_pb.ServiceType_CONSENSUS,
 		Type:        msg_pb.MessageType_NEWVIEW,
@@ -92,21 +91,21 @@ func (consensus *Consensus) constructNewViewMessage() []byte {
 	vcMsg.SenderPubkey = consensus.PubKey.Serialize()
 	vcMsg.Payload = consensus.m1Payload
 
-	sig2arr := consensus.GetNilSigsArray()
+	sig2arr := consensus.GetNilSigsArray(viewID)
 	utils.Logger().Debug().Int("len", len(sig2arr)).Msg("[constructNewViewMessage] M2 (NIL) type signatures")
 	if len(sig2arr) > 0 {
 		m2Sig := bls_cosi.AggregateSig(sig2arr)
 		vcMsg.M2Aggsigs = m2Sig.Serialize()
-		vcMsg.M2Bitmap = consensus.nilBitmap.Bitmap
+		vcMsg.M2Bitmap = consensus.nilBitmap[viewID].Bitmap
 	}
 
-	sig3arr := consensus.Decider.ReadAllSignatures(quorum.ViewChange)
+	sig3arr := consensus.GetViewIDSigsArray(viewID)
-	utils.Logger().Debug().Int("len", len(sig3arr)).Msg("[constructNewViewMessage] M3 (ViewID) type signatures")
+	consensus.getLogger().Debug().Int("len", len(sig3arr)).Msg("[constructNewViewMessage] M3 (ViewID) type signatures")
 	// even we check here for safty, m3 type signatures must >= 2f+1
 	if len(sig3arr) > 0 {
 		m3Sig := bls_cosi.AggregateSig(sig3arr)
 		vcMsg.M3Aggsigs = m3Sig.Serialize()
-		vcMsg.M3Bitmap = consensus.viewIDBitmap.Bitmap
+		vcMsg.M3Bitmap = consensus.viewIDBitmap[viewID].Bitmap
 	}
 
 	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)

consensus/view_change.go

@@ -90,16 +90,15 @@ func (consensus *Consensus) ResetViewChangeState() {
 		Str("Phase", consensus.phase.String()).
 		Msg("[ResetViewChangeState] Resetting view change state")
 	consensus.current.SetMode(Normal)
-	members := consensus.Decider.Participants()
-	bhpBitmap, _ := bls_cosi.NewMask(members, nil)
-	nilBitmap, _ := bls_cosi.NewMask(members, nil)
-	viewIDBitmap, _ := bls_cosi.NewMask(members, nil)
-	consensus.bhpBitmap = bhpBitmap
-	consensus.nilBitmap = nilBitmap
-	consensus.viewIDBitmap = viewIDBitmap
 	consensus.m1Payload = []byte{}
-	consensus.bhpSigs = map[string]*bls.Sign{}
+	consensus.bhpSigs = map[uint64]map[string]*bls.Sign{}
-	consensus.nilSigs = map[string]*bls.Sign{}
+	consensus.nilSigs = map[uint64]map[string]*bls.Sign{}
+	consensus.viewIDSigs = map[uint64]map[string]*bls.Sign{}
+
+	consensus.bhpBitmap = map[uint64]*bls_cosi.Mask{}
+	consensus.nilBitmap = map[uint64]*bls_cosi.Mask{}
+	consensus.viewIDBitmap = map[uint64]*bls_cosi.Mask{}
+
 	consensus.Decider.Reset([]quorum.Phase{quorum.ViewChange})
 }
 
@@ -202,10 +201,13 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 	consensus.vcLock.Lock()
 	defer consensus.vcLock.Unlock()
 
+	// update the dictionary key if the viewID is first time received
+	consensus.addViewIDKeyIfNotExist(recvMsg.ViewID)
+
 	// TODO: remove NIL type message
 	// add self m1 or m2 type message signature and bitmap
-	_, ok1 := consensus.nilSigs[consensus.PubKey.SerializeToHexStr()]
+	_, ok1 := consensus.nilSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()]
-	_, ok2 := consensus.bhpSigs[consensus.PubKey.SerializeToHexStr()]
+	_, ok2 := consensus.bhpSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()]
 	if !(ok1 || ok2) {
 		// add own signature for newview message
 		preparedMsgs := consensus.FBFTLog.GetMessagesByTypeSeq(
@@ -214,29 +216,27 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 		preparedMsg := consensus.FBFTLog.FindMessageByMaxViewID(preparedMsgs)
 		if preparedMsg == nil {
 			consensus.getLogger().Debug().Msg("[onViewChange] add my M2(NIL) type messaage")
-			consensus.nilSigs[consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(NIL)
+			consensus.nilSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(NIL)
-			consensus.nilBitmap.SetKey(consensus.PubKey, true)
+			consensus.nilBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true)
 		} else {
 			consensus.getLogger().Debug().Msg("[onViewChange] add my M1 type messaage")
 			msgToSign := append(preparedMsg.BlockHash[:], preparedMsg.Payload...)
-			consensus.bhpSigs[consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(msgToSign)
+			consensus.bhpSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(msgToSign)
-			consensus.bhpBitmap.SetKey(consensus.PubKey, true)
+			consensus.bhpBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true)
 		}
 	}
 	// add self m3 type message signature and bitmap
-	signature := consensus.Decider.ReadSignature(quorum.ViewChange, consensus.PubKey)
+	_, ok3 := consensus.viewIDSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()]
-	if signature == nil {
+	if !ok3 {
 		viewIDBytes := make([]byte, 8)
 		binary.LittleEndian.PutUint64(viewIDBytes, recvMsg.ViewID)
-		consensus.Decider.AddSignature(
+		consensus.viewIDSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(viewIDBytes)
-			quorum.ViewChange, consensus.PubKey, consensus.priKey.SignHash(viewIDBytes),
+		consensus.viewIDBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true)
-		)
-		consensus.viewIDBitmap.SetKey(consensus.PubKey, true)
 	}
 
 	// m2 type message
 	if len(recvMsg.Payload) == 0 {
-		_, ok := consensus.nilSigs[senderKey.SerializeToHexStr()]
+		_, ok := consensus.nilSigs[recvMsg.ViewID][senderKey.SerializeToHexStr()]
 		if ok {
 			consensus.getLogger().Debug().
 				Str("validatorPubKey", senderKey.SerializeToHexStr()).
@@ -252,10 +252,10 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 		consensus.getLogger().Debug().
 			Str("validatorPubKey", senderKey.SerializeToHexStr()).
 			Msg("[onViewChange] Add M2 (NIL) type message")
-		consensus.nilSigs[senderKey.SerializeToHexStr()] = recvMsg.ViewchangeSig
+		consensus.nilSigs[recvMsg.ViewID][senderKey.SerializeToHexStr()] = recvMsg.ViewchangeSig
-		consensus.nilBitmap.SetKey(recvMsg.SenderPubkey, true) // Set the bitmap indicating that this validator signed.
+		consensus.nilBitmap[recvMsg.ViewID].SetKey(recvMsg.SenderPubkey, true) // Set the bitmap indicating that this validator signed.
 	} else { // m1 type message
-		_, ok := consensus.bhpSigs[senderKey.SerializeToHexStr()]
+		_, ok := consensus.bhpSigs[recvMsg.ViewID][senderKey.SerializeToHexStr()]
 		if ok {
 			consensus.getLogger().Debug().
 				Str("validatorPubKey", senderKey.SerializeToHexStr()).
@@ -317,13 +317,12 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 		consensus.getLogger().Debug().
 			Str("validatorPubKey", senderKey.SerializeToHexStr()).
 			Msg("[onViewChange] Add M1 (prepared) type message")
-		consensus.bhpSigs[senderKey.SerializeToHexStr()] = recvMsg.ViewchangeSig
+		consensus.bhpSigs[recvMsg.ViewID][senderKey.SerializeToHexStr()] = recvMsg.ViewchangeSig
-		consensus.bhpBitmap.SetKey(recvMsg.SenderPubkey, true) // Set the bitmap indicating that this validator signed.
+		consensus.bhpBitmap[recvMsg.ViewID].SetKey(recvMsg.SenderPubkey, true) // Set the bitmap indicating that this validator signed.
 	}
 
 	// check and add viewID (m3 type) message signature
-	sig := consensus.Decider.ReadSignature(quorum.ViewChange, senderKey)
+	if _, ok := consensus.viewIDSigs[recvMsg.ViewID][senderKey.SerializeToHexStr()]; ok {
-	if sig != nil {
 		consensus.getLogger().Debug().
 			Str("validatorPubKey", senderKey.SerializeToHexStr()).
 			Msg("[onViewChange] Already Received M3(ViewID) message from the validator")
@@ -340,16 +339,17 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 	consensus.getLogger().Debug().
 		Str("validatorPubKey", senderKey.SerializeToHexStr()).
 		Msg("[onViewChange] Add M3 (ViewID) type message")
-	consensus.Decider.AddSignature(quorum.ViewChange, senderKey, recvMsg.ViewidSig)
+
+	consensus.viewIDSigs[recvMsg.ViewID][senderKey.SerializeToHexStr()] = recvMsg.ViewidSig
 	// Set the bitmap indicating that this validator signed.
-	consensus.viewIDBitmap.SetKey(recvMsg.SenderPubkey, true)
+	consensus.viewIDBitmap[recvMsg.ViewID].SetKey(recvMsg.SenderPubkey, true)
 	consensus.getLogger().Debug().
-		Int64("numSigs", consensus.Decider.SignersCount(quorum.ViewChange)).
+		Int("have", len(consensus.viewIDSigs[recvMsg.ViewID])).
 		Int64("needed", consensus.Decider.TwoThirdsSignersCount()).
 		Msg("[onViewChange]")
 
 	// received enough view change messages, change state to normal consensus
-	if consensus.Decider.IsQuorumAchieved(quorum.ViewChange) {
+	if consensus.Decider.IsQuorumAchievedByMask(consensus.viewIDBitmap[recvMsg.ViewID]) {
 		consensus.current.SetMode(Normal)
 		consensus.LeaderPubKey = consensus.PubKey
 		consensus.ResetState()
@@ -391,7 +391,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 		}
 
 		consensus.current.SetViewID(recvMsg.ViewID)
-		msgToSend := consensus.constructNewViewMessage()
+		msgToSend := consensus.constructNewViewMessage(recvMsg.ViewID)
 
 		consensus.getLogger().Warn().
 			Int("payloadSize", len(consensus.m1Payload)).