package consensus import ( "bytes" "encoding/binary" "sync" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core/types" bls_core "github.com/harmony-one/bls/ffi/go/bls" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/crypto/bls" bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/chain" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/multibls" "github.com/pkg/errors" "github.com/rs/zerolog" ) const ( // ValidPayloadLength is the valid length for viewchange payload ValidPayloadLength = 32 + bls.BLSSignatureSizeInBytes ) // viewChange encapsulate all the view change related data structure and functions type viewChange struct { // Commits collected from view change // for each viewID, we need keep track of corresponding sigs and bitmap // until one of the viewID has enough votes (>=2f+1) // after one of viewID has enough votes, we can reset and clean the map // honest nodes will never double votes on different viewID // bhpSigs: blockHashPreparedSigs is the signature on m1 type message bhpSigs map[uint64]map[string]*bls_core.Sign // nilSigs: there is no prepared message when view change, // it's signature on m2 type (i.e. nil) messages nilSigs map[uint64]map[string]*bls_core.Sign viewIDSigs map[uint64]map[string]*bls_core.Sign bhpBitmap map[uint64]*bls_cosi.Mask nilBitmap map[uint64]*bls_cosi.Mask viewIDBitmap map[uint64]*bls_cosi.Mask newViewMsg map[uint64]map[string]uint64 m1Payload []byte // message payload for type m1 := |vcBlockHash|prepared_agg_sigs|prepared_bitmap|, new leader only need one blockVerifier BlockVerifierFunc viewChangeDuration time.Duration // mutex for view change vcLock sync.RWMutex } // newViewChange returns a new viewChange object func newViewChange() *viewChange { vc := viewChange{} vc.Reset() return &vc } // SetBlockVerifier .. func (vc *viewChange) SetBlockVerifier(verifier BlockVerifierFunc) { vc.blockVerifier = verifier } // AddViewIDKeyIfNotExist .. func (vc *viewChange) AddViewIDKeyIfNotExist(viewID uint64, members multibls.PublicKeys) { vc.vcLock.Lock() defer vc.vcLock.Unlock() if _, ok := vc.bhpSigs[viewID]; !ok { vc.bhpSigs[viewID] = map[string]*bls_core.Sign{} } if _, ok := vc.nilSigs[viewID]; !ok { vc.nilSigs[viewID] = map[string]*bls_core.Sign{} } if _, ok := vc.viewIDSigs[viewID]; !ok { vc.viewIDSigs[viewID] = map[string]*bls_core.Sign{} } if _, ok := vc.bhpBitmap[viewID]; !ok { bhpBitmap, _ := bls_cosi.NewMask(members, nil) vc.bhpBitmap[viewID] = bhpBitmap } if _, ok := vc.nilBitmap[viewID]; !ok { nilBitmap, _ := bls_cosi.NewMask(members, nil) vc.nilBitmap[viewID] = nilBitmap } if _, ok := vc.viewIDBitmap[viewID]; !ok { viewIDBitmap, _ := bls_cosi.NewMask(members, nil) vc.viewIDBitmap[viewID] = viewIDBitmap } } // Reset reset the state for viewchange func (vc *viewChange) Reset() { vc.vcLock.Lock() defer vc.vcLock.Unlock() vc.m1Payload = []byte{} vc.bhpSigs = map[uint64]map[string]*bls_core.Sign{} vc.nilSigs = map[uint64]map[string]*bls_core.Sign{} vc.viewIDSigs = map[uint64]map[string]*bls_core.Sign{} vc.bhpBitmap = map[uint64]*bls_cosi.Mask{} vc.nilBitmap = map[uint64]*bls_cosi.Mask{} vc.viewIDBitmap = map[uint64]*bls_cosi.Mask{} vc.newViewMsg = map[uint64]map[string]uint64{} } // GetPreparedBlock returns the prepared block or nil if not found func (vc *viewChange) GetPreparedBlock(fbftlog *FBFTLog, hash [32]byte) ([]byte, []byte) { vc.vcLock.RLock() defer vc.vcLock.RUnlock() if vc.isM1PayloadEmpty() { return nil, nil } if block := fbftlog.GetBlockByHash(hash); block != nil { encodedBlock, err := rlp.EncodeToBytes(block) if err != nil || len(encodedBlock) == 0 { return nil, nil } return vc.m1Payload, encodedBlock } return nil, nil } // GetM2Bitmap returns the nilBitmap as M2Bitmap func (vc *viewChange) GetM2Bitmap(viewID uint64) ([]byte, []byte) { vc.vcLock.RLock() defer vc.vcLock.RUnlock() sig2arr := []*bls_core.Sign{} for _, sig := range vc.nilSigs[viewID] { sig2arr = append(sig2arr, sig) } if len(sig2arr) > 0 { m2Sig := bls_cosi.AggregateSig(sig2arr) vc.getLogger().Info().Int("len", len(sig2arr)).Msg("[GetM2Bitmap] M2 (NIL) type signatures") return m2Sig.Serialize(), vc.nilBitmap[viewID].Bitmap } vc.getLogger().Info().Uint64("viewID", viewID).Msg("[GetM2Bitmap] No M2 (NIL) type signatures") return nil, nil } // GetM3Bitmap returns the viewIDBitmap as M3Bitmap func (vc *viewChange) GetM3Bitmap(viewID uint64) ([]byte, []byte) { vc.vcLock.RLock() defer vc.vcLock.RUnlock() sig3arr := []*bls_core.Sign{} for _, sig := range vc.viewIDSigs[viewID] { sig3arr = append(sig3arr, sig) } // even we check here for safty, m3 type signatures must >= 2f+1 if len(sig3arr) > 0 { m3Sig := bls_cosi.AggregateSig(sig3arr) vc.getLogger().Info().Int("len", len(sig3arr)).Msg("[GetM3Bitmap] M3 (ViewID) type signatures") return m3Sig.Serialize(), vc.viewIDBitmap[viewID].Bitmap } vc.getLogger().Info().Uint64("viewID", viewID).Msg("[GetM3Bitmap] No M3 (ViewID) type signatures") return nil, nil } // VerifyNewViewMsg returns true if the new view message is valid func (vc *viewChange) VerifyNewViewMsg(recvMsg *FBFTMessage) (*types.Block, error) { vc.vcLock.Lock() defer vc.vcLock.Unlock() if recvMsg.M3AggSig == nil || recvMsg.M3Bitmap == nil { return nil, errors.New("[VerifyNewViewMsg] M3AggSig or M3Bitmap is nil") } senderKey := recvMsg.SenderPubkeys[0] senderKeyStr := senderKey.Bytes.Hex() // first time received the new view message _, ok := vc.newViewMsg[recvMsg.ViewID] if !ok { newViewMap := map[string]uint64{} vc.newViewMsg[recvMsg.ViewID] = newViewMap } _, ok = vc.newViewMsg[recvMsg.ViewID][senderKeyStr] if ok { vc.getLogger().Warn(). Uint64("MsgBlockNum", recvMsg.BlockNum). Uint64("MyBlockNum", vc.newViewMsg[recvMsg.ViewID][senderKeyStr]). Msg("[VerifyNewViewMsg] redundant NewView Message") } vc.newViewMsg[recvMsg.ViewID][senderKeyStr] = recvMsg.BlockNum m3Sig := recvMsg.M3AggSig m3Mask := recvMsg.M3Bitmap viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, recvMsg.ViewID) if !m3Sig.VerifyHash(m3Mask.AggregatePublic, viewIDBytes) { return nil, errors.New("[VerifyNewViewMsg] Unable to Verify Aggregated Signature of M3 (ViewID) payload") } m2Mask := recvMsg.M2Bitmap if recvMsg.M2AggSig != nil { m2Sig := recvMsg.M2AggSig if !m2Sig.VerifyHash(m2Mask.AggregatePublic, NIL) { return nil, errors.New("[VerifyNewViewMsg] Unable to Verify Aggregated Signature of M2 (NIL) payload") } } // check when M3 sigs > M2 sigs, then M1 (recvMsg.Payload) should not be empty preparedBlock := &types.Block{} if len(recvMsg.Payload) >= ValidPayloadLength && len(recvMsg.Block) != 0 { if err := rlp.DecodeBytes(recvMsg.Block, preparedBlock); err != nil { return nil, errors.New("[VerifyNewViewMsg] Unparseable prepared block data") } blockHash := recvMsg.Payload[:32] preparedBlockHash := preparedBlock.Hash() if !bytes.Equal(preparedBlockHash[:], blockHash) { return nil, errors.New("[VerifyNewViewMsg] Prepared block hash doesn't match msg block hash") } if err := vc.blockVerifier(preparedBlock); err != nil { return nil, errors.New("[VerifyNewViewMsg] Prepared block verification failed") } return preparedBlock, nil } return nil, nil } var ( errDupM1 = errors.New("received M1 (prepared) message already") errDupM2 = errors.New("received M2 (NIL) message already") errDupM3 = errors.New("received M3 (ViewID) message already") errVerifyM1 = errors.New("failed to verfiy signature for M1 message") errVerifyM2 = errors.New("failed to verfiy signature for M2 message") errM1Payload = errors.New("failed to verify multi signature for M1 prepared payload") errNoQuorum = errors.New("no quorum on M1 (prepared) payload") errIncorrectSender = errors.New("multiple senders not allowed") ) // ProcessViewChangeMsg process the view change message after verification func (vc *viewChange) ProcessViewChangeMsg( fbftlog *FBFTLog, decider quorum.Decider, recvMsg *FBFTMessage, ) error { vc.vcLock.Lock() defer vc.vcLock.Unlock() preparedBlock := &types.Block{} if !recvMsg.HasSingleSender() { return errIncorrectSender } senderKey := recvMsg.SenderPubkeys[0] senderKeyStr := senderKey.Bytes.Hex() // check and add viewID (m3 type) message signature if _, ok := vc.viewIDSigs[recvMsg.ViewID][senderKeyStr]; ok { return errDupM3 } if len(recvMsg.Payload) >= ValidPayloadLength && len(recvMsg.Block) != 0 { if err := rlp.DecodeBytes(recvMsg.Block, preparedBlock); err != nil { return err } if err := vc.blockVerifier(preparedBlock); err != nil { return err } _, ok := vc.bhpSigs[recvMsg.ViewID][senderKeyStr] if ok { return errDupM1 } if !recvMsg.ViewchangeSig.VerifyHash(senderKey.Object, recvMsg.Payload) { return errVerifyM1 } blockHash := recvMsg.Payload[:32] aggSig, mask, err := chain.ReadSignatureBitmapByPublicKeys(recvMsg.Payload[32:], decider.Participants()) if err != nil { return err } if !decider.IsQuorumAchievedByMask(mask) { return errNoQuorum } if !aggSig.VerifyHash(mask.AggregatePublic, blockHash[:]) { return errM1Payload } vc.getLogger().Info().Uint64("viewID", recvMsg.ViewID). Str("validatorPubKey", senderKeyStr). Msg("[ProcessViewChangeMsg] Add M1 (prepared) type message") vc.bhpSigs[recvMsg.ViewID][senderKeyStr] = recvMsg.ViewchangeSig vc.bhpBitmap[recvMsg.ViewID].SetKey(senderKey.Bytes, true) // Set the bitmap indicating that this validator signed. vc.getLogger().Info().Uint64("viewID", recvMsg.ViewID). Str("validatorPubKey", senderKeyStr). Msg("[ProcessViewChangeMsg] Add M3 (ViewID) type message") vc.viewIDSigs[recvMsg.ViewID][senderKeyStr] = recvMsg.ViewidSig // Set the bitmap indicating that this validator signed. vc.viewIDBitmap[recvMsg.ViewID].SetKey(senderKey.Bytes, true) if vc.isM1PayloadEmpty() { vc.m1Payload = append(recvMsg.Payload[:0:0], recvMsg.Payload...) // create prepared message for new leader preparedMsg := FBFTMessage{ MessageType: msg_pb.MessageType_PREPARED, ViewID: recvMsg.ViewID, BlockNum: recvMsg.BlockNum, } preparedMsg.BlockHash = common.Hash{} copy(preparedMsg.BlockHash[:], recvMsg.Payload[:32]) preparedMsg.Payload = make([]byte, len(recvMsg.Payload)-32) copy(preparedMsg.Payload[:], recvMsg.Payload[32:]) preparedMsg.SenderPubkeys = []*bls.PublicKeyWrapper{recvMsg.LeaderPubkey} vc.getLogger().Info().Msg("[ProcessViewChangeMsg] New Leader Prepared Message Added") fbftlog.AddMessage(&preparedMsg) fbftlog.AddBlock(preparedBlock) } return nil } _, ok := vc.nilSigs[recvMsg.ViewID][senderKeyStr] if ok { return errDupM2 } if !recvMsg.ViewchangeSig.VerifyHash(senderKey.Object, NIL) { return errVerifyM2 } vc.getLogger().Info().Uint64("viewID", recvMsg.ViewID). Str("validatorPubKey", senderKeyStr). Msg("[ProcessViewChangeMsg] Add M2 (NIL) type message") vc.nilSigs[recvMsg.ViewID][senderKeyStr] = recvMsg.ViewchangeSig vc.nilBitmap[recvMsg.ViewID].SetKey(senderKey.Bytes, true) // Set the bitmap indicating that this validator signed. vc.getLogger().Info().Uint64("viewID", recvMsg.ViewID). Str("validatorPubKey", senderKeyStr). Msg("[ProcessViewChangeMsg] Add M3 (ViewID) type message") vc.viewIDSigs[recvMsg.ViewID][senderKeyStr] = recvMsg.ViewidSig // Set the bitmap indicating that this validator signed. vc.viewIDBitmap[recvMsg.ViewID].SetKey(senderKey.Bytes, true) return nil } // InitPayload do it once when validator received view change message func (vc *viewChange) InitPayload( fbftlog *FBFTLog, viewID uint64, blockNum uint64, privKeys multibls.PrivateKeys, ) error { vc.vcLock.Lock() defer vc.vcLock.Unlock() // m1 or m2 init once per viewID/key. // m1 and m2 are mutually exclusive. // If the node has valid prepared block, it will add m1 signature. // If not, the node will add m2 signature. // Honest node should have only one kind of m1/m2 signature. inited := false for _, key := range privKeys { _, ok1 := vc.bhpSigs[viewID][key.Pub.Bytes.Hex()] _, ok2 := vc.nilSigs[viewID][key.Pub.Bytes.Hex()] if ok1 || ok2 { inited = true break } } // add my own M1/M2 type message signature and bitmap if !inited { preparedMsgs := fbftlog.GetMessagesByTypeSeq(msg_pb.MessageType_PREPARED, blockNum) preparedMsg := fbftlog.FindMessageByMaxViewID(preparedMsgs) hasBlock := false if preparedMsg != nil { if preparedBlock := fbftlog.GetBlockByHash(preparedMsg.BlockHash); preparedBlock != nil { if err := vc.blockVerifier(preparedBlock); err == nil { vc.getLogger().Info().Uint64("viewID", viewID).Uint64("blockNum", blockNum).Msg("[InitPayload] add my M1 (prepared) type messaage") msgToSign := append(preparedMsg.BlockHash[:], preparedMsg.Payload...) for _, key := range privKeys { if err := vc.bhpBitmap[viewID].SetKey(key.Pub.Bytes, true); err != nil { vc.getLogger().Warn().Str("key", key.Pub.Bytes.Hex()).Msg("[InitPayload] bhpBitmap setkey failed") continue } vc.bhpSigs[viewID][key.Pub.Bytes.Hex()] = key.Pri.SignHash(msgToSign) } hasBlock = true // if m1Payload is empty, we just add one if vc.isM1PayloadEmpty() { vc.m1Payload = append(preparedMsg.BlockHash[:], preparedMsg.Payload...) } } } } if !hasBlock { vc.getLogger().Info().Uint64("viewID", viewID).Uint64("blockNum", blockNum).Msg("[InitPayload] add my M2 (NIL) type messaage") for _, key := range privKeys { if err := vc.nilBitmap[viewID].SetKey(key.Pub.Bytes, true); err != nil { vc.getLogger().Warn().Str("key", key.Pub.Bytes.Hex()).Msg("[InitPayload] nilBitmap setkey failed") continue } vc.nilSigs[viewID][key.Pub.Bytes.Hex()] = key.Pri.SignHash(NIL) } } } inited = false for _, key := range privKeys { _, ok3 := vc.viewIDSigs[viewID][key.Pub.Bytes.Hex()] if ok3 { inited = true break } } // add my own M3 type message signature and bitmap if !inited { viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, viewID) vc.getLogger().Info().Uint64("viewID", viewID).Uint64("blockNum", blockNum).Msg("[InitPayload] add my M3 (ViewID) type messaage") for _, key := range privKeys { if err := vc.viewIDBitmap[viewID].SetKey(key.Pub.Bytes, true); err != nil { vc.getLogger().Warn().Str("key", key.Pub.Bytes.Hex()).Msg("[InitPayload] viewIDBitmap setkey failed") continue } vc.viewIDSigs[viewID][key.Pub.Bytes.Hex()] = key.Pri.SignHash(viewIDBytes) } } return nil } // isM1PayloadEmpty returns true if m1Payload is not set // this is an unlocked internal function call func (vc *viewChange) isM1PayloadEmpty() bool { return len(vc.m1Payload) == 0 } // IsM1PayloadEmpty returns true if m1Payload is not set func (vc *viewChange) IsM1PayloadEmpty() bool { vc.vcLock.RLock() defer vc.vcLock.RUnlock() return vc.isM1PayloadEmpty() } // GetViewIDBitmap returns the viewIDBitmap func (vc *viewChange) GetViewIDBitmap(viewID uint64) *bls_cosi.Mask { vc.vcLock.RLock() defer vc.vcLock.RUnlock() return vc.viewIDBitmap[viewID] } // GetM1Payload returns the m1Payload func (vc *viewChange) GetM1Payload() []byte { vc.vcLock.RLock() defer vc.vcLock.RUnlock() return vc.m1Payload } // getLogger returns logger for view change contexts added func (vc *viewChange) getLogger() *zerolog.Logger { logger := utils.Logger().With(). Str("context", "ViewChange"). Logger() return &logger }