package consensus import ( "encoding/binary" "fmt" "sync" "github.com/ethereum/go-ethereum/common" bls_core "github.com/harmony-one/bls/ffi/go/bls" "github.com/pkg/errors" "github.com/rs/zerolog" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" bls_cosi "github.com/harmony-one/harmony/crypto/bls" ) // FBFTMessage is the record of pbft messages received by a node during FBFT process type FBFTMessage struct { MessageType msg_pb.MessageType ViewID uint64 BlockNum uint64 BlockHash common.Hash Block []byte SenderPubkeys []*bls.PublicKeyWrapper SenderPubkeyBitmap []byte LeaderPubkey *bls.PublicKeyWrapper Payload []byte ViewchangeSig *bls_core.Sign ViewidSig *bls_core.Sign M2AggSig *bls_core.Sign M2Bitmap *bls_cosi.Mask M3AggSig *bls_core.Sign M3Bitmap *bls_cosi.Mask Verified bool } // String .. func (m *FBFTMessage) String() string { sender := "" for _, key := range m.SenderPubkeys { if sender == "" { sender = key.Bytes.Hex() } else { sender = sender + ";" + key.Bytes.Hex() } } leader := "" if m.LeaderPubkey != nil { leader = m.LeaderPubkey.Bytes.Hex() } return fmt.Sprintf( "[Type:%s ViewID:%d Num:%d BlockHash:%s Sender:%s Leader:%s]", m.MessageType.String(), m.ViewID, m.BlockNum, m.BlockHash.Hex(), sender, leader, ) } // HasSingleSender returns whether the message has only a single sender func (m *FBFTMessage) HasSingleSender() bool { return len(m.SenderPubkeys) == 1 } const ( idTypeBytes = 4 idViewIDBytes = 8 idHashBytes = common.HashLength idSenderBytes = bls.PublicKeySizeInBytes idBytes = idTypeBytes + idViewIDBytes + idHashBytes + idSenderBytes ) type ( // fbftMsgID is the id that uniquely defines a fbft message. fbftMsgID [idBytes]byte ) // id return the ID of the FBFT message which uniquely identifies a FBFT message. // The ID is a concatenation of MsgType, BlockHash, and sender key func (m *FBFTMessage) id() fbftMsgID { var id fbftMsgID binary.LittleEndian.PutUint32(id[:], uint32(m.MessageType)) binary.LittleEndian.PutUint64(id[idTypeBytes:], m.ViewID) copy(id[idTypeBytes+idViewIDBytes:], m.BlockHash[:]) if m.HasSingleSender() { copy(id[idTypeBytes+idViewIDBytes+idHashBytes:], m.SenderPubkeys[0].Bytes[:]) } else { // Currently this case is not reachable as only validator will use id() func // and validator won't receive message with multiple senders copy(id[idTypeBytes+idViewIDBytes+idHashBytes:], m.SenderPubkeyBitmap[:]) } return id } type FBFT interface { GetMessagesByTypeSeq(typ msg_pb.MessageType, blockNum uint64) []*FBFTMessage IsBlockVerified(hash common.Hash) bool DeleteBlockByNumber(number uint64) GetBlockByHash(hash common.Hash) *types.Block AddVerifiedMessage(msg *FBFTMessage) AddBlock(block *types.Block) GetMessagesByTypeSeqHash(typ msg_pb.MessageType, blockNum uint64, blockHash common.Hash) []*FBFTMessage } // FBFTLog represents the log stored by a node during FBFT process type FBFTLog struct { blocks map[common.Hash]*types.Block // store blocks received in FBFT verifiedBlocks map[common.Hash]struct{} // store block hashes for blocks that has already been verified messages map[fbftMsgID]*FBFTMessage // store messages received in FBFT verifyNewBlock func(*types.Block) error // block verification function } // NewFBFTLog returns new instance of FBFTLog func NewFBFTLog(verifyNewBlock func(*types.Block) error) *FBFTLog { pbftLog := FBFTLog{ blocks: make(map[common.Hash]*types.Block), messages: make(map[fbftMsgID]*FBFTMessage), verifiedBlocks: make(map[common.Hash]struct{}), verifyNewBlock: verifyNewBlock, } return &pbftLog } // AddBlock add a new block into the log func (log *FBFTLog) AddBlock(block *types.Block) { log.blocks[block.Hash()] = block } func (log *FBFTLog) BlockVerify(block *types.Block) error { return log.verifyNewBlock(block) } // MarkBlockVerified marks the block as verified func (log *FBFTLog) MarkBlockVerified(block *types.Block) { log.verifiedBlocks[block.Hash()] = struct{}{} } // IsBlockVerified checks whether the block is verified func (log *FBFTLog) IsBlockVerified(hash common.Hash) bool { _, exist := log.verifiedBlocks[hash] return exist } // GetBlockByHash returns the block matches the given block hash func (log *FBFTLog) GetBlockByHash(hash common.Hash) *types.Block { return log.blocks[hash] } // GetBlocksByNumber returns the blocks match the given block number func (log *FBFTLog) GetBlocksByNumber(number uint64) []*types.Block { var blocks []*types.Block for _, block := range log.blocks { if block.NumberU64() == number { blocks = append(blocks, block) } } return blocks } // DeleteBlocksLessThan deletes blocks less than given block number func (log *FBFTLog) deleteBlocksLessThan(number uint64) { for h, block := range log.blocks { if block.NumberU64() < number { delete(log.blocks, h) delete(log.verifiedBlocks, h) } } } // DeleteBlockByNumber deletes block of specific number func (log *FBFTLog) DeleteBlockByNumber(number uint64) { for h, block := range log.blocks { if block.NumberU64() == number { delete(log.blocks, h) delete(log.verifiedBlocks, h) } } } // DeleteMessagesLessThan deletes messages less than given block number func (log *FBFTLog) deleteMessagesLessThan(number uint64) { for h, msg := range log.messages { if msg.BlockNum < number { delete(log.messages, h) } } } // AddVerifiedMessage adds a signature verified pbft message into the log func (log *FBFTLog) AddVerifiedMessage(msg *FBFTMessage) { msg.Verified = true log.messages[msg.id()] = msg } // AddNotVerifiedMessage adds a not signature verified pbft message into the log func (log *FBFTLog) AddNotVerifiedMessage(msg *FBFTMessage) { msg.Verified = false log.messages[msg.id()] = msg } // GetNotVerifiedCommittedMessages returns not verified committed pbft messages with matching blockNum, viewID and blockHash func (log *FBFTLog) GetNotVerifiedCommittedMessages(blockNum uint64, viewID uint64, blockHash common.Hash) []*FBFTMessage { var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == msg_pb.MessageType_COMMITTED && msg.BlockNum == blockNum && msg.ViewID == viewID && msg.BlockHash == blockHash && !msg.Verified { found = append(found, msg) } } return found } // GetMessagesByTypeSeqViewHash returns pbft messages with matching type, blockNum, viewID and blockHash func (log *FBFTLog) GetMessagesByTypeSeqViewHash(typ msg_pb.MessageType, blockNum uint64, viewID uint64, blockHash common.Hash) []*FBFTMessage { var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.ViewID == viewID && msg.BlockHash == blockHash && msg.Verified { found = append(found, msg) } } return found } // GetMessagesByTypeSeq returns pbft messages with matching type, blockNum func (log *FBFTLog) GetMessagesByTypeSeq(typ msg_pb.MessageType, blockNum uint64) []*FBFTMessage { var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.Verified { found = append(found, msg) } } return found } // GetMessagesByTypeSeqHash returns pbft messages with matching type, blockNum func (log *FBFTLog) GetMessagesByTypeSeqHash(typ msg_pb.MessageType, blockNum uint64, blockHash common.Hash) []*FBFTMessage { var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType == typ && msg.BlockNum == blockNum && msg.BlockHash == blockHash && msg.Verified { found = append(found, msg) } } return found } // HasMatchingAnnounce returns whether the log contains announce type message with given blockNum, blockHash func (log *FBFTLog) HasMatchingAnnounce(blockNum uint64, blockHash common.Hash) bool { found := log.GetMessagesByTypeSeqHash(msg_pb.MessageType_ANNOUNCE, blockNum, blockHash) return len(found) >= 1 } // HasMatchingViewAnnounce returns whether the log contains announce type message with given blockNum, viewID and blockHash func (log *FBFTLog) HasMatchingViewAnnounce(blockNum uint64, viewID uint64, blockHash common.Hash) bool { found := log.GetMessagesByTypeSeqViewHash(msg_pb.MessageType_ANNOUNCE, blockNum, viewID, blockHash) return len(found) >= 1 } // HasMatchingPrepared returns whether the log contains prepared message with given blockNum, viewID and blockHash func (log *FBFTLog) HasMatchingPrepared(blockNum uint64, blockHash common.Hash) bool { found := log.GetMessagesByTypeSeqHash(msg_pb.MessageType_PREPARED, blockNum, blockHash) return len(found) >= 1 } // HasMatchingViewPrepared returns whether the log contains prepared message with given blockNum, viewID and blockHash func (log *FBFTLog) HasMatchingViewPrepared(blockNum uint64, viewID uint64, blockHash common.Hash) bool { found := log.GetMessagesByTypeSeqViewHash(msg_pb.MessageType_PREPARED, blockNum, viewID, blockHash) return len(found) >= 1 } // GetMessagesByTypeSeqView returns pbft messages with matching type, blockNum and viewID func (log *FBFTLog) GetMessagesByTypeSeqView(typ msg_pb.MessageType, blockNum uint64, viewID uint64) []*FBFTMessage { var found []*FBFTMessage for _, msg := range log.messages { if msg.MessageType != typ || msg.BlockNum != blockNum || msg.ViewID != viewID && msg.Verified { continue } found = append(found, msg) } return found } // FindMessageByMaxViewID returns the message that has maximum ViewID func (log *FBFTLog) FindMessageByMaxViewID(msgs []*FBFTMessage) *FBFTMessage { if len(msgs) == 0 { return nil } maxIdx := -1 maxViewID := uint64(0) for k, v := range msgs { if v.ViewID >= maxViewID { maxIdx = k maxViewID = v.ViewID } } return msgs[maxIdx] } // ParseFBFTMessage parses FBFT message into FBFTMessage structure func (consensus *Consensus) ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { consensus.mutex.Lock() defer consensus.mutex.Unlock() return consensus.parseFBFTMessage(msg) } func (consensus *Consensus) parseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) { // TODO Have this do sanity checks on the message please pbftMsg := FBFTMessage{} pbftMsg.MessageType = msg.GetType() consensusMsg := msg.GetConsensus() pbftMsg.ViewID = consensusMsg.ViewId pbftMsg.BlockNum = consensusMsg.BlockNum copy(pbftMsg.BlockHash[:], consensusMsg.BlockHash[:]) pbftMsg.Payload = make([]byte, len(consensusMsg.Payload)) copy(pbftMsg.Payload[:], consensusMsg.Payload[:]) pbftMsg.Block = make([]byte, len(consensusMsg.Block)) copy(pbftMsg.Block[:], consensusMsg.Block[:]) pbftMsg.SenderPubkeyBitmap = make([]byte, len(consensusMsg.SenderPubkeyBitmap)) copy(pbftMsg.SenderPubkeyBitmap[:], consensusMsg.SenderPubkeyBitmap[:]) if len(consensusMsg.SenderPubkey) != 0 { // If SenderPubKey is populated, treat it as a single key message pubKey, err := bls_cosi.BytesToBLSPublicKey(consensusMsg.SenderPubkey) if err != nil { return nil, err } pbftMsg.SenderPubkeys = []*bls.PublicKeyWrapper{{Object: pubKey}} copy(pbftMsg.SenderPubkeys[0].Bytes[:], consensusMsg.SenderPubkey[:]) } else { // else, it should be a multi-key message where the bitmap is populated pubKeys, err := consensus.multiSigBitmap.GetSignedPubKeysFromBitmap(pbftMsg.SenderPubkeyBitmap) if err != nil { return nil, err } pbftMsg.SenderPubkeys = pubKeys } return &pbftMsg, nil } var errFBFTLogNotFound = errors.New("FBFT log not found") // GetCommittedBlockAndMsgsFromNumber get committed block and message starting from block number bn. func (log *FBFTLog) GetCommittedBlockAndMsgsFromNumber(bn uint64, logger *zerolog.Logger) (*types.Block, *FBFTMessage, error) { msgs := log.GetMessagesByTypeSeq( msg_pb.MessageType_COMMITTED, bn, ) if len(msgs) == 0 { return nil, nil, errFBFTLogNotFound } if len(msgs) > 1 { logger.Error().Int("numMsgs", len(msgs)).Err(errors.New("DANGER!!! multiple COMMITTED message in PBFT log observed")) } for i := range msgs { block := log.GetBlockByHash(msgs[i].BlockHash) if block == nil { logger.Debug(). Uint64("blockNum", msgs[i].BlockNum). Uint64("viewID", msgs[i].ViewID). Str("blockHash", msgs[i].BlockHash.Hex()). Err(errors.New("failed finding a matching block for committed message")) continue } return block, msgs[i], nil } return nil, nil, errFBFTLogNotFound } // PruneCacheBeforeBlock prune all blocks before bn func (log *FBFTLog) PruneCacheBeforeBlock(bn uint64) { log.deleteBlocksLessThan(bn - 1) log.deleteMessagesLessThan(bn - 1) } type threadsafeFBFTLog struct { log *FBFTLog mu *sync.RWMutex } func (t threadsafeFBFTLog) GetMessagesByTypeSeq(typ msg_pb.MessageType, blockNum uint64) []*FBFTMessage { t.mu.RLock() defer t.mu.RUnlock() return t.log.GetMessagesByTypeSeq(typ, blockNum) } func (t threadsafeFBFTLog) IsBlockVerified(hash common.Hash) bool { t.mu.RLock() defer t.mu.RUnlock() return t.log.IsBlockVerified(hash) } func (t threadsafeFBFTLog) DeleteBlockByNumber(number uint64) { t.mu.Lock() defer t.mu.Unlock() t.log.DeleteBlockByNumber(number) } func (t threadsafeFBFTLog) GetBlockByHash(hash common.Hash) *types.Block { t.mu.RLock() defer t.mu.RUnlock() return t.log.GetBlockByHash(hash) } func (t threadsafeFBFTLog) AddVerifiedMessage(msg *FBFTMessage) { t.mu.Lock() defer t.mu.Unlock() t.log.AddVerifiedMessage(msg) } func (t threadsafeFBFTLog) AddBlock(block *types.Block) { t.mu.Lock() defer t.mu.Unlock() t.log.AddBlock(block) } func (t threadsafeFBFTLog) GetMessagesByTypeSeqHash(typ msg_pb.MessageType, blockNum uint64, blockHash common.Hash) []*FBFTMessage { t.mu.RLock() defer t.mu.RUnlock() return t.log.GetMessagesByTypeSeqHash(typ, blockNum, blockHash) }