[slash][consensus] Notice double sign & broadcast, factor out tech debt of consensus (#2152)

* [slash] Remove dead interface, associated piping

* [slash] Expand out structs

* [consensus] Write to a chan when find a case of double-signing, remove dead code

* [slash] Broadcast the noticing of a double signing

* [rawdb] CRUD for slashing candidates

* [slashing][node][proto] Broadcast the slash record after receive from consensus, handle received proto message, persist in off-chain db while pending

* [slash][node][propose-block] Add verified slashes proposed into the header in block proposal

* [slash][shard] Factor out external validator as method on shard state, add double-signature field

* [slash][engine] Apply slash, name boolean expression for sorts, use stable sort

* [slash] Abstract Ballot results so keep track of both pre and post double sign event

* [slash] Fix type errors on test code

* [slash] Read from correct rawdb

* [slash] Add epoch based guards in CRUD of slashing

* [slash] Write to correct cache for slashing candidates

* [shard] Use explicit named type of BLS Signature, use convention

* [slash] Fix mistake done in refactor, improper header used. Factor out fromSlice to set

* [slash][node] Restore newblock to master, try again minimial change

* [cx-receipts] Break up one-liner, use SliceStable, not Slice

* [network] Finish refactor that makes network message headers once

* [network] Simplify creation further of headers write

* [slash] Adjust data structure of slash after offline discussion with RJ, Chao

* [slash] Still did need signature of the double signature

* [consensus] Prepare message does not have block header

* [consensus] Soft reset three files to 968517d~1

* [consensus] Begin factor consensus network intended message out with prepare first

* [consensus] Factor out Prepared message

* [consensus] Factor out announce message creation

* [consensus] Committed Message, branch on verify sender key for clearer log

* [consensus] Committed Message Factor out

* [consensus] Do jenkins MVP of signatures adjustment

* [main][slash] Provide YAML config as webhook config for double sign event

* [consensus] Adjust signatures, whitespace, lessen GC pressure

* [consensus] Remove dead code

* [consensus] Factor out commit overloaded message, give commit payload override in construct

* [consensus] Fix travis tests

* [consensus] Provide block bytes in SubmitVote(quorum.Commit)

* [consensus] Factor out noisy sanity checks in BFT, move existing commit check earlier as was before

* [quorum] Adjust signatures in quorum

* [staking] Adjust after merge from master

* [consensus] Finish refactor of consensus

* [node] Fix import

* [consensus] Fix travis

* [consensus] Use origin/master copy of block, fix mistake of pointer to empty byte

* [consensus] Less verbose bools

* [consensus] Remove unused trailing mutation hook in message construct

* [consensus] Address some TODOs on err, comment out double sign

api/proto/node/node.go

@@ -6,17 +6,16 @@ import (
 	"fmt"
 	"log"
 
-	types2 "github.com/harmony-one/harmony/staking/types"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
-	peer "github.com/libp2p/go-libp2p-peer"
-
 	"github.com/harmony-one/harmony/api/proto"
 	"github.com/harmony-one/harmony/block"
 	"github.com/harmony-one/harmony/consensus/engine"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/staking/slash"
+	staking "github.com/harmony-one/harmony/staking/types"
+	peer "github.com/libp2p/go-libp2p-peer"
 )
 
 // MessageType is to indicate the specific type of message under Node category
@@ -99,8 +98,29 @@ type BlockMessageType int
 const (
 	Sync BlockMessageType = iota
 
-	CrossLink // used for crosslink from beacon chain to shard chain
-	Receipt   // cross-shard transaction receipts
+	CrossLink      // used for crosslink from beacon chain to shard chain
+	Receipt        // cross-shard transaction receipts
+	SlashCandidate // A report of a double-signing event
+)
+
+var (
+	// B suffix means Byte
+	nodeB      = byte(proto.Node)
+	blockB     = byte(Block)
+	slashB     = byte(SlashCandidate)
+	txnB       = byte(Transaction)
+	sendB      = byte(Send)
+	stakingB   = byte(Staking)
+	syncB      = byte(Sync)
+	crossLinkB = byte(CrossLink)
+	receiptB   = byte(Receipt)
+	// H suffix means header
+	slashH           = []byte{nodeB, blockB, slashB}
+	transactionListH = []byte{nodeB, txnB, sendB}
+	stakingTxnListH  = []byte{nodeB, stakingB, sendB}
+	syncH            = []byte{nodeB, blockB, syncB}
+	crossLinkH       = []byte{nodeB, blockB, crossLinkB}
+	cxReceiptH       = []byte{nodeB, blockB, receiptB}
 )
 
 // SerializeBlockchainSyncMessage serializes BlockchainSyncMessage.
@@ -127,10 +147,7 @@ func DeserializeBlockchainSyncMessage(d []byte) (*BlockchainSyncMessage, error)
 
 // ConstructTransactionListMessageAccount constructs serialized transactions in account model
 func ConstructTransactionListMessageAccount(transactions types.Transactions) []byte {
-	byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
-	byteBuffer.WriteByte(byte(Transaction))
-	byteBuffer.WriteByte(byte(Send))
-
+	byteBuffer := bytes.NewBuffer(transactionListH)
 	txs, err := rlp.EncodeToBytes(transactions)
 	if err != nil {
 		log.Fatal(err)
@@ -141,11 +158,10 @@ func ConstructTransactionListMessageAccount(transactions types.Transactions) []b
 }
 
 // ConstructStakingTransactionListMessageAccount constructs serialized staking transactions in account model
-func ConstructStakingTransactionListMessageAccount(transactions types2.StakingTransactions) []byte {
-	byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
-	byteBuffer.WriteByte(byte(Staking))
-	byteBuffer.WriteByte(byte(Send))
-
+func ConstructStakingTransactionListMessageAccount(
+	transactions staking.StakingTransactions,
+) []byte {
+	byteBuffer := bytes.NewBuffer(stakingTxnListH)
 	txs, err := rlp.EncodeToBytes(transactions)
 	if err != nil {
 		log.Fatal(err)
@@ -157,21 +173,23 @@ func ConstructStakingTransactionListMessageAccount(transactions types2.StakingTr
 
 // ConstructBlocksSyncMessage constructs blocks sync message to send blocks to other nodes
 func ConstructBlocksSyncMessage(blocks []*types.Block) []byte {
-	byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
-	byteBuffer.WriteByte(byte(Block))
-	byteBuffer.WriteByte(byte(Sync))
-
+	byteBuffer := bytes.NewBuffer(syncH)
 	blocksData, _ := rlp.EncodeToBytes(blocks)
 	byteBuffer.Write(blocksData)
 	return byteBuffer.Bytes()
 }
 
+// ConstructSlashMessage ..
+func ConstructSlashMessage(witness *slash.Record) []byte {
+	byteBuffer := bytes.NewBuffer(slashH)
+	slashData, _ := rlp.EncodeToBytes(witness)
+	byteBuffer.Write(slashData)
+	return byteBuffer.Bytes()
+}
+
 // ConstructCrossLinkMessage constructs cross link message to send to beacon chain
 func ConstructCrossLinkMessage(bc engine.ChainReader, headers []*block.Header) []byte {
-	byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
-	byteBuffer.WriteByte(byte(Block))
-	byteBuffer.WriteByte(byte(CrossLink))
-
+	byteBuffer := bytes.NewBuffer(crossLinkH)
 	crosslinks := []types.CrossLink{}
 	for _, header := range headers {
 		if header.Number().Uint64() <= 1 || !bc.Config().IsCrossLink(header.Epoch()) {
@@ -192,13 +210,11 @@ func ConstructCrossLinkMessage(bc engine.ChainReader, headers []*block.Header) [
 // ConstructCXReceiptsProof constructs cross shard receipts and related proof including
 // merkle proof, blockHeader and  commitSignatures
 func ConstructCXReceiptsProof(cxReceiptsProof *types.CXReceiptsProof) []byte {
-	byteBuffer := bytes.NewBuffer([]byte{byte(proto.Node)})
-	byteBuffer.WriteByte(byte(Block))
-	byteBuffer.WriteByte(byte(Receipt))
+	byteBuffer := bytes.NewBuffer(cxReceiptH)
 	by, err := rlp.EncodeToBytes(cxReceiptsProof)
-
 	if err != nil {
-		utils.Logger().Error().Err(err).Msg("[ConstructCXReceiptsProof] Encode CXReceiptsProof Error")
+		const msg = "[ConstructCXReceiptsProof] Encode CXReceiptsProof Error"
+		utils.Logger().Error().Err(err).Msg(msg)
 		return []byte{}
 	}
 	byteBuffer.Write(by)

cmd/harmony/main.go

@@ -17,8 +17,6 @@ import (
 	ethCommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/harmony-one/bls/ffi/go/bls"
-	"github.com/pkg/errors"
-
 	"github.com/harmony-one/harmony/api/service/syncing"
 	"github.com/harmony-one/harmony/consensus"
 	"github.com/harmony-one/harmony/consensus/quorum"
@@ -37,8 +35,9 @@ import (
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
 	p2putils "github.com/harmony-one/harmony/p2p/utils"
 	"github.com/harmony-one/harmony/shard"
-
+	"github.com/harmony-one/harmony/staking/slash"
 	golog "github.com/ipfs/go-log"
+	"github.com/pkg/errors"
 	gologging "github.com/whyrusleeping/go-logging"
 )
 
@@ -130,7 +129,10 @@ var (
 	revertTo       = flag.Int("revert_to", 0, "The revert will rollback all blocks until and including block number revert_to")
 	revertBeacon   = flag.Bool("revert_beacon", false, "Whether to revert beacon chain or the chain this node is assigned to")
 	// Blacklist of addresses
-	blacklistPath = flag.String("blacklist", "./.hmy/blacklist.txt", "Path to newline delimited file of blacklisted wallet addresses")
+	blacklistPath   = flag.String("blacklist", "./.hmy/blacklist.txt", "Path to newline delimited file of blacklisted wallet addresses")
+	webHookYamlPath = flag.String(
+		"webhook_yaml", "", "path for yaml config reporting double signing",
+	)
 )
 
 func initSetup() {
@@ -333,8 +335,20 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node {
 
 	// Current node.
 	chainDBFactory := &shardchain.LDBFactory{RootDir: nodeConfig.DBDir}
+
 	currentNode := node.New(myHost, currentConsensus, chainDBFactory, blacklist, *isArchival)
 
+	if p := *webHookYamlPath; p != "" {
+		config, err := slash.NewDoubleSignWebHooksFromPath(p)
+		if err != nil {
+			fmt.Fprintf(
+				os.Stderr, "ERROR provided yaml path %s but yaml found is illegal", p,
+			)
+			os.Exit(1)
+		}
+		currentNode.NodeConfig.WebHooks.DoubleSigning = config
+	}
+
 	switch {
 	case *networkType == nodeconfig.Localnet:
 		epochConfig := shard.Schedule.InstanceForEpoch(ethCommon.Big0)
@@ -364,14 +378,20 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node {
 	currentNode.NodeConfig.SetPushgatewayPort(nodeConfig.PushgatewayPort)
 	currentNode.NodeConfig.SetMetricsFlag(nodeConfig.MetricsFlag)
 
-	currentNode.NodeConfig.SetBeaconGroupID(nodeconfig.NewGroupIDByShardID(0))
+	currentNode.NodeConfig.SetBeaconGroupID(
+		nodeconfig.NewGroupIDByShardID(shard.BeaconChainShardID),
+	)
 
 	switch *nodeType {
 	case "explorer":
 		nodeconfig.SetDefaultRole(nodeconfig.ExplorerNode)
 		currentNode.NodeConfig.SetRole(nodeconfig.ExplorerNode)
-		currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(*shardID)))
-		currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(nodeconfig.ShardID(*shardID)))
+		currentNode.NodeConfig.SetShardGroupID(
+			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(*shardID)),
+		)
+		currentNode.NodeConfig.SetClientGroupID(
+			nodeconfig.NewClientGroupIDByShardID(nodeconfig.ShardID(*shardID)),
+		)
 	case "validator":
 		nodeconfig.SetDefaultRole(nodeconfig.Validator)
 		currentNode.NodeConfig.SetRole(nodeconfig.Validator)

consensus/checks.go

@@ -0,0 +1,129 @@
+package consensus
+
+import (
+	msg_pb "github.com/harmony-one/harmony/api/proto/message"
+	"github.com/harmony-one/harmony/core/types"
+	"github.com/harmony-one/harmony/internal/chain"
+)
+
+func (consensus *Consensus) validatorSanityChecks(msg *msg_pb.Message) bool {
+	senderKey, err := consensus.verifySenderKey(msg)
+	if err != nil {
+		if err == errValidNotInCommittee {
+			consensus.getLogger().Info().
+				Msg("sender key not in this slot's subcommittee")
+		} else {
+			consensus.getLogger().Error().Err(err).Msg("VerifySenderKey failed")
+		}
+		return false
+	}
+
+	if !senderKey.IsEqual(consensus.LeaderPubKey) &&
+		consensus.current.Mode() == Normal && !consensus.ignoreViewIDCheck {
+		consensus.getLogger().Warn().Msg("[OnPrepared] SenderKey not match leader PubKey")
+		return false
+	}
+
+	if err := verifyMessageSig(senderKey, msg); err != nil {
+		consensus.getLogger().Error().Err(err).Msg(
+			"Failed to verify sender's signature",
+		)
+		return false
+	}
+
+	return true
+}
+
+func (consensus *Consensus) leaderSanityChecks(msg *msg_pb.Message) bool {
+	senderKey, err := consensus.verifySenderKey(msg)
+	if err != nil {
+		if err == errValidNotInCommittee {
+			consensus.getLogger().Info().Msg(
+				"[OnAnnounce] sender key not in this slot's subcommittee",
+			)
+		} else {
+			consensus.getLogger().Error().Err(err).Msg("[OnAnnounce] erifySenderKey failed")
+		}
+		return false
+	}
+	if err = verifyMessageSig(senderKey, msg); err != nil {
+		consensus.getLogger().Error().Err(err).Msg(
+			"[OnPrepare] Failed to verify sender's signature",
+		)
+		return false
+	}
+
+	return true
+}
+
+func (consensus *Consensus) onCommitSanityChecks(
+	recvMsg *FBFTMessage,
+) bool {
+	if recvMsg.ViewID != consensus.viewID || recvMsg.BlockNum != consensus.blockNum {
+		consensus.getLogger().Debug().
+			Uint64("MsgViewID", recvMsg.ViewID).
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("blockNum", consensus.blockNum).
+			Str("ValidatorPubKey", recvMsg.SenderPubkey.SerializeToHexStr()).
+			Msg("[OnCommit] BlockNum/viewID not match")
+		return false
+	}
+
+	if !consensus.FBFTLog.HasMatchingAnnounce(consensus.blockNum, recvMsg.BlockHash) {
+		consensus.getLogger().Debug().
+			Hex("MsgBlockHash", recvMsg.BlockHash[:]).
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("blockNum", consensus.blockNum).
+			Msg("[OnCommit] Cannot find matching blockhash")
+		return false
+	}
+
+	if !consensus.FBFTLog.HasMatchingPrepared(consensus.blockNum, recvMsg.BlockHash) {
+		consensus.getLogger().Debug().
+			Hex("blockHash", recvMsg.BlockHash[:]).
+			Uint64("blockNum", consensus.blockNum).
+			Msg("[OnCommit] Cannot find matching prepared message")
+		return false
+	}
+	return true
+}
+
+func (consensus *Consensus) onPreparedSanityChecks(
+	blockObj *types.Block, recvMsg *FBFTMessage,
+) bool {
+	if blockObj.NumberU64() != recvMsg.BlockNum ||
+		recvMsg.BlockNum < consensus.blockNum {
+		consensus.getLogger().Warn().
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("blockNum", blockObj.NumberU64()).
+			Msg("[OnPrepared] BlockNum not match")
+		return false
+	}
+	if blockObj.Header().Hash() != recvMsg.BlockHash {
+		consensus.getLogger().Warn().
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Hex("MsgBlockHash", recvMsg.BlockHash[:]).
+			Str("blockObjHash", blockObj.Header().Hash().Hex()).
+			Msg("[OnPrepared] BlockHash not match")
+		return false
+	}
+	if consensus.current.Mode() == Normal {
+		err := chain.Engine.VerifyHeader(consensus.ChainReader, blockObj.Header(), true)
+		if err != nil {
+			consensus.getLogger().Error().
+				Err(err).
+				Str("inChain", consensus.ChainReader.CurrentHeader().Number().String()).
+				Str("MsgBlockNum", blockObj.Header().Number().String()).
+				Msg("[OnPrepared] Block header is not verified successfully")
+			return false
+		}
+		if consensus.BlockVerifier == nil {
+			// do nothing
+		} else if err := consensus.BlockVerifier(blockObj); err != nil {
+			consensus.getLogger().Error().Err(err).Msg("[OnPrepared] Block verification failed")
+			return false
+		}
+	}
+
+	return true
+}

consensus/consensus.go

@@ -15,6 +15,7 @@ import (
 	"github.com/harmony-one/harmony/internal/memprofiling"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/p2p"
+	"github.com/harmony-one/harmony/staking/slash"
 )
 
 const (
@@ -126,6 +127,8 @@ type Consensus struct {
 	disableViewChange bool
 	// last node block reward for metrics
 	lastBlockReward *big.Int
+	// Have a dedicated reader thread pull from this chan, like in node
+	SlashChan chan slash.Record
 }
 
 // SetCommitDelay sets the commit message delay.  If set to non-zero,
@@ -208,6 +211,7 @@ func New(
 	consensus.MsgChan = make(chan []byte)
 	consensus.syncReadyChan = make(chan struct{})
 	consensus.syncNotReadyChan = make(chan struct{})
+	consensus.SlashChan = make(chan slash.Record)
 	consensus.commitFinishChan = make(chan uint64)
 	consensus.ReadySignal = make(chan struct{})
 	consensus.lastBlockReward = big.NewInt(0)

consensus/consensus_leader_msg.go

@@ -1,150 +0,0 @@
-package consensus
-
-import (
-	"bytes"
-
-	"github.com/harmony-one/bls/ffi/go/bls"
-	"github.com/harmony-one/harmony/api/proto"
-	msg_pb "github.com/harmony-one/harmony/api/proto/message"
-	"github.com/harmony-one/harmony/consensus/quorum"
-	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
-	"github.com/harmony-one/harmony/internal/utils"
-)
-
-// LeaderNetworkMessage is a message intended to be
-// created only by the leader for distribution to
-// all the other quorum members.
-type LeaderNetworkMessage struct {
-	Phase                      quorum.Phase
-	Bytes                      []byte
-	OptionalAggregateSignature *bls.Sign
-}
-
-// TODO(Edgar) Finish refactoring other three message constructions folded into this function.
-func (consensus *Consensus) construct(p quorum.Phase) *LeaderNetworkMessage {
-
-	msgType := msg_pb.MessageType_ANNOUNCE
-
-	switch p {
-	case quorum.Commit:
-		msgType = msg_pb.MessageType_COMMITTED
-	case quorum.Prepare:
-		msgType = msg_pb.MessageType_PREPARED
-	}
-
-	message := &msg_pb.Message{
-		ServiceType: msg_pb.ServiceType_CONSENSUS,
-		Type:        msgType,
-		Request: &msg_pb.Message_Consensus{
-			Consensus: &msg_pb.ConsensusRequest{},
-		},
-	}
-
-	consensusMsg := message.GetConsensus()
-	consensus.populateMessageFields(consensusMsg)
-	consensusMsg.Payload = consensus.blockHeader
-
-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
-	if err != nil {
-		utils.Logger().Info().
-			Str("phase", p.String()).
-			Str("reason", err.Error()).
-			Msg("Failed to sign and marshal consensus message")
-	}
-
-	return &LeaderNetworkMessage{
-		Phase:                      p,
-		Bytes:                      proto.ConstructConsensusMessage(marshaledMessage),
-		OptionalAggregateSignature: nil,
-	}
-}
-
-// Constructs the announce message
-func (consensus *Consensus) constructAnnounceMessage() []byte {
-	message := &msg_pb.Message{
-		ServiceType: msg_pb.ServiceType_CONSENSUS,
-		Type:        msg_pb.MessageType_ANNOUNCE,
-		Request: &msg_pb.Message_Consensus{
-			Consensus: &msg_pb.ConsensusRequest{},
-		},
-	}
-	consensusMsg := message.GetConsensus()
-	consensus.populateMessageFields(consensusMsg)
-	consensusMsg.Payload = consensus.blockHeader
-
-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
-	if err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to sign and marshal the Announce message")
-	}
-	return proto.ConstructConsensusMessage(marshaledMessage)
-}
-
-// Construct the prepared message, returning prepared message in bytes.
-func (consensus *Consensus) constructPreparedMessage() ([]byte, *bls.Sign) {
-	message := &msg_pb.Message{
-		ServiceType: msg_pb.ServiceType_CONSENSUS,
-		Type:        msg_pb.MessageType_PREPARED,
-		Request: &msg_pb.Message_Consensus{
-			Consensus: &msg_pb.ConsensusRequest{},
-		},
-	}
-
-	consensusMsg := message.GetConsensus()
-	consensus.populateMessageFields(consensusMsg)
-	// add block content in prepared message for slow validators to catchup
-	consensusMsg.Block = consensus.block
-
-	// Payload
-	buffer := bytes.Buffer{}
-
-	// 96 bytes aggregated signature
-	aggSig := bls_cosi.AggregateSig(consensus.Decider.ReadAllSignatures(quorum.Prepare))
-	// TODO(Edgar) Finish refactoring with this API
-	// aggSig := consensus.Decider.AggregateVotes(quorum.Announce)
-	buffer.Write(aggSig.Serialize())
-
-	// Bitmap
-	buffer.Write(consensus.prepareBitmap.Bitmap)
-
-	consensusMsg.Payload = buffer.Bytes()
-	// END Payload
-
-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
-	if err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to sign and marshal the Prepared message")
-	}
-	return proto.ConstructConsensusMessage(marshaledMessage), aggSig
-}
-
-// Construct the committed message, returning committed message in bytes.
-func (consensus *Consensus) constructCommittedMessage() ([]byte, *bls.Sign) {
-	message := &msg_pb.Message{
-		ServiceType: msg_pb.ServiceType_CONSENSUS,
-		Type:        msg_pb.MessageType_COMMITTED,
-		Request: &msg_pb.Message_Consensus{
-			Consensus: &msg_pb.ConsensusRequest{},
-		},
-	}
-
-	consensusMsg := message.GetConsensus()
-	consensus.populateMessageFields(consensusMsg)
-
-	//// Payload
-	buffer := bytes.Buffer{}
-
-	// 96 bytes aggregated signature
-	aggSig := bls_cosi.AggregateSig(consensus.Decider.ReadAllSignatures(quorum.Commit))
-	buffer.Write(aggSig.Serialize())
-
-	// Bitmap
-	buffer.Write(consensus.commitBitmap.Bitmap)
-
-	consensusMsg.Payload = buffer.Bytes()
-	// END Payload
-
-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
-	if err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to sign and marshal the Committed message")
-	}
-	return proto.ConstructConsensusMessage(marshaledMessage), aggSig
-}

consensus/consensus_service.go

@@ -2,7 +2,6 @@ package consensus
 
 import (
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"math/big"
 	"time"
@@ -25,9 +24,14 @@ import (
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/shard/committee"
 	libp2p_peer "github.com/libp2p/go-libp2p-peer"
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 )
 
+var (
+	errValidNotInCommittee = errors.New("slot signer not this slot's subcommittee")
+)
+
 // WaitForNewRandomness listens to the RndChannel to receive new VDF randomness.
 func (consensus *Consensus) WaitForNewRandomness() {
 	go func() {
@@ -262,7 +266,7 @@ func (consensus *Consensus) verifySenderKey(msg *msg_pb.Message) (*bls.PublicKey
 	}
 
 	if !consensus.IsValidatorInCommittee(senderKey) {
-		return nil, fmt.Errorf("Validator %s is not in committee", senderKey.SerializeToHexStr())
+		return nil, errValidNotInCommittee
 	}
 	return senderKey, nil
 }
@@ -275,7 +279,7 @@ func (consensus *Consensus) verifyViewChangeSenderKey(msg *msg_pb.Message) (*bls
 	}
 
 	if !consensus.IsValidatorInCommittee(senderKey) {
-		return nil, fmt.Errorf("Validator %s is not in committee", senderKey.SerializeToHexStr())
+		return nil, errValidNotInCommittee
 	}
 	return senderKey, nil
 }

consensus/consensus_v2.go

@@ -2,21 +2,15 @@ package consensus
 
 import (
 	"bytes"
-	"encoding/binary"
 	"encoding/hex"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/rlp"
 	protobuf "github.com/golang/protobuf/proto"
-	"github.com/harmony-one/bls/ffi/go/bls"
-	"github.com/harmony-one/harmony/api/proto"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/block"
 	"github.com/harmony-one/harmony/consensus/quorum"
 	"github.com/harmony-one/harmony/core/types"
 	vrf_bls "github.com/harmony-one/harmony/crypto/vrf/bls"
-	"github.com/harmony-one/harmony/internal/chain"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/ctxerror"
 	"github.com/harmony-one/harmony/p2p/host"
@@ -24,14 +18,13 @@ import (
 	"github.com/harmony-one/vdf/src/vdf_go"
 )
 
-// handleMessageUpdate will update the consensus state according to received message
+// handlemessageupdate will update the consensus state according to received message
 func (consensus *Consensus) handleMessageUpdate(payload []byte) {
 	if len(payload) == 0 {
 		return
 	}
 	msg := &msg_pb.Message{}
-	err := protobuf.Unmarshal(payload, msg)
-	if err != nil {
+	if err := protobuf.Unmarshal(payload, msg); err != nil {
 		consensus.getLogger().Error().Err(err).Msg("Failed to unmarshal message payload.")
 		return
 	}
@@ -41,14 +34,17 @@ func (consensus *Consensus) handleMessageUpdate(payload []byte) {
 	// which are message types specifically for a node acting as leader
 	switch {
 	case (consensus.current.Mode() == ViewChanging) &&
-		(msg.Type == msg_pb.MessageType_PREPARE || msg.Type == msg_pb.MessageType_COMMIT):
+		(msg.Type == msg_pb.MessageType_PREPARE ||
+			msg.Type == msg_pb.MessageType_COMMIT):
 		return
 	case consensus.current.Mode() == Listening:
 		return
 	}
 
-	if msg.Type == msg_pb.MessageType_VIEWCHANGE || msg.Type == msg_pb.MessageType_NEWVIEW {
-		if msg.GetViewchange() != nil && msg.GetViewchange().ShardId != consensus.ShardID {
+	if msg.Type == msg_pb.MessageType_VIEWCHANGE ||
+		msg.Type == msg_pb.MessageType_NEWVIEW {
+		if msg.GetViewchange() != nil &&
+			msg.GetViewchange().ShardId != consensus.ShardID {
 			consensus.getLogger().Warn().
 				Uint32("myShardId", consensus.ShardID).
 				Uint32("receivedShardId", msg.GetViewchange().ShardId).
@@ -56,7 +52,8 @@ func (consensus *Consensus) handleMessageUpdate(payload []byte) {
 			return
 		}
 	} else {
-		if msg.GetConsensus() != nil && msg.GetConsensus().ShardId != consensus.ShardID {
+		if msg.GetConsensus() != nil &&
+			msg.GetConsensus().ShardId != consensus.ShardID {
 			consensus.getLogger().Warn().
 				Uint32("myShardId", consensus.ShardID).
 				Uint32("receivedShardId", msg.GetConsensus().ShardId).
@@ -73,745 +70,58 @@ func (consensus *Consensus) handleMessageUpdate(payload []byte) {
 		return
 	}
 
-	switch msg.Type {
-	case msg_pb.MessageType_ANNOUNCE:
+	intendedForValidator, intendedForLeader :=
+		!(consensus.IsLeader() && consensus.current.Mode() == Normal),
+		consensus.IsLeader()
+
+	switch t := msg.Type; true {
+	// Handle validator intended messages first
+	case t == msg_pb.MessageType_ANNOUNCE &&
+		intendedForValidator &&
+		consensus.validatorSanityChecks(msg):
 		consensus.onAnnounce(msg)
-	case msg_pb.MessageType_PREPARE:
-		consensus.onPrepare(msg)
-	case msg_pb.MessageType_PREPARED:
+	case t == msg_pb.MessageType_PREPARED &&
+		intendedForValidator &&
+		consensus.validatorSanityChecks(msg):
 		consensus.onPrepared(msg)
-	case msg_pb.MessageType_COMMIT:
-		consensus.onCommit(msg)
-	case msg_pb.MessageType_COMMITTED:
+	case t == msg_pb.MessageType_COMMITTED &&
+		intendedForValidator &&
+		consensus.validatorSanityChecks(msg):
 		consensus.onCommitted(msg)
-	case msg_pb.MessageType_VIEWCHANGE:
+	// Handle leader intended messages now
+	case t == msg_pb.MessageType_PREPARE &&
+		intendedForLeader &&
+		consensus.leaderSanityChecks(msg):
+		consensus.onPrepare(msg)
+	case t == msg_pb.MessageType_COMMIT &&
+		intendedForLeader &&
+		consensus.leaderSanityChecks(msg):
+		consensus.onCommit(msg)
+	case t == msg_pb.MessageType_VIEWCHANGE:
 		consensus.onViewChange(msg)
-	case msg_pb.MessageType_NEWVIEW:
+	case t == msg_pb.MessageType_NEWVIEW:
 		consensus.onNewView(msg)
 	}
 }
 
-// TODO: move to consensus_leader.go later
-func (consensus *Consensus) announce(block *types.Block) {
-	blockHash := block.Hash()
-	copy(consensus.blockHash[:], blockHash[:])
-
-	// prepare message and broadcast to validators
-	encodedBlock, err := rlp.EncodeToBytes(block)
-	if err != nil {
-		consensus.getLogger().Debug().Msg("[Announce] Failed encoding block")
-		return
-	}
-	encodedBlockHeader, err := rlp.EncodeToBytes(block.Header())
-	if err != nil {
-		consensus.getLogger().Debug().Msg("[Announce] Failed encoding block header")
-		return
-	}
-
-	consensus.block = encodedBlock
-	consensus.blockHeader = encodedBlockHeader
-	msgToSend := consensus.constructAnnounceMessage()
-	// TODO Finish using this refactored way
-	// msgToSend := consensus.construct(quorum.Announce)
-
-	// save announce message to FBFTLog
-	// msgPayload, _ := proto.GetConsensusMessagePayload(msgToSend.Bytes)
-	msgPayload, _ := proto.GetConsensusMessagePayload(msgToSend)
-
-	// TODO(chao): don't unmarshall the message here and direclty pass the original object.
-	msg := &msg_pb.Message{}
-	_ = protobuf.Unmarshal(msgPayload, msg)
-	FPBTMsg, err := ParseFBFTMessage(msg)
-	if err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[Announce] Unable to parse FPBT message")
-		return
-	}
-
-	// TODO(chao): review FPBT log data structure
-	consensus.FBFTLog.AddMessage(FPBTMsg)
-	consensus.getLogger().Debug().
-		Str("MsgBlockHash", FPBTMsg.BlockHash.Hex()).
-		Uint64("MsgViewID", FPBTMsg.ViewID).
-		Uint64("MsgBlockNum", FPBTMsg.BlockNum).
-		Msg("[Announce] Added Announce message in FPBT")
-	consensus.FBFTLog.AddBlock(block)
-
-	// Leader sign the block hash itself
-	consensus.Decider.AddSignature(
-		quorum.Prepare,
-		consensus.PubKey,
-		consensus.priKey.SignHash(consensus.blockHash[:]),
-		consensus.LeaderPubKey,
-		consensus.blockNum,
-	)
-	if err := consensus.prepareBitmap.SetKey(consensus.PubKey, true); err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[Announce] Leader prepareBitmap SetKey failed")
-		return
-	}
-
-	// Construct broadcast p2p message
-	if err := consensus.msgSender.SendWithRetry(
-		consensus.blockNum, msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{
-			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)),
-		}, host.ConstructP2pMessage(byte(17), msgToSend)); err != nil {
-		consensus.getLogger().Warn().
-			Str("groupID", string(nodeconfig.NewGroupIDByShardID(
-				nodeconfig.ShardID(consensus.ShardID),
-			))).
-			Msg("[Announce] Cannot send announce message")
-	} else {
-		consensus.getLogger().Info().
-			Str("blockHash", block.Hash().Hex()).
-			Uint64("blockNum", block.NumberU64()).
-			Msg("[Announce] Sent Announce Message!!")
-	}
-
-	consensus.getLogger().Debug().
-		Str("From", consensus.phase.String()).
-		Str("To", FBFTPrepare.String()).
-		Msg("[Announce] Switching phase")
-	consensus.switchPhase(FBFTPrepare, true)
-}
-
-func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) {
-	consensus.getLogger().Debug().Msg("[OnAnnounce] Receive announce message")
-	if consensus.IsLeader() && consensus.current.Mode() == Normal {
-		return
-	}
-
-	senderKey, err := consensus.verifySenderKey(msg)
-	if err != nil {
-		consensus.getLogger().Error().Err(err).Msg("[OnAnnounce] VerifySenderKey failed")
-		return
-	}
-	if !senderKey.IsEqual(consensus.LeaderPubKey) &&
-		consensus.current.Mode() == Normal && !consensus.ignoreViewIDCheck {
-		consensus.getLogger().Warn().
-			Str("senderKey", senderKey.SerializeToHexStr()).
-			Str("leaderKey", consensus.LeaderPubKey.SerializeToHexStr()).
-			Msg("[OnAnnounce] SenderKey does not match leader PubKey")
-		return
-	}
-	if err = verifyMessageSig(senderKey, msg); err != nil {
-		consensus.getLogger().Error().Err(err).Msg("[OnAnnounce] Failed to verify leader signature")
-		return
-	}
-
-	recvMsg, err := ParseFBFTMessage(msg)
-	if err != nil {
-		consensus.getLogger().Error().
-			Err(err).
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Msg("[OnAnnounce] Unparseable leader message")
-		return
-	}
-
-	// verify validity of block header object
-	// TODO: think about just sending the block hash instead of the header.
-	encodedHeader := recvMsg.Payload
-	header := new(block.Header)
-	err = rlp.DecodeBytes(encodedHeader, header)
-	if err != nil {
-		consensus.getLogger().Warn().
-			Err(err).
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Msg("[OnAnnounce] Unparseable block header data")
-		return
-	}
-
-	if recvMsg.BlockNum < consensus.blockNum || recvMsg.BlockNum != header.Number().Uint64() {
-		consensus.getLogger().Debug().
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("hdrBlockNum", header.Number().Uint64()).
-			Uint64("consensuBlockNum", consensus.blockNum).
-			Msg("[OnAnnounce] BlockNum does not match")
-		return
-	}
-	if consensus.current.Mode() == Normal {
-		if err = chain.Engine.VerifyHeader(consensus.ChainReader, header, true); err != nil {
-			consensus.getLogger().Warn().
-				Err(err).
-				Str("inChain", consensus.ChainReader.CurrentHeader().Number().String()).
-				Str("MsgBlockNum", header.Number().String()).
-				Msg("[OnAnnounce] Block content is not verified successfully")
-			return
-		}
-
-		//VRF/VDF is only generated in the beach chain
-		if consensus.NeedsRandomNumberGeneration(header.Epoch()) {
-			//validate the VRF with proof if a non zero VRF is found in header
-			if len(header.Vrf()) > 0 {
-				if !consensus.ValidateVrfAndProof(header) {
-					return
-				}
-			}
-
-			//validate the VDF with proof if a non zero VDF is found in header
-			if len(header.Vdf()) > 0 {
-				if !consensus.ValidateVdfAndProof(header) {
-					return
-				}
-			}
-		}
-	}
-
-	logMsgs := consensus.FBFTLog.GetMessagesByTypeSeqView(
-		msg_pb.MessageType_ANNOUNCE, recvMsg.BlockNum, recvMsg.ViewID,
-	)
-	if len(logMsgs) > 0 {
-		if logMsgs[0].BlockHash != recvMsg.BlockHash &&
-			logMsgs[0].SenderPubkey.IsEqual(recvMsg.SenderPubkey) {
-			consensus.getLogger().Debug().
-				Str("logMsgSenderKey", logMsgs[0].SenderPubkey.SerializeToHexStr()).
-				Str("logMsgBlockHash", logMsgs[0].BlockHash.Hex()).
-				Str("recvMsg.SenderPubkey", recvMsg.SenderPubkey.SerializeToHexStr()).
-				Uint64("recvMsg.BlockNum", recvMsg.BlockNum).
-				Uint64("recvMsg.ViewID", recvMsg.ViewID).
-				Str("recvMsgBlockHash", recvMsg.BlockHash.Hex()).
-				Str("LeaderKey", consensus.LeaderPubKey.SerializeToHexStr()).
-				Msg("[OnAnnounce] Leader is malicious")
-			if consensus.current.Mode() == ViewChanging {
-				viewID := consensus.current.ViewID()
-				consensus.startViewChange(viewID + 1)
-			} else {
-				consensus.startViewChange(consensus.viewID + 1)
-			}
-		}
-		consensus.getLogger().Debug().
-			Str("leaderKey", consensus.LeaderPubKey.SerializeToHexStr()).
-			Msg("[OnAnnounce] Announce message received again")
-		//return
-	}
-
-	consensus.getLogger().Debug().
-		Uint64("MsgViewID", recvMsg.ViewID).
-		Uint64("MsgBlockNum", recvMsg.BlockNum).
-		Msg("[OnAnnounce] Announce message Added")
-	consensus.FBFTLog.AddMessage(recvMsg)
-
-	consensus.mutex.Lock()
-	defer consensus.mutex.Unlock()
-
-	consensus.blockHash = recvMsg.BlockHash
-
-	// we have already added message and block, skip check viewID and send prepare message if is in ViewChanging mode
-	if consensus.current.Mode() == ViewChanging {
-		consensus.getLogger().Debug().Msg("[OnAnnounce] Still in ViewChanging Mode, Exiting !!")
-		return
-	}
-
-	if consensus.checkViewID(recvMsg) != nil {
-		if consensus.current.Mode() == Normal {
-			consensus.getLogger().Debug().
-				Uint64("MsgViewID", recvMsg.ViewID).
-				Uint64("MsgBlockNum", recvMsg.BlockNum).
-				Msg("[OnAnnounce] ViewID check failed")
-		}
-		return
-	}
-	consensus.prepare()
-}
-
-// tryPrepare will try to send prepare message
-func (consensus *Consensus) prepare() {
-	// Construct and send prepare message
-	msgToSend := consensus.constructPrepareMessage()
-	// TODO: this will not return immediatey, may block
-
-	if err := consensus.msgSender.SendWithoutRetry([]nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, host.ConstructP2pMessage(byte(17), msgToSend)); err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[OnAnnounce] Cannot send prepare message")
-	} else {
-		consensus.getLogger().Info().
-			Str("blockHash", hex.EncodeToString(consensus.blockHash[:])).
-			Msg("[OnAnnounce] Sent Prepare Message!!")
-	}
-	consensus.getLogger().Debug().
-		Str("From", consensus.phase.String()).
-		Str("To", FBFTPrepare.String()).
-		Msg("[Announce] Switching Phase")
-	consensus.switchPhase(FBFTPrepare, true)
-}
-
-// TODO: move to consensus_leader.go later
-func (consensus *Consensus) onPrepare(msg *msg_pb.Message) {
-	if !consensus.IsLeader() {
-		return
-	}
-
-	senderKey, err := consensus.verifySenderKey(msg)
-	if err != nil {
-		consensus.getLogger().Error().Err(err).Msg("[OnPrepare] VerifySenderKey failed")
-		return
-	}
-	if err = verifyMessageSig(senderKey, msg); err != nil {
-		consensus.getLogger().Error().Err(err).Msg("[OnPrepare] Failed to verify sender's signature")
-		return
-	}
-
-	recvMsg, err := ParseFBFTMessage(msg)
-	if err != nil {
-		consensus.getLogger().Error().Err(err).Msg("[OnPrepare] Unparseable validator message")
-		return
-	}
-
-	if recvMsg.ViewID != consensus.viewID || recvMsg.BlockNum != consensus.blockNum {
-		consensus.getLogger().Debug().
-			Uint64("MsgViewID", recvMsg.ViewID).
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("blockNum", consensus.blockNum).
-			Msg("[OnPrepare] Message ViewId or BlockNum not match")
-		return
-	}
-
-	if !consensus.FBFTLog.HasMatchingViewAnnounce(
-		consensus.blockNum, consensus.viewID, recvMsg.BlockHash,
-	) {
-		consensus.getLogger().Debug().
-			Uint64("MsgViewID", recvMsg.ViewID).
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("blockNum", consensus.blockNum).
-			Msg("[OnPrepare] No Matching Announce message")
-		//return
-	}
-
-	validatorPubKey := recvMsg.SenderPubkey
-	prepareSig := recvMsg.Payload
-	prepareBitmap := consensus.prepareBitmap
-
-	consensus.mutex.Lock()
-	defer consensus.mutex.Unlock()
-	logger := consensus.getLogger().With().
-		Str("validatorPubKey", validatorPubKey.SerializeToHexStr()).Logger()
-
-	// proceed only when the message is not received before
-	signed := consensus.Decider.ReadSignature(quorum.Prepare, validatorPubKey)
-	if signed != nil {
-		logger.Debug().
-			Msg("[OnPrepare] Already Received prepare message from the validator")
-		return
-	}
-
-	if consensus.Decider.IsQuorumAchieved(quorum.Prepare) {
-		// already have enough signatures
-		logger.Debug().Msg("[OnPrepare] Received Additional Prepare Message")
-		return
-	}
-
-	// Check BLS signature for the multi-sig
-	var sign bls.Sign
-	err = sign.Deserialize(prepareSig)
-	if err != nil {
-		consensus.getLogger().Error().Err(err).
-			Msg("[OnPrepare] Failed to deserialize bls signature")
-		return
-	}
-	if !sign.VerifyHash(recvMsg.SenderPubkey, consensus.blockHash[:]) {
-		consensus.getLogger().Error().Msg("[OnPrepare] Received invalid BLS signature")
-		return
-	}
-
-	logger = logger.With().
-		Int64("NumReceivedSoFar", consensus.Decider.SignersCount(quorum.Prepare)).
-		Int64("PublicKeys", consensus.Decider.ParticipantsCount()).Logger()
-	logger.Info().Msg("[OnPrepare] Received New Prepare Signature")
-	consensus.Decider.AddSignature(
-		quorum.Prepare, validatorPubKey, &sign, consensus.LeaderPubKey, consensus.blockNum,
-	)
-	// Set the bitmap indicating that this validator signed.
-	if err := prepareBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[OnPrepare] prepareBitmap.SetKey failed")
-		return
-	}
-
-	if consensus.Decider.IsQuorumAchieved(quorum.Prepare) {
-		logger.Debug().Msg("[OnPrepare] Received Enough Prepare Signatures")
-		// Construct and broadcast prepared message
-		msgToSend, aggSig := consensus.constructPreparedMessage()
-		consensus.aggregatedPrepareSig = aggSig
-
-		//leader adds prepared message to log
-		// TODO(chao): don't unmarshall the payload again
-		msgPayload, _ := proto.GetConsensusMessagePayload(msgToSend)
-		msg := &msg_pb.Message{}
-		_ = protobuf.Unmarshal(msgPayload, msg)
-		FBFTMsg, err := ParseFBFTMessage(msg)
-
-		if err != nil {
-			consensus.getLogger().Warn().Err(err).Msg("[OnPrepare] Unable to parse pbft message")
-			return
-		}
-
-		consensus.FBFTLog.AddMessage(FBFTMsg)
-		// Leader add commit phase signature
-		blockNumHash := make([]byte, 8)
-		binary.LittleEndian.PutUint64(blockNumHash, consensus.blockNum)
-		commitPayload := append(blockNumHash, consensus.blockHash[:]...)
-		consensus.Decider.AddSignature(
-			quorum.Commit,
-			consensus.PubKey,
-			consensus.priKey.SignHash(commitPayload),
-			consensus.LeaderPubKey,
-			consensus.blockNum,
-		)
-
-		if err := consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil {
-			consensus.getLogger().Debug().Msg("[OnPrepare] Leader commit bitmap set failed")
-			return
-		}
-
-		if err := consensus.msgSender.SendWithRetry(
-			consensus.blockNum,
-			msg_pb.MessageType_PREPARED, []nodeconfig.GroupID{
-				nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)),
-			},
-			host.ConstructP2pMessage(byte(17), msgToSend),
-		); err != nil {
-			consensus.getLogger().Warn().Msg("[OnPrepare] Cannot send prepared message")
-		} else {
-			consensus.getLogger().Debug().
-				Hex("blockHash", consensus.blockHash[:]).
-				Uint64("blockNum", consensus.blockNum).
-				Msg("[OnPrepare] Sent Prepared Message!!")
-		}
-		consensus.msgSender.StopRetry(msg_pb.MessageType_ANNOUNCE)
-		// Stop retry committed msg of last consensus
-		consensus.msgSender.StopRetry(msg_pb.MessageType_COMMITTED)
-
-		consensus.getLogger().Debug().
-			Str("From", consensus.phase.String()).
-			Str("To", FBFTCommit.String()).
-			Msg("[OnPrepare] Switching phase")
-		consensus.switchPhase(FBFTCommit, true)
-	}
-}
-
-func (consensus *Consensus) onPrepared(msg *msg_pb.Message) {
-	consensus.getLogger().Debug().Msg("[OnPrepared] Received Prepared message")
-	if consensus.IsLeader() && consensus.current.Mode() == Normal {
-		return
-	}
-
-	senderKey, err := consensus.verifySenderKey(msg)
-	if err != nil {
-		consensus.getLogger().Debug().Err(err).Msg("[OnPrepared] VerifySenderKey failed")
-		return
-	}
-	if !senderKey.IsEqual(consensus.LeaderPubKey) &&
-		consensus.current.Mode() == Normal && !consensus.ignoreViewIDCheck {
-		consensus.getLogger().Warn().Msg("[OnPrepared] SenderKey not match leader PubKey")
-		return
-	}
-	if err := verifyMessageSig(senderKey, msg); err != nil {
-		consensus.getLogger().Debug().Err(err).Msg("[OnPrepared] Failed to verify sender's signature")
-		return
-	}
-
-	recvMsg, err := ParseFBFTMessage(msg)
-	if err != nil {
-		consensus.getLogger().Debug().Err(err).Msg("[OnPrepared] Unparseable validator message")
-		return
-	}
-	consensus.getLogger().Info().
-		Uint64("MsgBlockNum", recvMsg.BlockNum).
-		Uint64("MsgViewID", recvMsg.ViewID).
-		Msg("[OnPrepared] Received prepared message")
-
-	if recvMsg.BlockNum < consensus.blockNum {
-		consensus.getLogger().Debug().Uint64("MsgBlockNum", recvMsg.BlockNum).Msg("Old Block Received, ignoring!!")
-		return
-	}
-
-	// check validity of prepared signature
-	blockHash := recvMsg.BlockHash
-	aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 0)
-	if err != nil {
-		consensus.getLogger().Error().Err(err).Msg("ReadSignatureBitmapPayload failed!!")
-		return
-	}
-
-	if !consensus.Decider.IsQuorumAchievedByMask(mask, true) {
-		consensus.getLogger().Warn().
-			Msgf("[OnPrepared] Quorum Not achieved")
-		return
-	}
-
-	if !aggSig.VerifyHash(mask.AggregatePublic, blockHash[:]) {
-		myBlockHash := common.Hash{}
-		myBlockHash.SetBytes(consensus.blockHash[:])
-		consensus.getLogger().Warn().
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("MsgViewID", recvMsg.ViewID).
-			Msg("[OnPrepared] failed to verify multi signature for prepare phase")
-		return
-	}
-
-	// check validity of block
-	block := recvMsg.Block
-	var blockObj types.Block
-	err = rlp.DecodeBytes(block, &blockObj)
-	if err != nil {
-		consensus.getLogger().Warn().
-			Err(err).
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Msg("[OnPrepared] Unparseable block header data")
-		return
-	}
-	if blockObj.NumberU64() != recvMsg.BlockNum || recvMsg.BlockNum < consensus.blockNum {
-		consensus.getLogger().Warn().
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("blockNum", blockObj.NumberU64()).
-			Msg("[OnPrepared] BlockNum not match")
-		return
-	}
-	if blockObj.Header().Hash() != recvMsg.BlockHash {
-		consensus.getLogger().Warn().
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Hex("MsgBlockHash", recvMsg.BlockHash[:]).
-			Str("blockObjHash", blockObj.Header().Hash().Hex()).
-			Msg("[OnPrepared] BlockHash not match")
-		return
-	}
-	if consensus.current.Mode() == Normal {
-		err := chain.Engine.VerifyHeader(consensus.ChainReader, blockObj.Header(), true)
-		if err != nil {
-			consensus.getLogger().Error().
-				Err(err).
-				Str("inChain", consensus.ChainReader.CurrentHeader().Number().String()).
-				Str("MsgBlockNum", blockObj.Header().Number().String()).
-				Msg("[OnPrepared] Block header is not verified successfully")
-			return
-		}
-		if consensus.BlockVerifier == nil {
-			// do nothing
-		} else if err := consensus.BlockVerifier(&blockObj); err != nil {
-			consensus.getLogger().Error().Err(err).Msg("[OnPrepared] Block verification failed")
-			return
-		}
-	}
-
-	consensus.FBFTLog.AddBlock(&blockObj)
-	recvMsg.Block = []byte{} // save memory space
-	consensus.FBFTLog.AddMessage(recvMsg)
-	consensus.getLogger().Debug().
-		Uint64("MsgViewID", recvMsg.ViewID).
-		Uint64("MsgBlockNum", recvMsg.BlockNum).
-		Hex("blockHash", recvMsg.BlockHash[:]).
-		Msg("[OnPrepared] Prepared message and block added")
-
-	consensus.mutex.Lock()
-	defer consensus.mutex.Unlock()
-
-	consensus.tryCatchup()
-	if consensus.current.Mode() == ViewChanging {
-		consensus.getLogger().Debug().Msg("[OnPrepared] Still in ViewChanging mode, Exiting!!")
-		return
-	}
-
-	if consensus.checkViewID(recvMsg) != nil {
-		if consensus.current.Mode() == Normal {
-			consensus.getLogger().Debug().
-				Uint64("MsgViewID", recvMsg.ViewID).
-				Uint64("MsgBlockNum", recvMsg.BlockNum).
-				Msg("[OnPrepared] ViewID check failed")
-		}
-		return
-	}
-	if recvMsg.BlockNum > consensus.blockNum {
-		consensus.getLogger().Debug().
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("blockNum", consensus.blockNum).
-			Msg("[OnPrepared] Future Block Received, ignoring!!")
-		return
-	}
-
-	// add block field
-	blockPayload := make([]byte, len(block))
-	copy(blockPayload[:], block[:])
-	consensus.block = blockPayload
-
-	// add preparedSig field
-	consensus.aggregatedPrepareSig = aggSig
-	consensus.prepareBitmap = mask
-
-	// Optimistically add blockhash field of prepare message
-	emptyHash := [32]byte{}
-	if bytes.Compare(consensus.blockHash[:], emptyHash[:]) == 0 {
-		copy(consensus.blockHash[:], blockHash[:])
-	}
-
-	// Construct and send the commit message
-	// TODO: should only sign on block hash
-	blockNumBytes := make([]byte, 8)
-	binary.LittleEndian.PutUint64(blockNumBytes, consensus.blockNum)
-	commitPayload := append(blockNumBytes, consensus.blockHash[:]...)
-	msgToSend := consensus.constructCommitMessage(commitPayload)
-
-	// TODO: genesis account node delay for 1 second, this is a temp fix for allows FN nodes to earning reward
-	if consensus.delayCommit > 0 {
-		time.Sleep(consensus.delayCommit)
-	}
-
-	if err := consensus.msgSender.SendWithoutRetry([]nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, host.ConstructP2pMessage(byte(17), msgToSend)); err != nil {
-		consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!")
-	} else {
-		consensus.getLogger().Info().
-			Uint64("blockNum", consensus.blockNum).
-			Hex("blockHash", consensus.blockHash[:]).
-			Msg("[OnPrepared] Sent Commit Message!!")
-	}
-
-	consensus.getLogger().Debug().
-		Str("From", consensus.phase.String()).
-		Str("To", FBFTCommit.String()).
-		Msg("[OnPrepared] Switching phase")
-	consensus.switchPhase(FBFTCommit, true)
-
-	return
-}
-
-// TODO: move it to consensus_leader.go later
-func (consensus *Consensus) onCommit(msg *msg_pb.Message) {
-	if !consensus.IsLeader() {
-		return
-	}
-
-	senderKey, err := consensus.verifySenderKey(msg)
-	if err != nil {
-		consensus.getLogger().Debug().Msgf("[OnCommit] VerifySenderKey Failed %s", err.Error())
-		return
-	}
-	if err = verifyMessageSig(senderKey, msg); err != nil {
-		consensus.getLogger().Debug().Err(err).Msg("[OnCommit] Failed to verify sender's signature")
-		return
-	}
-
-	recvMsg, err := ParseFBFTMessage(msg)
-	if err != nil {
-		consensus.getLogger().Debug().Err(err).Msg("[OnCommit] Parse pbft message failed")
-		return
-	}
-
-	if recvMsg.ViewID != consensus.viewID || recvMsg.BlockNum != consensus.blockNum {
-		consensus.getLogger().Debug().
-			Uint64("MsgViewID", recvMsg.ViewID).
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("blockNum", consensus.blockNum).
-			Str("ValidatorPubKey", recvMsg.SenderPubkey.SerializeToHexStr()).
-			Msg("[OnCommit] BlockNum/viewID not match")
-		return
-	}
-
-	if !consensus.FBFTLog.HasMatchingAnnounce(consensus.blockNum, recvMsg.BlockHash) {
-		consensus.getLogger().Debug().
-			Hex("MsgBlockHash", recvMsg.BlockHash[:]).
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("blockNum", consensus.blockNum).
-			Msg("[OnCommit] Cannot find matching blockhash")
-		return
-	}
-
-	if !consensus.FBFTLog.HasMatchingPrepared(consensus.blockNum, recvMsg.BlockHash) {
-		consensus.getLogger().Debug().
-			Hex("blockHash", recvMsg.BlockHash[:]).
-			Uint64("blockNum", consensus.blockNum).
-			Msg("[OnCommit] Cannot find matching prepared message")
-		return
-	}
-
-	validatorPubKey := recvMsg.SenderPubkey
-	commitSig := recvMsg.Payload
-	consensus.mutex.Lock()
-	defer consensus.mutex.Unlock()
-
-	logger := consensus.getLogger().With().
-		Str("validatorPubKey", validatorPubKey.SerializeToHexStr()).Logger()
-	if !consensus.IsValidatorInCommittee(recvMsg.SenderPubkey) {
-		logger.Error().Msg("[OnCommit] Invalid validator")
-		return
-	}
-
-	commitBitmap := consensus.commitBitmap
-	// proceed only when the message is not received before
-	signed := consensus.Decider.ReadSignature(quorum.Commit, validatorPubKey)
-
-	if signed != nil {
-		logger.Debug().
-			Msg("[OnCommit] Already received commit message from the validator")
-		return
-	}
-
-	// has to be called before verifying signature
-	quorumWasMet := consensus.Decider.IsQuorumAchieved(quorum.Commit)
-	// Verify the signature on commitPayload is correct
-	var sign bls.Sign
-	err = sign.Deserialize(commitSig)
-	if err != nil {
-		logger.Debug().Msg("[OnCommit] Failed to deserialize bls signature")
-		return
-	}
-	blockNumHash := make([]byte, 8)
-	binary.LittleEndian.PutUint64(blockNumHash, recvMsg.BlockNum)
-	commitPayload := append(blockNumHash, recvMsg.BlockHash[:]...)
-	logger = logger.With().Uint64("MsgViewID", recvMsg.ViewID).Uint64("MsgBlockNum", recvMsg.BlockNum).Logger()
-	if !sign.VerifyHash(recvMsg.SenderPubkey, commitPayload) {
-		logger.Error().Msg("[OnCommit] Cannot verify commit message")
-		return
-	}
-
-	logger = logger.With().
-		Int64("numReceivedSoFar", consensus.Decider.SignersCount(quorum.Commit)).
-		Logger()
-	logger.Info().Msg("[OnCommit] Received new commit message")
-	consensus.Decider.AddSignature(
-		quorum.Commit, validatorPubKey, &sign, consensus.LeaderPubKey, consensus.blockNum,
-	)
-	// Set the bitmap indicating that this validator signed.
-	if err := commitBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[OnCommit] commitBitmap.SetKey failed")
-		return
-	}
-
-	quorumIsMet := consensus.Decider.IsQuorumAchieved(quorum.Commit)
-	if !quorumWasMet && quorumIsMet {
-		logger.Info().Msg("[OnCommit] 2/3 Enough commits received")
-		go func(viewID uint64) {
-			time.Sleep(2 * time.Second)
-			logger.Debug().Msg("[OnCommit] Commit Grace Period Ended")
-			consensus.commitFinishChan <- viewID
-		}(consensus.viewID)
-
-		consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED)
-	}
-
-	if consensus.Decider.IsRewardThresholdAchieved() {
-		go func(viewID uint64) {
-			consensus.commitFinishChan <- viewID
-			logger.Info().Msg("[OnCommit] 90% Enough commits received")
-		}(consensus.viewID)
-	}
-}
-
 func (consensus *Consensus) finalizeCommits() {
 	consensus.getLogger().Info().
 		Int64("NumCommits", consensus.Decider.SignersCount(quorum.Commit)).
 		Msg("[Finalizing] Finalizing Block")
-
 	beforeCatchupNum := consensus.blockNum
 	// Construct committed message
-	msgToSend, aggSig := consensus.constructCommittedMessage()
-	consensus.aggregatedCommitSig = aggSig // this may not needed
-
-	// leader adds committed message to log
-	msgPayload, _ := proto.GetConsensusMessagePayload(msgToSend)
-	msg := &msg_pb.Message{}
-	_ = protobuf.Unmarshal(msgPayload, msg)
-	pbftMsg, err := ParseFBFTMessage(msg)
+	network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil)
 	if err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[FinalizeCommits] Unable to parse pbft message")
+		consensus.getLogger().Warn().Err(err).
+			Msg("[FinalizeCommits] Unable to construct Committed message")
 		return
 	}
-	consensus.FBFTLog.AddMessage(pbftMsg)
-
+	msgToSend, aggSig, FBFTMsg :=
+		network.Bytes,
+		network.OptionalAggregateSignature,
+		network.FBFTMsg
+	consensus.aggregatedCommitSig = aggSig // this may not needed
+	consensus.FBFTLog.AddMessage(FBFTMsg)
 	// find correct block content
 	curBlockHash := consensus.blockHash
 	block := consensus.FBFTLog.GetBlockByHash(curBlockHash)
@@ -830,7 +140,7 @@ func (consensus *Consensus) finalizeCommits() {
 		return
 	}
 
-	consensus.ChainReader.WriteLastCommits(pbftMsg.Payload)
+	consensus.ChainReader.WriteLastCommits(FBFTMsg.Payload)
 
 	// if leader success finalize the block, send committed message to validators
 	if err := consensus.msgSender.SendWithRetry(
@@ -876,114 +186,8 @@ func (consensus *Consensus) finalizeCommits() {
 	consensus.ReadySignal <- struct{}{}
 }
 
-func (consensus *Consensus) onCommitted(msg *msg_pb.Message) {
-	consensus.getLogger().Debug().Msg("[OnCommitted] Receive committed message")
-
-	if consensus.IsLeader() && consensus.current.Mode() == Normal {
-		return
-	}
-
-	senderKey, err := consensus.verifySenderKey(msg)
-	if err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[OnCommitted] verifySenderKey failed")
-		return
-	}
-	if !senderKey.IsEqual(consensus.LeaderPubKey) &&
-		consensus.current.Mode() == Normal && !consensus.ignoreViewIDCheck {
-		consensus.getLogger().Warn().Msg("[OnCommitted] senderKey not match leader PubKey")
-		return
-	}
-	if err = verifyMessageSig(senderKey, msg); err != nil {
-		consensus.getLogger().Warn().Err(err).Msg("[OnCommitted] Failed to verify sender's signature")
-		return
-	}
-
-	recvMsg, err := ParseFBFTMessage(msg)
-	if err != nil {
-		consensus.getLogger().Warn().Msg("[OnCommitted] unable to parse msg")
-		return
-	}
-
-	if recvMsg.BlockNum < consensus.blockNum {
-		consensus.getLogger().Info().
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Uint64("blockNum", consensus.blockNum).
-			Msg("[OnCommitted] Received Old Blocks!!")
-		return
-	}
-
-	aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 0)
-	if err != nil {
-		consensus.getLogger().Error().Err(err).Msg("[OnCommitted] readSignatureBitmapPayload failed")
-		return
-	}
-
-	if !consensus.Decider.IsQuorumAchievedByMask(mask, true) {
-		consensus.getLogger().Warn().
-			Msgf("[OnCommitted] Quorum Not achieved")
-		return
-	}
-
-	blockNumBytes := make([]byte, 8)
-	binary.LittleEndian.PutUint64(blockNumBytes, recvMsg.BlockNum)
-	commitPayload := append(blockNumBytes, recvMsg.BlockHash[:]...)
-	if !aggSig.VerifyHash(mask.AggregatePublic, commitPayload) {
-		consensus.getLogger().Error().
-			Uint64("MsgBlockNum", recvMsg.BlockNum).
-			Msg("[OnCommitted] Failed to verify the multi signature for commit phase")
-		return
-	}
-
-	consensus.FBFTLog.AddMessage(recvMsg)
-	consensus.ChainReader.WriteLastCommits(recvMsg.Payload)
-	consensus.getLogger().Debug().
-		Uint64("MsgViewID", recvMsg.ViewID).
-		Uint64("MsgBlockNum", recvMsg.BlockNum).
-		Msg("[OnCommitted] Committed message added")
-
-	consensus.mutex.Lock()
-	defer consensus.mutex.Unlock()
-
-	consensus.aggregatedCommitSig = aggSig
-	consensus.commitBitmap = mask
-
-	if recvMsg.BlockNum-consensus.blockNum > consensusBlockNumBuffer {
-		consensus.getLogger().Debug().Uint64("MsgBlockNum", recvMsg.BlockNum).Msg("[OnCommitted] out of sync")
-		go func() {
-			select {
-			case consensus.blockNumLowChan <- struct{}{}:
-				consensus.current.SetMode(Syncing)
-				for _, v := range consensus.consensusTimeout {
-					v.Stop()
-				}
-			case <-time.After(1 * time.Second):
-			}
-		}()
-		return
-	}
-
-	//	if consensus.checkViewID(recvMsg) != nil {
-	//		consensus.getLogger().Debug("viewID check failed", "viewID", recvMsg.ViewID, "myViewID", consensus.viewID)
-	//		return
-	//	}
-
-	consensus.tryCatchup()
-	if consensus.current.Mode() == ViewChanging {
-		consensus.getLogger().Debug().Msg("[OnCommitted] Still in ViewChanging mode, Exiting!!")
-		return
-	}
-
-	if consensus.consensusTimeout[timeoutBootstrap].IsActive() {
-		consensus.consensusTimeout[timeoutBootstrap].Stop()
-		consensus.getLogger().Debug().Msg("[OnCommitted] Start consensus timer; stop bootstrap timer only once")
-	} else {
-		consensus.getLogger().Debug().Msg("[OnCommitted] Start consensus timer")
-	}
-	consensus.consensusTimeout[timeoutConsensus].Start()
-	return
-}
-
-// LastCommitSig returns the byte array of aggregated commit signature and bitmap of last block
+// LastCommitSig returns the byte array of aggregated
+// commit signature and bitmap of last block
 func (consensus *Consensus) LastCommitSig() ([]byte, []byte, error) {
 	if consensus.blockNum <= 1 {
 		return nil, nil, nil
@@ -1013,9 +217,6 @@ func (consensus *Consensus) LastCommitSig() ([]byte, []byte, error) {
 // try to catch up if fall behind
 func (consensus *Consensus) tryCatchup() {
 	consensus.getLogger().Info().Msg("[TryCatchup] commit new blocks")
-	//	if consensus.phase != Commit && consensus.mode.Mode() == Normal {
-	//		return
-	//	}
 	currentBlockNum := consensus.blockNum
 	for {
 		msgs := consensus.FBFTLog.GetMessagesByTypeSeq(msg_pb.MessageType_COMMITTED, consensus.blockNum)
@@ -1047,7 +248,9 @@ func (consensus *Consensus) tryCatchup() {
 		}
 		consensus.getLogger().Info().Msg("[TryCatchup] block found to commit")
 
-		preparedMsgs := consensus.FBFTLog.GetMessagesByTypeSeqHash(msg_pb.MessageType_PREPARED, msgs[0].BlockNum, msgs[0].BlockHash)
+		preparedMsgs := consensus.FBFTLog.GetMessagesByTypeSeqHash(
+			msg_pb.MessageType_PREPARED, msgs[0].BlockNum, msgs[0].BlockHash,
+		)
 		msg := consensus.FBFTLog.FindMessageByMaxViewID(preparedMsgs)
 		if msg == nil {
 			break

consensus/consensus_validator_msg.go

@@ -1,59 +0,0 @@
-package consensus
-
-import (
-	"github.com/harmony-one/harmony/api/proto"
-	msg_pb "github.com/harmony-one/harmony/api/proto/message"
-	"github.com/harmony-one/harmony/internal/utils"
-)
-
-// Construct the prepare message to send to leader (assumption the consensus data is already verified)
-func (consensus *Consensus) constructPrepareMessage() []byte {
-	message := &msg_pb.Message{
-		ServiceType: msg_pb.ServiceType_CONSENSUS,
-		Type:        msg_pb.MessageType_PREPARE,
-		Request: &msg_pb.Message_Consensus{
-			Consensus: &msg_pb.ConsensusRequest{},
-		},
-	}
-
-	consensusMsg := message.GetConsensus()
-	consensus.populateMessageFields(consensusMsg)
-
-	// 96 byte of bls signature
-	sign := consensus.priKey.SignHash(consensusMsg.BlockHash)
-	if sign != nil {
-		consensusMsg.Payload = sign.Serialize()
-	}
-
-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
-	if err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to sign and marshal the Prepare message")
-	}
-	return proto.ConstructConsensusMessage(marshaledMessage)
-}
-
-// Construct the commit message which contains the signature on the multi-sig of prepare phase.
-func (consensus *Consensus) constructCommitMessage(commitPayload []byte) []byte {
-	message := &msg_pb.Message{
-		ServiceType: msg_pb.ServiceType_CONSENSUS,
-		Type:        msg_pb.MessageType_COMMIT,
-		Request: &msg_pb.Message_Consensus{
-			Consensus: &msg_pb.ConsensusRequest{},
-		},
-	}
-
-	consensusMsg := message.GetConsensus()
-	consensus.populateMessageFields(consensusMsg)
-
-	// 96 byte of bls signature
-	sign := consensus.priKey.SignHash(commitPayload)
-	if sign != nil {
-		consensusMsg.Payload = sign.Serialize()
-	}
-
-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
-	if err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to sign and marshal the Commit message")
-	}
-	return proto.ConstructConsensusMessage(marshaledMessage)
-}

consensus/consensus_validator_msg_test.go

@@ -1,73 +0,0 @@
-package consensus
-
-import (
-	"testing"
-
-	protobuf "github.com/golang/protobuf/proto"
-	"github.com/harmony-one/harmony/api/proto"
-	msg_pb "github.com/harmony-one/harmony/api/proto/message"
-	"github.com/harmony-one/harmony/consensus/quorum"
-	"github.com/harmony-one/harmony/crypto/bls"
-	"github.com/harmony-one/harmony/internal/utils"
-	"github.com/harmony-one/harmony/p2p"
-	"github.com/harmony-one/harmony/p2p/p2pimpl"
-	"github.com/harmony-one/harmony/shard"
-)
-
-func TestConstructPrepareMessage(test *testing.T) {
-	leader := p2p.Peer{IP: "127.0.0.1", Port: "9992"}
-	priKey, _, _ := utils.GenKeyP2P("127.0.0.1", "9902")
-	host, err := p2pimpl.NewHost(&leader, priKey)
-	if err != nil {
-		test.Fatalf("newhost failure: %v", err)
-	}
-	decider := quorum.NewDecider(quorum.SuperMajorityVote)
-	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider,
-	)
-	if err != nil {
-		test.Fatalf("Cannot craeate consensus: %v", err)
-	}
-	consensus.blockHash = [32]byte{}
-	msgBytes := consensus.constructPrepareMessage()
-	msgBytes, err = proto.GetConsensusMessagePayload(msgBytes)
-	if err != nil {
-		test.Error("Error when getting consensus message", "error", err)
-	}
-
-	msg := &msg_pb.Message{}
-	if err := protobuf.Unmarshal(msgBytes, msg); err != nil {
-		test.Error("Can not parse the message", err)
-	} else {
-		if msg.GetConsensus() == nil {
-			test.Error("Wrong message")
-		}
-	}
-}
-
-func TestConstructCommitMessage(test *testing.T) {
-	leader := p2p.Peer{IP: "127.0.0.1", Port: "9902"}
-	priKey, _, _ := utils.GenKeyP2P("127.0.0.1", "9902")
-	host, err := p2pimpl.NewHost(&leader, priKey)
-	if err != nil {
-		test.Fatalf("newhost failure: %v", err)
-	}
-	decider := quorum.NewDecider(quorum.SuperMajorityVote)
-	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider,
-	)
-	if err != nil {
-		test.Fatalf("Cannot craeate consensus: %v", err)
-	}
-	consensus.blockHash = [32]byte{}
-	msg := consensus.constructCommitMessage([]byte("random string"))
-	msg, err = proto.GetConsensusMessagePayload(msg)
-	if err != nil {
-		test.Errorf("Failed to get consensus message")
-	}
-
-	message := &msg_pb.Message{}
-	if err = protobuf.Unmarshal(msg, message); err != nil {
-		test.Errorf("Error when unmarshalling a message")
-	}
-}

consensus/construct.go

@@ -0,0 +1,98 @@
+package consensus
+
+import (
+	"bytes"
+
+	"github.com/harmony-one/bls/ffi/go/bls"
+	"github.com/harmony-one/harmony/api/proto"
+	msg_pb "github.com/harmony-one/harmony/api/proto/message"
+	"github.com/harmony-one/harmony/consensus/quorum"
+	"github.com/harmony-one/harmony/internal/utils"
+)
+
+// NetworkMessage is a message intended to be
+// created only for distribution to
+// all the other quorum members.
+type NetworkMessage struct {
+	Phase                      msg_pb.MessageType
+	Bytes                      []byte
+	FBFTMsg                    *FBFTMessage
+	OptionalAggregateSignature *bls.Sign
+}
+
+// construct is the single creation point of messages intended for the wire.
+// The trailing callback provides a hook for custom mutation so call can control
+// extra nuance on the network message, it is called after the BFT specific logic
+func (consensus *Consensus) construct(
+	p msg_pb.MessageType, payloadForSignOverride []byte,
+) (*NetworkMessage, error) {
+	message := &msg_pb.Message{
+		ServiceType: msg_pb.ServiceType_CONSENSUS,
+		Type:        p,
+		Request: &msg_pb.Message_Consensus{
+			Consensus: &msg_pb.ConsensusRequest{},
+		},
+	}
+
+	consensusMsg := message.GetConsensus()
+	consensus.populateMessageFields(consensusMsg)
+
+	var aggSig *bls.Sign
+
+	// Do the signing, 96 byte of bls signature
+	switch p {
+	case msg_pb.MessageType_PREPARED:
+		consensusMsg.Block = consensus.block
+		// Payload
+		buffer := bytes.Buffer{}
+		// 96 bytes aggregated signature
+		aggSig = consensus.Decider.AggregateVotes(quorum.Prepare)
+		buffer.Write(aggSig.Serialize())
+		// Bitmap
+		buffer.Write(consensus.prepareBitmap.Bitmap)
+		consensusMsg.Payload = buffer.Bytes()
+	case msg_pb.MessageType_PREPARE:
+		if s := consensus.priKey.SignHash(consensusMsg.BlockHash); s != nil {
+			consensusMsg.Payload = s.Serialize()
+		}
+	case msg_pb.MessageType_COMMIT:
+		if s := consensus.priKey.SignHash(payloadForSignOverride); s != nil {
+			consensusMsg.Block = consensus.block
+			consensusMsg.Payload = s.Serialize()
+		}
+	case msg_pb.MessageType_COMMITTED:
+		buffer := bytes.Buffer{}
+		// 96 bytes aggregated signature
+		aggSig = consensus.Decider.AggregateVotes(quorum.Commit)
+		buffer.Write(aggSig.Serialize())
+		// Bitmap
+		buffer.Write(consensus.commitBitmap.Bitmap)
+		consensusMsg.Payload = buffer.Bytes()
+	case msg_pb.MessageType_ANNOUNCE:
+		consensusMsg.Payload = consensus.blockHeader
+	}
+
+	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
+	if err != nil {
+		utils.Logger().Error().Err(err).
+			Str("phase", p.String()).
+			Msg("Failed to sign and marshal consensus message")
+		return nil, err
+	}
+
+	FBFTMsg, err2 := ParseFBFTMessage(message)
+
+	if err2 != nil {
+		utils.Logger().Error().Err(err).
+			Str("phase", p.String()).
+			Msg("failed to deal with the FBFT message")
+		return nil, err
+	}
+
+	return &NetworkMessage{
+		Phase:                      p,
+		Bytes:                      proto.ConstructConsensusMessage(marshaledMessage),
+		FBFTMsg:                    FBFTMsg,
+		OptionalAggregateSignature: aggSig,
+	}, nil
+}

consensus/construct_test.go

@@ -3,8 +3,6 @@ package consensus
 import (
 	"testing"
 
-	protobuf "github.com/golang/protobuf/proto"
-	"github.com/harmony-one/harmony/api/proto"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/consensus/quorum"
 	"github.com/harmony-one/harmony/crypto/bls"
@@ -30,16 +28,8 @@ func TestConstructAnnounceMessage(test *testing.T) {
 		test.Fatalf("Cannot create consensus: %v", err)
 	}
 	consensus.blockHash = [32]byte{}
-
-	message := &msg_pb.Message{}
-	msgBytes := consensus.constructAnnounceMessage()
-	msgPayload, _ := proto.GetConsensusMessagePayload(msgBytes)
-
-	if err := protobuf.Unmarshal(msgPayload, message); err != nil {
-		test.Errorf("Error when creating announce message")
-	}
-	if message.Type != msg_pb.MessageType_ANNOUNCE {
-		test.Error("it did not created announce message")
+	if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil); err != nil {
+		test.Fatalf("could not construct announce: %v", err)
 	}
 }
 
@@ -66,12 +56,13 @@ func TestConstructPreparedMessage(test *testing.T) {
 	consensus.blockHash = [32]byte{}
 
 	message := "test string"
-	consensus.Decider.AddSignature(
-		quorum.Prepare, leaderPubKey, leaderPriKey.Sign(message), leaderPubKey, 9999,
+	consensus.Decider.SubmitVote(
+		quorum.Prepare, leaderPubKey, leaderPriKey.Sign(message), nil,
 	)
-	consensus.Decider.AddSignature(
-		quorum.Prepare, validatorPubKey, validatorPriKey.Sign(message), leaderPubKey, 9999,
+	consensus.Decider.SubmitVote(
+		quorum.Prepare, validatorPubKey, validatorPriKey.Sign(message), nil,
 	)
+
 	// According to RJ these failures are benign.
 	if err := consensus.prepareBitmap.SetKey(leaderPubKey, true); err != nil {
 		test.Log(ctxerror.New("prepareBitmap.SetKey").WithCause(err))
@@ -80,14 +71,11 @@ func TestConstructPreparedMessage(test *testing.T) {
 		test.Log(ctxerror.New("prepareBitmap.SetKey").WithCause(err))
 	}
 
-	msgBytes, _ := consensus.constructPreparedMessage()
-	msgPayload, _ := proto.GetConsensusMessagePayload(msgBytes)
-
-	msg := &msg_pb.Message{}
-	if err = protobuf.Unmarshal(msgPayload, msg); err != nil {
+	network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil)
+	if err != nil {
 		test.Errorf("Error when creating prepared message")
 	}
-	if msg.Type != msg_pb.MessageType_PREPARED {
+	if network.Phase != msg_pb.MessageType_PREPARED {
 		test.Error("it did not created prepared message")
 	}
 }

consensus/engine/consensus_engine.go

@@ -11,7 +11,6 @@ import (
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/shard/committee"
-	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
 )
 
@@ -107,12 +106,6 @@ type Engine interface {
 	// SetRewarder assigns the Distributor used in block reward
 	SetRewarder(reward.Distributor)
 
-	// Slasher handles slashing accounts due to inavailibility or double-signing
-	Slasher() slash.Slasher
-
-	// SetSlasher assigns the slasher used
-	SetSlasher(slash.Slasher)
-
 	// Beaconchain provides the handle for Beaconchain
 	Beaconchain() ChainReader
 

consensus/fbft_log.go

@@ -223,16 +223,13 @@ func ParseFBFTMessage(msg *msg_pb.Message) (*FBFTMessage, error) {
 	pbftMsg := FBFTMessage{}
 	pbftMsg.MessageType = msg.GetType()
 	consensusMsg := msg.GetConsensus()
-
 	pbftMsg.ViewID = consensusMsg.ViewId
 	pbftMsg.BlockNum = consensusMsg.BlockNum
-	pbftMsg.BlockHash = common.Hash{}
 	copy(pbftMsg.BlockHash[:], consensusMsg.BlockHash[:])
 	pbftMsg.Payload = make([]byte, len(consensusMsg.Payload))
 	copy(pbftMsg.Payload[:], consensusMsg.Payload[:])
 	pbftMsg.Block = make([]byte, len(consensusMsg.Block))
 	copy(pbftMsg.Block[:], consensusMsg.Block[:])
-
 	pubKey, err := bls_cosi.BytesToBlsPublicKey(consensusMsg.SenderPubkey)
 	if err != nil {
 		return nil, err

consensus/fbft_log_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	protobuf "github.com/golang/protobuf/proto"
-	"github.com/harmony-one/harmony/api/proto"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	"github.com/harmony-one/harmony/consensus/quorum"
 	"github.com/harmony-one/harmony/crypto/bls"
@@ -14,7 +13,7 @@ import (
 	"github.com/harmony-one/harmony/shard"
 )
 
-func constructAnnounceMessage(t *testing.T) []byte {
+func constructAnnounceMessage(t *testing.T) (*NetworkMessage, error) {
 	leader := p2p.Peer{IP: "127.0.0.1", Port: "19999"}
 	priKey, _, _ := utils.GenKeyP2P("127.0.0.1", "9902")
 	host, err := p2pimpl.NewHost(&leader, priKey)
@@ -29,10 +28,7 @@ func constructAnnounceMessage(t *testing.T) []byte {
 		t.Fatalf("Cannot create consensus: %v", err)
 	}
 	consensus.blockHash = [32]byte{}
-
-	msgBytes := consensus.constructAnnounceMessage()
-	msgPayload, _ := proto.GetConsensusMessagePayload(msgBytes)
-	return msgPayload
+	return consensus.construct(msg_pb.MessageType_ANNOUNCE, nil)
 }
 
 func getConsensusMessage(payload []byte) (*msg_pb.Message, error) {
@@ -44,18 +40,6 @@ func getConsensusMessage(payload []byte) (*msg_pb.Message, error) {
 	return msg, nil
 }
 
-func TestParseFBFTMessage(t *testing.T) {
-	payload := constructAnnounceMessage(t)
-	msg, err := getConsensusMessage(payload)
-	if err != nil {
-		t.Error("create consensus message error")
-	}
-	_, err = ParseFBFTMessage(msg)
-	if err != nil {
-		t.Error("unable to parse FBFTMessage")
-	}
-}
-
 func TestGetMessagesByTypeSeqViewHash(t *testing.T) {
 	pbftMsg := FBFTMessage{MessageType: msg_pb.MessageType_ANNOUNCE, BlockNum: 2, ViewID: 3, BlockHash: [32]byte{01, 02}}
 	log := NewFBFTLog()

consensus/leader.go

@@ -0,0 +1,348 @@
+package consensus
+
+import (
+	"encoding/binary"
+	"time"
+
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/harmony-one/bls/ffi/go/bls"
+	msg_pb "github.com/harmony-one/harmony/api/proto/message"
+	"github.com/harmony-one/harmony/consensus/quorum"
+	"github.com/harmony-one/harmony/core/types"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/harmony-one/harmony/p2p/host"
+)
+
+func (consensus *Consensus) announce(block *types.Block) {
+	blockHash := block.Hash()
+	copy(consensus.blockHash[:], blockHash[:])
+
+	// prepare message and broadcast to validators
+	encodedBlock, err := rlp.EncodeToBytes(block)
+	if err != nil {
+		consensus.getLogger().Debug().Msg("[Announce] Failed encoding block")
+		return
+	}
+	encodedBlockHeader, err := rlp.EncodeToBytes(block.Header())
+	if err != nil {
+		consensus.getLogger().Debug().Msg("[Announce] Failed encoding block header")
+		return
+	}
+
+	consensus.block = encodedBlock
+	consensus.blockHeader = encodedBlockHeader
+	network, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil)
+	if err != nil {
+		consensus.getLogger().Err(err).
+			Str("message-type", msg_pb.MessageType_ANNOUNCE.String()).
+			Msg("failed constructing message")
+		return
+	}
+	msgToSend, FPBTMsg := network.Bytes, network.FBFTMsg
+
+	// TODO(chao): review FPBT log data structure
+	consensus.FBFTLog.AddMessage(FPBTMsg)
+	consensus.getLogger().Debug().
+		Str("MsgBlockHash", FPBTMsg.BlockHash.Hex()).
+		Uint64("MsgViewID", FPBTMsg.ViewID).
+		Uint64("MsgBlockNum", FPBTMsg.BlockNum).
+		Msg("[Announce] Added Announce message in FPBT")
+	consensus.FBFTLog.AddBlock(block)
+
+	// Leader sign the block hash itself
+	consensus.Decider.SubmitVote(
+		quorum.Prepare,
+		consensus.PubKey,
+		consensus.priKey.SignHash(consensus.blockHash[:]),
+		nil,
+	)
+	if err := consensus.prepareBitmap.SetKey(consensus.PubKey, true); err != nil {
+		consensus.getLogger().Warn().Err(err).Msg(
+			"[Announce] Leader prepareBitmap SetKey failed",
+		)
+		return
+	}
+
+	// Construct broadcast p2p message
+	if err := consensus.msgSender.SendWithRetry(
+		consensus.blockNum, msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{
+			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)),
+		}, host.ConstructP2pMessage(byte(17), msgToSend)); err != nil {
+		consensus.getLogger().Warn().
+			Str("groupID", string(nodeconfig.NewGroupIDByShardID(
+				nodeconfig.ShardID(consensus.ShardID),
+			))).
+			Msg("[Announce] Cannot send announce message")
+	} else {
+		consensus.getLogger().Info().
+			Str("blockHash", block.Hash().Hex()).
+			Uint64("blockNum", block.NumberU64()).
+			Msg("[Announce] Sent Announce Message!!")
+	}
+
+	consensus.getLogger().Debug().
+		Str("From", consensus.phase.String()).
+		Str("To", FBFTPrepare.String()).
+		Msg("[Announce] Switching phase")
+	consensus.switchPhase(FBFTPrepare, true)
+}
+
+func (consensus *Consensus) onPrepare(msg *msg_pb.Message) {
+	recvMsg, err := ParseFBFTMessage(msg)
+	if err != nil {
+		consensus.getLogger().Error().Err(err).Msg("[OnPrepare] Unparseable validator message")
+		return
+	}
+
+	if recvMsg.ViewID != consensus.viewID || recvMsg.BlockNum != consensus.blockNum {
+		consensus.getLogger().Debug().
+			Uint64("MsgViewID", recvMsg.ViewID).
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("blockNum", consensus.blockNum).
+			Msg("[OnPrepare] Message ViewId or BlockNum not match")
+		return
+	}
+
+	if !consensus.FBFTLog.HasMatchingViewAnnounce(
+		consensus.blockNum, consensus.viewID, recvMsg.BlockHash,
+	) {
+		consensus.getLogger().Debug().
+			Uint64("MsgViewID", recvMsg.ViewID).
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("blockNum", consensus.blockNum).
+			Msg("[OnPrepare] No Matching Announce message")
+		//return
+	}
+
+	validatorPubKey := recvMsg.SenderPubkey
+	prepareSig := recvMsg.Payload
+	prepareBitmap := consensus.prepareBitmap
+
+	consensus.mutex.Lock()
+	defer consensus.mutex.Unlock()
+	logger := consensus.getLogger().With().
+		Str("validatorPubKey", validatorPubKey.SerializeToHexStr()).Logger()
+
+	// proceed only when the message is not received before
+	signed := consensus.Decider.ReadBallot(quorum.Prepare, validatorPubKey)
+	if signed != nil {
+		logger.Debug().
+			Msg("[OnPrepare] Already Received prepare message from the validator")
+		return
+	}
+
+	if consensus.Decider.IsQuorumAchieved(quorum.Prepare) {
+		// already have enough signatures
+		logger.Debug().Msg("[OnPrepare] Received Additional Prepare Message")
+		return
+	}
+
+	// Check BLS signature for the multi-sig
+	var sign bls.Sign
+	err = sign.Deserialize(prepareSig)
+	if err != nil {
+		consensus.getLogger().Error().Err(err).
+			Msg("[OnPrepare] Failed to deserialize bls signature")
+		return
+	}
+	if !sign.VerifyHash(recvMsg.SenderPubkey, consensus.blockHash[:]) {
+		consensus.getLogger().Error().Msg("[OnPrepare] Received invalid BLS signature")
+		return
+	}
+
+	logger = logger.With().
+		Int64("NumReceivedSoFar", consensus.Decider.SignersCount(quorum.Prepare)).
+		Int64("PublicKeys", consensus.Decider.ParticipantsCount()).Logger()
+	logger.Info().Msg("[OnPrepare] Received New Prepare Signature")
+	consensus.Decider.SubmitVote(
+		quorum.Prepare, validatorPubKey, &sign, nil,
+	)
+	// Set the bitmap indicating that this validator signed.
+	if err := prepareBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil {
+		consensus.getLogger().Warn().Err(err).Msg("[OnPrepare] prepareBitmap.SetKey failed")
+		return
+	}
+
+	if consensus.Decider.IsQuorumAchieved(quorum.Prepare) {
+		logger.Debug().Msg("[OnPrepare] Received Enough Prepare Signatures")
+		// Construct and broadcast prepared message
+		network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil)
+		if err != nil {
+			consensus.getLogger().Err(err).
+				Str("message-type", msg_pb.MessageType_PREPARED.String()).
+				Msg("failed constructing message")
+			return
+		}
+		msgToSend, FBFTMsg, aggSig :=
+			network.Bytes,
+			network.FBFTMsg,
+			network.OptionalAggregateSignature
+
+		consensus.aggregatedPrepareSig = aggSig
+		consensus.FBFTLog.AddMessage(FBFTMsg)
+		// Leader add commit phase signature
+		blockNumHash := make([]byte, 8)
+		binary.LittleEndian.PutUint64(blockNumHash, consensus.blockNum)
+		commitPayload := append(blockNumHash, consensus.blockHash[:]...)
+		consensus.Decider.SubmitVote(
+			quorum.Commit,
+			consensus.PubKey,
+			consensus.priKey.SignHash(commitPayload),
+			consensus.block[:],
+		)
+
+		if err := consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil {
+			consensus.getLogger().Debug().Msg("[OnPrepare] Leader commit bitmap set failed")
+			return
+		}
+
+		if err := consensus.msgSender.SendWithRetry(
+			consensus.blockNum,
+			msg_pb.MessageType_PREPARED, []nodeconfig.GroupID{
+				nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)),
+			},
+			host.ConstructP2pMessage(byte(17), msgToSend),
+		); err != nil {
+			consensus.getLogger().Warn().Msg("[OnPrepare] Cannot send prepared message")
+		} else {
+			consensus.getLogger().Debug().
+				Hex("blockHash", consensus.blockHash[:]).
+				Uint64("blockNum", consensus.blockNum).
+				Msg("[OnPrepare] Sent Prepared Message!!")
+		}
+		consensus.msgSender.StopRetry(msg_pb.MessageType_ANNOUNCE)
+		// Stop retry committed msg of last consensus
+		consensus.msgSender.StopRetry(msg_pb.MessageType_COMMITTED)
+
+		consensus.getLogger().Debug().
+			Str("From", consensus.phase.String()).
+			Str("To", FBFTCommit.String()).
+			Msg("[OnPrepare] Switching phase")
+		consensus.switchPhase(FBFTCommit, true)
+	}
+}
+
+func (consensus *Consensus) onCommit(msg *msg_pb.Message) {
+	recvMsg, err := ParseFBFTMessage(msg)
+
+	if err != nil {
+		consensus.getLogger().Debug().Err(err).Msg("[OnCommit] Parse pbft message failed")
+		return
+	}
+
+	// let it handle its own logs
+	// TODO(Edgar)(TEMP DISABLE while testing double sign)
+	if !consensus.onCommitSanityChecks(recvMsg) {
+		return
+	}
+
+	consensus.mutex.Lock()
+	defer consensus.mutex.Unlock()
+
+	validatorPubKey, commitSig, commitBitmap :=
+		recvMsg.SenderPubkey, recvMsg.Payload, consensus.commitBitmap
+	logger := consensus.getLogger().With().
+		Str("validatorPubKey", validatorPubKey.SerializeToHexStr()).Logger()
+
+	if alreadyCastBallot := consensus.Decider.ReadBallot(
+		quorum.Commit, validatorPubKey,
+	); alreadyCastBallot != nil {
+		logger.Debug().Msg("voter already cast commit message")
+		return
+
+		// TODO(Edgar) Still working out double sign
+		// var signed, received *types.Block
+		// if err := rlp.DecodeBytes(
+		// 	alreadyCastBallot.OptSerializedBlock, &signed,
+		// ); err != nil {
+		// 	// TODO handle
+		// }
+
+		// areHeightsEqual := signed.Header().Number().Uint64() == recvMsg.BlockNum
+		// areViewIDsEqual := signed.Header().ViewID().Uint64() == recvMsg.ViewID
+		// areHeadersEqual := bytes.Compare(
+		// 	signed.Hash().Bytes(), recvMsg.BlockHash.Bytes(),
+		// ) == 0
+
+		// // If signer already signed, and the block height is the same, then we
+		// // need to verify the block hash, and if block hash is different, then
+		// // that is a clear case of double signing
+		// if areHeightsEqual && areViewIDsEqual && !areHeadersEqual {
+		// 	// TODO
+		// 	if err := rlp.DecodeBytes(recvMsg.Block, &received); err != nil {
+		// 		// TODO Some log
+		// 		return
+		// 	}
+
+		// 	var doubleSign bls.Sign
+		// 	if err := doubleSign.Deserialize(recvMsg.Payload); err != nil {
+		// 		return
+		// 	}
+
+		// 	go func() {
+		// 		consensus.SlashChan <- slash.NewRecord(
+		// 			*shard.FromLibBLSPublicKeyUnsafe(validatorPubKey),
+		// 			signed.Header(), received.Header(),
+		// 			alreadyCastBallot.Signature, &doubleSign,
+		// 			consensus.SelfAddress,
+		// 		)
+		// 	}()
+		// }
+		// return
+	}
+
+	// has to be called before verifying signature
+	quorumWasMet := consensus.Decider.IsQuorumAchieved(quorum.Commit)
+	// Verify the signature on commitPayload is correct
+	var sign bls.Sign
+	if err := sign.Deserialize(commitSig); err != nil {
+		logger.Debug().Msg("[OnCommit] Failed to deserialize bls signature")
+		return
+	}
+
+	blockNumHash := make([]byte, 8)
+	binary.LittleEndian.PutUint64(blockNumHash, recvMsg.BlockNum)
+	commitPayload := append(blockNumHash, recvMsg.BlockHash[:]...)
+	logger = logger.With().
+		Uint64("MsgViewID", recvMsg.ViewID).
+		Uint64("MsgBlockNum", recvMsg.BlockNum).
+		Logger()
+
+	if !sign.VerifyHash(recvMsg.SenderPubkey, commitPayload) {
+		logger.Error().Msg("[OnCommit] Cannot verify commit message")
+		return
+	}
+
+	logger = logger.With().
+		Int64("numReceivedSoFar", consensus.Decider.SignersCount(quorum.Commit)).
+		Logger()
+	logger.Info().Msg("[OnCommit] Received new commit message")
+
+	consensus.Decider.SubmitVote(
+		quorum.Commit, validatorPubKey, &sign, consensus.block,
+	)
+	// Set the bitmap indicating that this validator signed.
+	if err := commitBitmap.SetKey(recvMsg.SenderPubkey, true); err != nil {
+		consensus.getLogger().Warn().Err(err).Msg("[OnCommit] commitBitmap.SetKey failed")
+		return
+	}
+
+	quorumIsMet := consensus.Decider.IsQuorumAchieved(quorum.Commit)
+	if !quorumWasMet && quorumIsMet {
+		logger.Info().Msg("[OnCommit] 2/3 Enough commits received")
+		go func(viewID uint64) {
+			time.Sleep(2 * time.Second)
+			logger.Debug().Msg("[OnCommit] Commit Grace Period Ended")
+			consensus.commitFinishChan <- viewID
+		}(consensus.viewID)
+
+		consensus.msgSender.StopRetry(msg_pb.MessageType_PREPARED)
+	}
+
+	if consensus.Decider.IsRewardThresholdAchieved() {
+		go func(viewID uint64) {
+			consensus.commitFinishChan <- viewID
+			logger.Info().Msg("[OnCommit] 90% Enough commits received")
+		}(consensus.viewID)
+	}
+}

consensus/quorum/one-node-one-vote.go

@@ -96,11 +96,6 @@ func (v *uniformVoteWeight) Award(
 	return payout
 }
 
-func (v *uniformVoteWeight) ShouldSlash(k shard.BlsPublicKey) bool {
-	// No-op, no semantic meaning in one-slot-one-vote
-	return false
-}
-
 func (v *uniformVoteWeight) JSON() string {
 	s, _ := v.ShardIDProvider()()
 

consensus/quorum/one-node-staked-vote.go

@@ -110,7 +110,7 @@ func (v *stakedVoteWeight) computeCurrentTotalPower(p Phase) (*numeric.Dec, erro
 	for i := range members {
 		w.FromLibBLSPublicKey(members[i])
 		if _, didVote := ballot.voters[w]; !didVote &&
-			v.ReadSignature(p, members[i]) != nil {
+			v.ReadBallot(p, members[i]) != nil {
 			err := w.FromLibBLSPublicKey(members[i])
 			if err != nil {
 				return nil, err
@@ -208,7 +208,6 @@ func (v *stakedVoteWeight) ToggleActive(k *bls.PublicKey) bool {
 func (v *stakedVoteWeight) JSON() string {
 	s, _ := v.ShardIDProvider()()
 	voterCount := len(v.roster.Voters)
-
 	type u struct {
 		IsHarmony      bool   `json:"is-harmony-slot"`
 		Identity       string `json:"bls-public-key"`

consensus/quorum/one-node-staked-vote_test.go

@@ -104,7 +104,7 @@ func sign(d Decider, k secretKeyMap, p Phase) {
 		pubKey := v.GetPublicKey()
 		sig := v.Sign(msg)
 		// TODO Make upstream test provide meaningful test values
-		d.AddSignature(p, pubKey, sig, pubKey, 99999)
+		d.SubmitVote(p, pubKey, sig, nil)
 	}
 }
 

consensus/quorum/quorum.go

@@ -6,6 +6,7 @@ import (
 	"github.com/harmony-one/bls/ffi/go/bls"
 	"github.com/harmony-one/harmony/consensus/votepower"
 	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
+	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/shard"
 )
@@ -24,8 +25,8 @@ const (
 
 var phaseNames = map[Phase]string{
 	Prepare:    "Prepare",
-	Commit:     "Prepare",
-	ViewChange: "Commit",
+	Commit:     "Commit",
+	ViewChange: "viewChange",
 }
 
 func (p Phase) String() string {
@@ -71,9 +72,9 @@ type ParticipantTracker interface {
 // SignatoryTracker ..
 type SignatoryTracker interface {
 	ParticipantTracker
-	AddSignature(
+	SubmitVote(
 		p Phase, PubKey *bls.PublicKey,
-		sig *bls.Sign, roundLeader *bls.PublicKey, roundNumber uint64,
+		sig *bls.Sign, optSerializedBlock []byte,
 	)
 	// Caller assumes concurrency protection
 	SignersCount(Phase) int64
@@ -83,8 +84,8 @@ type SignatoryTracker interface {
 // SignatureReader ..
 type SignatureReader interface {
 	SignatoryTracker
-	ReadAllSignatures(Phase) []*bls.Sign
-	ReadSignature(p Phase, PubKey *bls.PublicKey) *bls.Sign
+	ReadAllBallots(Phase) []*votepower.Ballot
+	ReadBallot(p Phase, PubKey *bls.PublicKey) *votepower.Ballot
 	TwoThirdsSignersCount() int64
 	// 96 bytes aggregated signature
 	AggregateVotes(p Phase) *bls.Sign
@@ -128,13 +129,12 @@ type Decider interface {
 // and values are BLS private key signed signatures
 type cIdentities struct {
 	// Public keys of the committee including leader and validators
-	publicKeys   []*bls.PublicKey
-	announcement *votepower.Round
-	commit       *votepower.Round
+	publicKeys []*bls.PublicKey
+	prepare    *votepower.Round
+	commit     *votepower.Round
 	// viewIDSigs: every validator
 	// sign on |viewID|blockHash| in view changing message
-	viewID      *votepower.Round
-	seenCounter map[[shard.PublicKeySizeInBytes]byte]int
+	viewChange *votepower.Round
 }
 
 type depInject struct {
@@ -143,7 +143,12 @@ type depInject struct {
 }
 
 func (s *cIdentities) AggregateVotes(p Phase) *bls.Sign {
-	return bls_cosi.AggregateSig(s.ReadAllSignatures(p))
+	ballots := s.ReadAllBallots(p)
+	sigs := make([]*bls.Sign, 0, len(ballots))
+	for _, ballot := range ballots {
+		sigs = append(sigs, ballot.Signature)
+	}
+	return bls_cosi.AggregateSig(sigs)
 }
 
 func (s *cIdentities) IndexOf(pubKey *bls.PublicKey) int {
@@ -171,12 +176,9 @@ func (s *cIdentities) Participants() []*bls.PublicKey {
 }
 
 func (s *cIdentities) UpdateParticipants(pubKeys []*bls.PublicKey) {
-	// TODO - might need to put reset of seen counter in separate method
-	s.seenCounter = make(map[[shard.PublicKeySizeInBytes]byte]int, len(pubKeys))
 	for i := range pubKeys {
 		k := shard.BlsPublicKey{}
 		k.FromLibBLSPublicKey(pubKeys[i])
-		s.seenCounter[k] = 0
 	}
 	s.publicKeys = append(pubKeys[:0:0], pubKeys...)
 }
@@ -196,36 +198,39 @@ func (s *cIdentities) ParticipantsCount() int64 {
 func (s *cIdentities) SignersCount(p Phase) int64 {
 	switch p {
 	case Prepare:
-		return int64(len(s.announcement.BallotBox))
+		return int64(len(s.prepare.BallotBox))
 	case Commit:
 		return int64(len(s.commit.BallotBox))
 	case ViewChange:
-		return int64(len(s.viewID.BallotBox))
+		return int64(len(s.viewChange.BallotBox))
 	default:
 		return 0
 
 	}
 }
 
-func (s *cIdentities) AddSignature(
+func (s *cIdentities) SubmitVote(
 	p Phase, PubKey *bls.PublicKey,
-	sig *bls.Sign, roundLeader *bls.PublicKey, roundNumber uint64,
+	sig *bls.Sign, optSerializedBlock []byte,
 ) {
-	hex := PubKey.SerializeToHexStr()
+	if p != Commit && optSerializedBlock != nil && len(optSerializedBlock) != 0 {
+		utils.Logger().Debug().Str("phase", p.String()).
+			Msg("non-commit phase has non-nil, non-empty block")
+	}
+
 	ballot := &votepower.Ballot{
-		*shard.FromLibBLSPublicKeyUnsafe(PubKey),
-		*shard.FromLibBLSPublicKeyUnsafe(roundLeader),
-		roundNumber,
-		sig,
+		SignerPubKey:       *shard.FromLibBLSPublicKeyUnsafe(PubKey),
+		Signature:          sig,
+		OptSerializedBlock: optSerializedBlock[:],
 	}
 
-	switch p {
+	switch hex := PubKey.SerializeToHexStr(); p {
 	case Prepare:
-		s.announcement.BallotBox[hex] = ballot
+		s.prepare.BallotBox[hex] = ballot
 	case Commit:
 		s.commit.BallotBox[hex] = ballot
 	case ViewChange:
-		s.viewID.BallotBox[hex] = ballot
+		s.viewChange.BallotBox[hex] = ballot
 	}
 }
 
@@ -233,11 +238,11 @@ func (s *cIdentities) reset(ps []Phase) {
 	for i := range ps {
 		switch m := votepower.NewRound(); ps[i] {
 		case Prepare:
-			s.announcement = m
+			s.prepare = m
 		case Commit:
 			s.commit = m
 		case ViewChange:
-			s.viewID = m
+			s.viewChange = m
 		}
 	}
 }
@@ -246,50 +251,49 @@ func (s *cIdentities) TwoThirdsSignersCount() int64 {
 	return s.ParticipantsCount()*2/3 + 1
 }
 
-func (s *cIdentities) ReadSignature(p Phase, PubKey *bls.PublicKey) *bls.Sign {
+func (s *cIdentities) ReadBallot(p Phase, PubKey *bls.PublicKey) *votepower.Ballot {
 	var ballotBox map[string]*votepower.Ballot
 	hex := PubKey.SerializeToHexStr()
 
 	switch p {
 	case Prepare:
-		ballotBox = s.announcement.BallotBox
+		ballotBox = s.prepare.BallotBox
 	case Commit:
 		ballotBox = s.commit.BallotBox
 	case ViewChange:
-		ballotBox = s.viewID.BallotBox
+		ballotBox = s.viewChange.BallotBox
 	}
 
 	payload, ok := ballotBox[hex]
 	if !ok {
 		return nil
 	}
-	return payload.Signature
+	return payload
 }
 
-func (s *cIdentities) ReadAllSignatures(p Phase) []*bls.Sign {
+func (s *cIdentities) ReadAllBallots(p Phase) []*votepower.Ballot {
 	var m map[string]*votepower.Ballot
 	switch p {
 	case Prepare:
-		m = s.announcement.BallotBox
+		m = s.prepare.BallotBox
 	case Commit:
 		m = s.commit.BallotBox
 	case ViewChange:
-		m = s.viewID.BallotBox
+		m = s.viewChange.BallotBox
 	}
-	sigs := make([]*bls.Sign, 0, len(m))
-	for _, value := range m {
-		sigs = append(sigs, value.Signature)
+	ballots := make([]*votepower.Ballot, 0, len(m))
+	for _, ballot := range m {
+		ballots = append(ballots, ballot)
 	}
-	return sigs
+	return ballots
 }
 
-func newMapBackedSignatureReader() *cIdentities {
+func newBallotsBackedSignatureReader() *cIdentities {
 	return &cIdentities{
-		publicKeys:   []*bls.PublicKey{},
-		announcement: votepower.NewRound(),
-		commit:       votepower.NewRound(),
-		viewID:       votepower.NewRound(),
-		seenCounter:  map[[shard.PublicKeySizeInBytes]byte]int{},
+		publicKeys: []*bls.PublicKey{},
+		prepare:    votepower.NewRound(),
+		commit:     votepower.NewRound(),
+		viewChange: votepower.NewRound(),
 	}
 }
 
@@ -317,7 +321,7 @@ func (d *depInject) MyPublicKey() func() (*bls.PublicKey, error) {
 
 // NewDecider ..
 func NewDecider(p Policy) Decider {
-	signatureStore := newMapBackedSignatureReader()
+	signatureStore := newBallotsBackedSignatureReader()
 	deps := &depInject{}
 	c := &composite{deps, deps, signatureStore}
 	switch p {

consensus/validator.go

@@ -0,0 +1,379 @@
+package consensus
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/hex"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	msg_pb "github.com/harmony-one/harmony/api/proto/message"
+	"github.com/harmony-one/harmony/block"
+	"github.com/harmony-one/harmony/core/types"
+	"github.com/harmony-one/harmony/internal/chain"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
+	"github.com/harmony-one/harmony/p2p/host"
+)
+
+func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) {
+	recvMsg, err := ParseFBFTMessage(msg)
+	if err != nil {
+		consensus.getLogger().Error().
+			Err(err).
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Msg("[OnAnnounce] Unparseable leader message")
+		return
+	}
+
+	// verify validity of block header object
+	// TODO: think about just sending the block hash instead of the header.
+	encodedHeader := recvMsg.Payload
+	header := new(block.Header)
+	if err := rlp.DecodeBytes(encodedHeader, header); err != nil {
+		consensus.getLogger().Warn().
+			Err(err).
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Msg("[OnAnnounce] Unparseable block header data")
+		return
+	}
+
+	if recvMsg.BlockNum < consensus.blockNum ||
+		recvMsg.BlockNum != header.Number().Uint64() {
+		consensus.getLogger().Debug().
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("hdrBlockNum", header.Number().Uint64()).
+			Uint64("consensuBlockNum", consensus.blockNum).
+			Msg("[OnAnnounce] BlockNum does not match")
+		return
+	}
+
+	if err := chain.Engine.VerifyHeader(consensus.ChainReader, header, true); err != nil {
+		consensus.getLogger().Warn().
+			Err(err).
+			Str("inChain", consensus.ChainReader.CurrentHeader().Number().String()).
+			Str("MsgBlockNum", header.Number().String()).
+			Msg("[OnAnnounce] Block content is not verified successfully")
+		return
+	}
+
+	//VRF/VDF is only generated in the beach chain
+	if consensus.NeedsRandomNumberGeneration(header.Epoch()) {
+		//validate the VRF with proof if a non zero VRF is found in header
+		if len(header.Vrf()) > 0 {
+			if !consensus.ValidateVrfAndProof(header) {
+				return
+			}
+		}
+
+		//validate the VDF with proof if a non zero VDF is found in header
+		if len(header.Vdf()) > 0 {
+			if !consensus.ValidateVdfAndProof(header) {
+				return
+			}
+		}
+	}
+
+	logMsgs := consensus.FBFTLog.GetMessagesByTypeSeqView(
+		msg_pb.MessageType_ANNOUNCE, recvMsg.BlockNum, recvMsg.ViewID,
+	)
+	if len(logMsgs) > 0 {
+		if logMsgs[0].BlockHash != recvMsg.BlockHash &&
+			logMsgs[0].SenderPubkey.IsEqual(recvMsg.SenderPubkey) {
+			consensus.getLogger().Debug().
+				Str("logMsgSenderKey", logMsgs[0].SenderPubkey.SerializeToHexStr()).
+				Str("logMsgBlockHash", logMsgs[0].BlockHash.Hex()).
+				Str("recvMsg.SenderPubkey", recvMsg.SenderPubkey.SerializeToHexStr()).
+				Uint64("recvMsg.BlockNum", recvMsg.BlockNum).
+				Uint64("recvMsg.ViewID", recvMsg.ViewID).
+				Str("recvMsgBlockHash", recvMsg.BlockHash.Hex()).
+				Str("LeaderKey", consensus.LeaderPubKey.SerializeToHexStr()).
+				Msg("[OnAnnounce] Leader is malicious")
+			if consensus.current.Mode() == ViewChanging {
+				viewID := consensus.current.ViewID()
+				consensus.startViewChange(viewID + 1)
+			} else {
+				consensus.startViewChange(consensus.viewID + 1)
+			}
+		}
+		consensus.getLogger().Debug().
+			Str("leaderKey", consensus.LeaderPubKey.SerializeToHexStr()).
+			Msg("[OnAnnounce] Announce message received again")
+	}
+
+	consensus.getLogger().Debug().
+		Uint64("MsgViewID", recvMsg.ViewID).
+		Uint64("MsgBlockNum", recvMsg.BlockNum).
+		Msg("[OnAnnounce] Announce message Added")
+	consensus.FBFTLog.AddMessage(recvMsg)
+	consensus.mutex.Lock()
+	defer consensus.mutex.Unlock()
+	consensus.blockHash = recvMsg.BlockHash
+	// we have already added message and block, skip check viewID
+	// and send prepare message if is in ViewChanging mode
+	if consensus.current.Mode() == ViewChanging {
+		consensus.getLogger().Debug().
+			Msg("[OnAnnounce] Still in ViewChanging Mode, Exiting !!")
+		return
+	}
+
+	if consensus.checkViewID(recvMsg) != nil {
+		if consensus.current.Mode() == Normal {
+			consensus.getLogger().Debug().
+				Uint64("MsgViewID", recvMsg.ViewID).
+				Uint64("MsgBlockNum", recvMsg.BlockNum).
+				Msg("[OnAnnounce] ViewID check failed")
+		}
+		return
+	}
+	consensus.prepare()
+}
+
+func (consensus *Consensus) prepare() {
+	network, err := consensus.construct(msg_pb.MessageType_PREPARE, nil)
+	if err != nil {
+		consensus.getLogger().Err(err).
+			Str("message-type", msg_pb.MessageType_PREPARE.String()).
+			Msg("could not construct message")
+		return
+	}
+	msgToSend := network.Bytes
+
+	// TODO: this will not return immediatey, may block
+	if err := consensus.msgSender.SendWithoutRetry(
+		[]nodeconfig.GroupID{
+			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)),
+		},
+		host.ConstructP2pMessage(byte(17), msgToSend),
+	); err != nil {
+		consensus.getLogger().Warn().Err(err).Msg("[OnAnnounce] Cannot send prepare message")
+	} else {
+		consensus.getLogger().Info().
+			Str("blockHash", hex.EncodeToString(consensus.blockHash[:])).
+			Msg("[OnAnnounce] Sent Prepare Message!!")
+	}
+	consensus.getLogger().Debug().
+		Str("From", consensus.phase.String()).
+		Str("To", FBFTPrepare.String()).
+		Msg("[Announce] Switching Phase")
+	consensus.switchPhase(FBFTPrepare, true)
+}
+
+// if onPrepared accepts the prepared message from the leader, then
+// it will send a COMMIT message for the leader to receive on the network.
+func (consensus *Consensus) onPrepared(msg *msg_pb.Message) {
+	recvMsg, err := ParseFBFTMessage(msg)
+	if err != nil {
+		consensus.getLogger().Debug().Err(err).Msg("[OnPrepared] Unparseable validator message")
+		return
+	}
+	consensus.getLogger().Info().
+		Uint64("MsgBlockNum", recvMsg.BlockNum).
+		Uint64("MsgViewID", recvMsg.ViewID).
+		Msg("[OnPrepared] Received prepared message")
+
+	if recvMsg.BlockNum < consensus.blockNum {
+		consensus.getLogger().Debug().Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Msg("Old Block Received, ignoring!!")
+		return
+	}
+
+	// check validity of prepared signature
+	blockHash := recvMsg.BlockHash
+	aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 0)
+	if err != nil {
+		consensus.getLogger().Error().Err(err).Msg("ReadSignatureBitmapPayload failed!!")
+		return
+	}
+
+	if !consensus.Decider.IsQuorumAchievedByMask(mask, true) {
+		consensus.getLogger().Warn().
+			Msgf("[OnPrepared] Quorum Not achieved")
+		return
+	}
+
+	if !aggSig.VerifyHash(mask.AggregatePublic, blockHash[:]) {
+		myBlockHash := common.Hash{}
+		myBlockHash.SetBytes(consensus.blockHash[:])
+		consensus.getLogger().Warn().
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("MsgViewID", recvMsg.ViewID).
+			Msg("[OnPrepared] failed to verify multi signature for prepare phase")
+		return
+	}
+
+	// check validity of block
+	var blockObj types.Block
+	if err := rlp.DecodeBytes(recvMsg.Block, &blockObj); err != nil {
+		consensus.getLogger().Warn().
+			Err(err).
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Msg("[OnPrepared] Unparseable block header data")
+		return
+	}
+	// let this handle it own logs
+	if !consensus.onPreparedSanityChecks(&blockObj, recvMsg) {
+		return
+	}
+	consensus.mutex.Lock()
+	defer consensus.mutex.Unlock()
+
+	consensus.FBFTLog.AddBlock(&blockObj)
+	// add block field
+	blockPayload := make([]byte, len(recvMsg.Block))
+	copy(blockPayload[:], recvMsg.Block[:])
+	consensus.block = blockPayload
+	recvMsg.Block = []byte{} // save memory space
+	consensus.FBFTLog.AddMessage(recvMsg)
+	consensus.getLogger().Debug().
+		Uint64("MsgViewID", recvMsg.ViewID).
+		Uint64("MsgBlockNum", recvMsg.BlockNum).
+		Hex("blockHash", recvMsg.BlockHash[:]).
+		Msg("[OnPrepared] Prepared message and block added")
+
+	consensus.tryCatchup()
+	if consensus.current.Mode() == ViewChanging {
+		consensus.getLogger().Debug().Msg("[OnPrepared] Still in ViewChanging mode, Exiting!!")
+		return
+	}
+
+	if consensus.checkViewID(recvMsg) != nil {
+		if consensus.current.Mode() == Normal {
+			consensus.getLogger().Debug().
+				Uint64("MsgViewID", recvMsg.ViewID).
+				Uint64("MsgBlockNum", recvMsg.BlockNum).
+				Msg("[OnPrepared] ViewID check failed")
+		}
+		return
+	}
+	if recvMsg.BlockNum > consensus.blockNum {
+		consensus.getLogger().Debug().
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("blockNum", consensus.blockNum).
+			Msg("[OnPrepared] Future Block Received, ignoring!!")
+		return
+	}
+
+	// add preparedSig field
+	consensus.aggregatedPrepareSig = aggSig
+	consensus.prepareBitmap = mask
+
+	// Optimistically add blockhash field of prepare message
+	emptyHash := [32]byte{}
+	if bytes.Compare(consensus.blockHash[:], emptyHash[:]) == 0 {
+		copy(consensus.blockHash[:], blockHash[:])
+	}
+	blockNumBytes := make([]byte, 8)
+	binary.LittleEndian.PutUint64(blockNumBytes, consensus.blockNum)
+	network, _ := consensus.construct(
+		// TODO: should only sign on block hash
+		msg_pb.MessageType_COMMIT,
+		append(blockNumBytes, consensus.blockHash[:]...),
+	)
+	msgToSend := network.Bytes
+
+	// TODO: genesis account node delay for 1 second,
+	// this is a temp fix for allows FN nodes to earning reward
+	if consensus.delayCommit > 0 {
+		time.Sleep(consensus.delayCommit)
+	}
+
+	if err := consensus.msgSender.SendWithoutRetry(
+		[]nodeconfig.GroupID{
+			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))},
+		host.ConstructP2pMessage(byte(17), msgToSend),
+	); err != nil {
+		consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!")
+	} else {
+		consensus.getLogger().Info().
+			Uint64("blockNum", consensus.blockNum).
+			Hex("blockHash", consensus.blockHash[:]).
+			Msg("[OnPrepared] Sent Commit Message!!")
+	}
+
+	consensus.getLogger().Debug().
+		Str("From", consensus.phase.String()).
+		Str("To", FBFTCommit.String()).
+		Msg("[OnPrepared] Switching phase")
+	consensus.switchPhase(FBFTCommit, true)
+}
+
+func (consensus *Consensus) onCommitted(msg *msg_pb.Message) {
+	recvMsg, err := ParseFBFTMessage(msg)
+	if err != nil {
+		consensus.getLogger().Warn().Msg("[OnCommitted] unable to parse msg")
+		return
+	}
+
+	if recvMsg.BlockNum < consensus.blockNum {
+		consensus.getLogger().Info().
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Uint64("blockNum", consensus.blockNum).
+			Msg("[OnCommitted] Received Old Blocks!!")
+		return
+	}
+
+	aggSig, mask, err := consensus.ReadSignatureBitmapPayload(recvMsg.Payload, 0)
+	if err != nil {
+		consensus.getLogger().Error().Err(err).Msg("[OnCommitted] readSignatureBitmapPayload failed")
+		return
+	}
+
+	if !consensus.Decider.IsQuorumAchievedByMask(mask, true) {
+		consensus.getLogger().Warn().
+			Msgf("[OnCommitted] Quorum Not achieved")
+		return
+	}
+
+	blockNumBytes := make([]byte, 8)
+	binary.LittleEndian.PutUint64(blockNumBytes, recvMsg.BlockNum)
+	commitPayload := append(blockNumBytes, recvMsg.BlockHash[:]...)
+	if !aggSig.VerifyHash(mask.AggregatePublic, commitPayload) {
+		consensus.getLogger().Error().
+			Uint64("MsgBlockNum", recvMsg.BlockNum).
+			Msg("[OnCommitted] Failed to verify the multi signature for commit phase")
+		return
+	}
+
+	consensus.FBFTLog.AddMessage(recvMsg)
+	consensus.ChainReader.WriteLastCommits(recvMsg.Payload)
+	consensus.getLogger().Debug().
+		Uint64("MsgViewID", recvMsg.ViewID).
+		Uint64("MsgBlockNum", recvMsg.BlockNum).
+		Msg("[OnCommitted] Committed message added")
+
+	consensus.mutex.Lock()
+	defer consensus.mutex.Unlock()
+
+	consensus.aggregatedCommitSig = aggSig
+	consensus.commitBitmap = mask
+
+	if recvMsg.BlockNum-consensus.blockNum > consensusBlockNumBuffer {
+		consensus.getLogger().Debug().Uint64("MsgBlockNum", recvMsg.BlockNum).Msg("[OnCommitted] out of sync")
+		go func() {
+			select {
+			case consensus.blockNumLowChan <- struct{}{}:
+				consensus.current.SetMode(Syncing)
+				for _, v := range consensus.consensusTimeout {
+					v.Stop()
+				}
+			case <-time.After(1 * time.Second):
+			}
+		}()
+		return
+	}
+
+	consensus.tryCatchup()
+	if consensus.current.Mode() == ViewChanging {
+		consensus.getLogger().Debug().Msg("[OnCommitted] Still in ViewChanging mode, Exiting!!")
+		return
+	}
+
+	if consensus.consensusTimeout[timeoutBootstrap].IsActive() {
+		consensus.consensusTimeout[timeoutBootstrap].Stop()
+		consensus.getLogger().Debug().Msg("[OnCommitted] Start consensus timer; stop bootstrap timer only once")
+	} else {
+		consensus.getLogger().Debug().Msg("[OnCommitted] Start consensus timer")
+	}
+	consensus.consensusTimeout[timeoutConsensus].Start()
+}

consensus/view_change.go

@@ -377,12 +377,11 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 			blockNumBytes := make([]byte, 8)
 			binary.LittleEndian.PutUint64(blockNumBytes, consensus.blockNum)
 			commitPayload := append(blockNumBytes, consensus.blockHash[:]...)
-			consensus.Decider.AddSignature(
+			consensus.Decider.SubmitVote(
 				quorum.Commit,
 				consensus.PubKey,
 				consensus.priKey.SignHash(commitPayload),
-				consensus.LeaderPubKey,
-				consensus.blockNum,
+				nil,
 			)
 
 			if err = consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil {
@@ -533,11 +532,21 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) {
 		// Construct and send the commit message
 		blockNumHash := make([]byte, 8)
 		binary.LittleEndian.PutUint64(blockNumHash, consensus.blockNum)
-		commitPayload := append(blockNumHash, consensus.blockHash[:]...)
-		msgToSend := consensus.constructCommitMessage(commitPayload)
-
+		network, err := consensus.construct(
+			msg_pb.MessageType_COMMIT,
+			append(blockNumHash, consensus.blockHash[:]...),
+		)
+		if err != nil {
+			consensus.getLogger().Err(err).Msg("could not create commit message")
+			return
+		}
+		msgToSend := network.Bytes
 		consensus.getLogger().Info().Msg("onNewView === commit")
-		consensus.host.SendMessageToGroups([]nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, host.ConstructP2pMessage(byte(17), msgToSend))
+		consensus.host.SendMessageToGroups(
+			[]nodeconfig.GroupID{
+				nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))},
+			host.ConstructP2pMessage(byte(17), msgToSend),
+		)
 		consensus.getLogger().Debug().
 			Str("From", consensus.phase.String()).
 			Str("To", FBFTCommit.String()).

consensus/votepower/roster.go

@@ -1,6 +1,7 @@
 package votepower
 
 import (
+	"encoding/hex"
 	"encoding/json"
 	"sort"
 
@@ -24,10 +25,20 @@ var (
 
 // Ballot is a vote cast by a validator
 type Ballot struct {
-	SignerPubKey      shard.BlsPublicKey `json:"bls-public-key"`
-	BlockLeader       shard.BlsPublicKey `json:"leader-when-signed"`
-	BlockHeightHeight uint64             `json:"block-height"`
-	Signature         *bls.Sign          `json:"signature"`
+	SignerPubKey       shard.BlsPublicKey `json:"bls-public-key"`
+	Signature          *bls.Sign          `json:"signature"`
+	OptSerializedBlock []byte             `json:"opt-rlp-encoded-block"`
+}
+
+// BallotResults are a completed round of votes
+type BallotResults struct {
+	Signature shard.BLSSignature // (aggregated) signature
+	Bitmap    []byte             // corresponding bitmap mask for agg signature
+}
+
+// EncodePair returns hex encoded tuple (signature, bitmap)
+func (b BallotResults) EncodePair() (string, string) {
+	return hex.EncodeToString(b.Signature[:]), hex.EncodeToString(b.Bitmap[:])
 }
 
 // Round is a round of voting in any FBFT phase
@@ -207,7 +218,6 @@ func Compute(staked shard.SlotList) (*Roster, error) {
 	roster.OurVotingPowerTotalPercentage = ourPercentage
 	roster.TheirVotingPowerTotalPercentage = theirPercentage
 	return roster, nil
-
 }
 
 // NewRoster ..

core/blockchain.go

@@ -50,6 +50,7 @@ import (
 	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/shard/committee"
+	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
 	lru "github.com/hashicorp/golang-lru"
 )
@@ -79,11 +80,17 @@ const (
 	validatorListByDelegatorCacheLimit = 1024
 	pendingCrossLinksCacheLimit        = 2
 	blockAccumulatorCacheLimit         = 256
+	pendingSlashingCandidateCacheLimit = 2
 
 	// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
 	BlockChainVersion = 3
 )
 
+const (
+	pendingCLCacheKey = "pendingCLs"
+	pendingSCCacheKey = "pendingSCs"
+)
+
 // CacheConfig contains the configuration values for the trie caching/pruning
 // that's resident in a blockchain.
 type CacheConfig struct {
@@ -123,10 +130,11 @@ type BlockChain struct {
 	scope         event.SubscriptionScope
 	genesisBlock  *types.Block
 
-	mu                     sync.RWMutex // global mutex for locking chain operations
-	chainmu                sync.RWMutex // blockchain insertion lock
-	procmu                 sync.RWMutex // block processor lock
-	pendingCrossLinksMutex sync.RWMutex // pending crosslinks lock
+	mu                          sync.RWMutex // global mutex for locking chain operations
+	chainmu                     sync.RWMutex // blockchain insertion lock
+	procmu                      sync.RWMutex // block processor lock
+	pendingCrossLinksMutex      sync.RWMutex // pending crosslinks lock
+	pendingSlashingCandidatesMU sync.RWMutex // pending slashing candidates
 
 	checkpoint       int          // checkpoint counts towards the new checkpoint
 	currentBlock     atomic.Value // Current head of the block chain
@@ -148,6 +156,7 @@ type BlockChain struct {
 	validatorListByDelegatorCache *lru.Cache // Cache of validator list by delegator
 	pendingCrossLinksCache        *lru.Cache // Cache of last pending crosslinks
 	blockAccumulatorCache         *lru.Cache // Cache of block accumulators
+	pendingSlashingCandidates     *lru.Cache // Cache of last pending slashing candidates
 
 	quit    chan struct{} // blockchain quit channel
 	running int32         // running must be called atomically
@@ -190,6 +199,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	validatorListByDelegatorCache, _ := lru.New(validatorListByDelegatorCacheLimit)
 	pendingCrossLinksCache, _ := lru.New(pendingCrossLinksCacheLimit)
 	blockAccumulatorCache, _ := lru.New(blockAccumulatorCacheLimit)
+	pendingSlashingCandidateCache, _ := lru.New(pendingSlashingCandidateCacheLimit)
 
 	bc := &BlockChain{
 		chainConfig:                   chainConfig,
@@ -213,6 +223,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		validatorListCache:            validatorListCache,
 		validatorListByDelegatorCache: validatorListByDelegatorCache,
 		pendingCrossLinksCache:        pendingCrossLinksCache,
+		pendingSlashingCandidates:     pendingSlashingCandidateCache,
 		blockAccumulatorCache:         blockAccumulatorCache,
 		engine:                        engine,
 		vmConfig:                      vmConfig,
@@ -2192,10 +2203,59 @@ func (bc *BlockChain) ReadShardLastCrossLink(shardID uint32) (*types.CrossLink,
 	return crossLink, err
 }
 
+// ReadPendingSlashingCandidates retrieves pending slashing candidates
+func (bc *BlockChain) ReadPendingSlashingCandidates() ([]slash.Record, error) {
+	if !bc.Config().IsStaking(bc.CurrentHeader().Epoch()) {
+		return []slash.Record{}, nil
+	}
+
+	bytes := []byte{}
+	if cached, ok := bc.pendingSlashingCandidates.Get(pendingSCCacheKey); ok {
+		bytes = cached.([]byte)
+	} else {
+		bytes, err := rawdb.ReadPendingSlashingCandidates(bc.db)
+		if err != nil || len(bytes) == 0 {
+			utils.Logger().Info().Err(err).
+				Int("dataLen", len(bytes)).
+				Msg("ReadPendingSlashingCandidates")
+			return nil, err
+		}
+	}
+	cls := []slash.Record{}
+	if err := rlp.DecodeBytes(bytes, &cls); err != nil {
+		utils.Logger().Error().Err(err).Msg("Invalid pending slashing candidates RLP decoding")
+		return nil, err
+	}
+	return cls, nil
+}
+
+// WritePendingSlashingCandidates saves the pending slashing candidates
+func (bc *BlockChain) WritePendingSlashingCandidates(candidates []slash.Record) error {
+	if !bc.Config().IsStaking(bc.CurrentHeader().Epoch()) {
+		utils.Logger().Debug().Msg("Writing slashing candidates in prior to staking epoch")
+		return nil
+	}
+
+	bytes, err := rlp.EncodeToBytes(candidates)
+	if err != nil {
+		const msg = "[WritePendingSlashingCandidates] Failed to encode pending slashing candidates"
+		utils.Logger().Error().Msg(msg)
+		return err
+	}
+	if err := rawdb.WritePendingSlashingCandidates(bc.db, bytes); err != nil {
+		return err
+	}
+	if by, err := rlp.EncodeToBytes(candidates); err == nil {
+		bc.pendingSlashingCandidates.Add(pendingSCCacheKey, by)
+	}
+	return nil
+
+}
+
 // ReadPendingCrossLinks retrieves pending crosslinks
 func (bc *BlockChain) ReadPendingCrossLinks() ([]types.CrossLink, error) {
 	bytes := []byte{}
-	if cached, ok := bc.pendingCrossLinksCache.Get("pendingCLs"); ok {
+	if cached, ok := bc.pendingCrossLinksCache.Get(pendingCLCacheKey); ok {
 		bytes = cached.([]byte)
 	} else {
 		bytes, err := rawdb.ReadPendingCrossLinks(bc.db)
@@ -2241,9 +2301,26 @@ func (bc *BlockChain) WritePendingCrossLinks(crossLinks []types.CrossLink) error
 	}
 	by, err := rlp.EncodeToBytes(cls)
 	if err == nil {
-		bc.pendingCrossLinksCache.Add("pendingCLs", by)
+		bc.pendingCrossLinksCache.Add(pendingCLCacheKey, by)
 	}
 	return nil
+
+}
+
+// AddPendingSlashingCandidate  appends pending slashing candidates
+func (bc *BlockChain) AddPendingSlashingCandidate(candidate *slash.Record) (int, error) {
+	bc.pendingSlashingCandidatesMU.Lock()
+	defer bc.pendingSlashingCandidatesMU.Unlock()
+
+	cls, err := bc.ReadPendingSlashingCandidates()
+	if err != nil || len(cls) == 0 {
+		err := bc.WritePendingSlashingCandidates([]slash.Record{*candidate})
+		return 1, err
+	}
+	cls = append(cls, *candidate)
+	err = bc.WritePendingSlashingCandidates(cls)
+	return len(cls), err
+
 }
 
 // AddPendingCrossLinks appends pending crosslinks

core/rawdb/accessors_chain.go

@@ -541,6 +541,21 @@ func DeletePendingCrossLinks(db DatabaseDeleter) error {
 	return db.Delete(pendingCrosslinkKey)
 }
 
+// ReadPendingSlashingCandidates retrieves last pending slashing candidates
+func ReadPendingSlashingCandidates(db DatabaseReader) ([]byte, error) {
+	return db.Get(pendingSlashingKey)
+}
+
+// WritePendingSlashingCandidates stores last pending slashing candidates into database.
+func WritePendingSlashingCandidates(db DatabaseWriter, bytes []byte) error {
+	return db.Put(pendingSlashingKey, bytes)
+}
+
+// DeletePendingSlashingCandidates stores last pending slashing candidates into database.
+func DeletePendingSlashingCandidates(db DatabaseDeleter) error {
+	return db.Delete(pendingSlashingKey)
+}
+
 // ReadCXReceipts retrieves all the transactions of receipts given destination shardID, number and blockHash
 func ReadCXReceipts(db DatabaseReader, shardID uint32, number uint64, hash common.Hash) (types.CXReceipts, error) {
 	data, err := db.Get(cxReceiptKey(shardID, number, hash))

core/rawdb/schema.go

@@ -58,10 +58,10 @@ var (
 	shardStatePrefix = []byte("ss") // shardStatePrefix + num (uint64 big endian) + hash -> shardState
 	lastCommitsKey   = []byte("LastCommits")
 
-	pendingCrosslinkKey = []byte("pendingCL") // prefix for shard last pending crosslink
-
-	preimagePrefix = []byte("secure-key-")      // preimagePrefix + hash -> preimage
-	configPrefix   = []byte("ethereum-config-") // config prefix for the db
+	pendingCrosslinkKey = []byte("pendingCL")        // prefix for shard last pending crosslink
+	pendingSlashingKey  = []byte("pendingSC")        // prefix for shard last pending slashing record
+	preimagePrefix      = []byte("secure-key-")      // preimagePrefix + hash -> preimage
+	configPrefix        = []byte("ethereum-config-") // config prefix for the db
 
 	shardLastCrosslinkPrefix = []byte("lcl") // prefix for shard last crosslink
 	crosslinkPrefix          = []byte("cl")  // prefix for crosslink

go.mod

@@ -71,4 +71,5 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
+	gopkg.in/yaml.v2 v2.2.7
 )

internal/chain/engine.go

@@ -29,12 +29,11 @@ import (
 
 type engineImpl struct {
 	d      reward.Distributor
-	s      slash.Slasher
 	beacon engine.ChainReader
 }
 
 // Engine is an algorithm-agnostic consensus engine.
-var Engine = &engineImpl{nil, nil, nil}
+var Engine = &engineImpl{nil, nil}
 
 // Rewarder handles the distribution of block rewards
 func (e *engineImpl) Rewarder() reward.Distributor {
@@ -46,21 +45,11 @@ func (e *engineImpl) SetRewarder(d reward.Distributor) {
 	e.d = d
 }
 
-// Slasher handles slashing accounts due to inavailibility or double-signing
-func (e *engineImpl) Slasher() slash.Slasher {
-	return e.s
-}
-
-// SetSlasher assigns the slasher used
-func (e *engineImpl) SetSlasher(s slash.Slasher) {
-	e.s = s
-}
-
 func (e *engineImpl) Beaconchain() engine.ChainReader {
 	return e.beacon
 }
 
-// SetSlasher assigns the slasher used
+// SetBeaconchain assigns the beaconchain handle used
 func (e *engineImpl) SetBeaconchain(beaconchain engine.ChainReader) {
 	e.beacon = beaconchain
 }
@@ -269,7 +258,7 @@ func (e *engineImpl) Finalize(
 	// Accumulate any block and uncle rewards and commit the final state root
 	// Header seems complete, assemble into a block and return
 	payout, err := AccumulateRewards(
-		chain, state, header, e.Rewarder(), e.Slasher(), e.Beaconchain(),
+		chain, state, header, e.Rewarder(), e.Beaconchain(),
 	)
 	if err != nil {
 		return nil, nil, ctxerror.New("cannot pay block reward").WithCause(err)
@@ -277,6 +266,13 @@ func (e *engineImpl) Finalize(
 	isBeaconChain := header.ShardID() == shard.BeaconChainShardID
 	isNewEpoch := len(header.ShardState()) > 0
 	inStakingEra := chain.Config().IsStaking(header.Epoch())
+	// Apply the slashes, invariant: assume been verified as legit slash by this point
+	if isBeaconChain && isNewEpoch && inStakingEra {
+		if err := slash.Apply(state, header.Slashes()); err != nil {
+			return nil, nil, ctxerror.New("[Finalize] could not apply slash").WithCause(err)
+		}
+	}
+
 	// Withdraw unlocked tokens to the delegators' accounts
 	// Only do such at the last block of an epoch
 	if isBeaconChain && isNewEpoch && inStakingEra {
@@ -309,23 +305,13 @@ func (e *engineImpl) Finalize(
 		if err != nil {
 			return nil, nil, ctxerror.New("[Finalize] failed to read shard state").WithCause(err)
 		}
-		processed := make(map[common.Address]struct{})
-		for i := range newShardState.Shards {
-			shard := newShardState.Shards[i]
-			for j := range shard.Slots {
-				slot := shard.Slots[j]
-				if slot.EffectiveStake != nil { // For external validator
-					_, ok := processed[slot.EcdsaAddress]
-					if !ok {
-						processed[slot.EcdsaAddress] = struct{}{}
-						wrapper := state.GetStakingInfo(slot.EcdsaAddress)
-						wrapper.LastEpochInCommittee = newShardState.Epoch
-
-						if err := state.UpdateStakingInfo(slot.EcdsaAddress, wrapper); err != nil {
-							return nil, nil, ctxerror.New("[Finalize] failed update validator info").WithCause(err)
-						}
-					}
-				}
+		for _, external := range newShardState.ExternalValidators() {
+			wrapper := state.GetStakingInfo(external)
+			wrapper.LastEpochInCommittee = newShardState.Epoch
+			if err := state.UpdateStakingInfo(external, wrapper); err != nil {
+				return nil, nil, ctxerror.New(
+					"[Finalize] failed update validator info",
+				).WithCause(err)
 			}
 		}
 	}

internal/chain/reward.go

@@ -18,7 +18,6 @@ import (
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/staking/availability"
 	"github.com/harmony-one/harmony/staking/network"
-	"github.com/harmony-one/harmony/staking/slash"
 	"github.com/pkg/errors"
 )
 
@@ -32,8 +31,7 @@ func ballotResultBeaconchain(
 // reward. The total reward consists of the static block reward
 func AccumulateRewards(
 	bc engine.ChainReader, state *state.DB, header *block.Header,
-	rewarder reward.Distributor, slasher slash.Slasher,
-	beaconChain engine.ChainReader,
+	rewarder reward.Distributor, beaconChain engine.ChainReader,
 ) (*big.Int, error) {
 	blockNum := header.Number().Uint64()
 

internal/configs/node/config.go

@@ -12,6 +12,7 @@ import (
 	shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding"
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/shard"
+	"github.com/harmony-one/harmony/staking/slash"
 	p2p_crypto "github.com/libp2p/go-libp2p-crypto"
 	"github.com/pkg/errors"
 )
@@ -88,6 +89,9 @@ type ConfigType struct {
 	DBDir            string
 	networkType      NetworkType
 	shardingSchedule shardingconfig.Schedule
+	WebHooks         struct {
+		DoubleSigning *slash.DoubleSignWebHooks
+	}
 }
 
 // configs is a list of node configuration.

node/node.go

@@ -36,7 +36,6 @@ import (
 	p2p_host "github.com/harmony-one/harmony/p2p/host"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/shard/committee"
-	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
 )
 
@@ -267,7 +266,9 @@ func (node *Node) tryBroadcast(tx *types.Transaction) {
 func (node *Node) tryBroadcastStaking(stakingTx *staking.StakingTransaction) {
 	msg := proto_node.ConstructStakingTransactionListMessageAccount(staking.StakingTransactions{stakingTx})
 
-	shardGroupID := nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(0)) // broadcast to beacon chain
+	shardGroupID := nodeconfig.NewGroupIDByShardID(
+		nodeconfig.ShardID(shard.BeaconChainShardID),
+	) // broadcast to beacon chain
 	utils.Logger().Info().Str("shardGroupID", string(shardGroupID)).Msg("tryBroadcastStaking")
 
 	for attempt := 0; attempt < NumTryBroadCast; attempt++ {
@@ -528,7 +529,6 @@ func New(host p2p.Host, consensusObj *consensus.Consensus,
 		node.pendingStakingTransactions = make(map[common.Hash]*staking.StakingTransaction)
 		node.Consensus.VerifiedNewBlock = make(chan *types.Block)
 		chain.Engine.SetRewarder(node.Consensus.Decider.(reward.Distributor))
-		chain.Engine.SetSlasher(node.Consensus.Decider.(slash.Slasher))
 		chain.Engine.SetBeaconchain(beaconChain)
 
 		// the sequence number is the next block number to be added in consensus protocol, which is
@@ -564,6 +564,17 @@ func New(host p2p.Host, consensusObj *consensus.Consensus,
 	node.peerRegistrationRecord = make(map[string]*syncConfig)
 	node.startConsensus = make(chan struct{})
 	go node.bootstrapConsensus()
+	// Broadcast double-signers reported by consensus
+	if node.Consensus != nil {
+		go func() {
+			for {
+				select {
+				case doubleSign := <-node.Consensus.SlashChan:
+					go node.BroadcastSlash(&doubleSign)
+				}
+			}
+		}()
+	}
 	return &node
 }
 

node/node_cross_link.go

@@ -62,7 +62,7 @@ func (node *Node) VerifyBlockCrossLinks(block *types.Block) error {
 
 // ProcessCrossLinkMessage verify and process Node/CrossLink message into crosslink when it's valid
 func (node *Node) ProcessCrossLinkMessage(msgPayload []byte) {
-	if node.NodeConfig.ShardID == 0 {
+	if node.NodeConfig.ShardID == shard.BeaconChainShardID {
 		pendingCLs, err := node.Blockchain().ReadPendingCrossLinks()
 		if err == nil && len(pendingCLs) >= maxPendingCrossLinkSize {
 			utils.Logger().Debug().

node/node_double_signing.go

@@ -0,0 +1,24 @@
+package node
+
+import (
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/shard"
+	"github.com/harmony-one/harmony/staking/slash"
+)
+
+// ProcessSlashCandidateMessage ..
+func (node *Node) processSlashCandidateMessage(msgPayload []byte) {
+	if node.NodeConfig.ShardID != shard.BeaconChainShardID {
+		return
+	}
+
+	candidate := slash.Record{}
+	if err := rlp.DecodeBytes(msgPayload, &candidate); err != nil {
+		utils.Logger().Error().
+			Err(err).
+			Msg("unable to decode slash candidate message")
+		return
+	}
+	node.Blockchain().AddPendingSlashingCandidate(&candidate)
+}

node/node_handler.go

@@ -59,6 +59,32 @@ func (node *Node) receiveGroupMessage(
 	}
 }
 
+// some messages have uninteresting fields in header, slash, receipt and crosslink are
+// such messages. This function assumes that input bytes are a slice which already
+// past those not relevant header bytes.
+func (node *Node) processSkippedMsgTypeByteValue(cat proto_node.BlockMessageType, content []byte) {
+	switch cat {
+	case proto_node.SlashCandidate:
+		node.processSlashCandidateMessage(content)
+	case proto_node.Receipt:
+		utils.Logger().Debug().Msg("NET: received message: Node/Receipt")
+		node.ProcessReceiptMessage(content)
+	case proto_node.CrossLink:
+		// only beacon chain will accept the header from other shards
+		utils.Logger().Debug().
+			Uint32("shardID", node.NodeConfig.ShardID).
+			Msg("NET: received message: Node/CrossLink")
+		if node.NodeConfig.ShardID != shard.BeaconChainShardID {
+			return
+		}
+		node.ProcessCrossLinkMessage(content)
+	default:
+		utils.Logger().Error().
+			Int("message-iota-value", int(cat)).
+			Msg("Invariant usage of processSkippedMsgTypeByteValue violated")
+	}
+}
+
 // HandleMessage parses the message and dispatch the actions.
 func (node *Node) HandleMessage(content []byte, sender libp2p_peer.ID) {
 	msgCategory, err := proto.GetMessageCategory(content)
@@ -116,8 +142,8 @@ func (node *Node) HandleMessage(content []byte, sender libp2p_peer.ID) {
 				utils.Logger().Debug().Msgf("Invalid block message size")
 				return
 			}
-			blockMsgType := proto_node.BlockMessageType(msgPayload[0])
-			switch blockMsgType {
+
+			switch blockMsgType := proto_node.BlockMessageType(msgPayload[0]); blockMsgType {
 			case proto_node.Sync:
 				utils.Logger().Debug().Msg("NET: received message: Node/Sync")
 				var blocks []*types.Block
@@ -143,19 +169,12 @@ func (node *Node) HandleMessage(content []byte, sender libp2p_peer.ID) {
 						node.Client.UpdateBlocks(blocks)
 					}
 				}
-
-			case proto_node.CrossLink:
-				// only beacon chain will accept the header from other shards
-				utils.Logger().Debug().Uint32("shardID", node.NodeConfig.ShardID).Msg("NET: received message: Node/CrossLink")
-				if node.NodeConfig.ShardID != 0 {
-					return
-				}
-				node.ProcessCrossLinkMessage(msgPayload[1:]) // skip first byte which is blockMsgType
-
-			case proto_node.Receipt:
-				utils.Logger().Debug().Msg("NET: received message: Node/Receipt")
-				node.ProcessReceiptMessage(msgPayload[1:]) // skip first byte which is blockMsgType
-
+			case
+				proto_node.SlashCandidate,
+				proto_node.Receipt,
+				proto_node.CrossLink:
+				// skip first byte which is blockMsgType
+				node.processSkippedMsgTypeByteValue(blockMsgType, msgPayload[1:])
 			}
 		case proto_node.PING:
 			node.pingMessageHandler(msgPayload, sender)
@@ -222,7 +241,24 @@ func (node *Node) BroadcastNewBlock(newBlock *types.Block) {
 
 // BroadcastSlash ..
 func (node *Node) BroadcastSlash(witness *slash.Record) {
-	//
+	// no point to broadcast the crosslink if we aren't even in the right epoch yet
+	if !node.Blockchain().Config().IsCrossLink(
+		node.Blockchain().CurrentHeader().Epoch(),
+	) {
+		return
+	}
+
+	// Send it to beaconchain if I'm shardchain, otherwise just add it to pending
+	if node.NodeConfig.ShardID != shard.BeaconChainShardID {
+		node.host.SendMessageToGroups(
+			[]nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(shard.BeaconChainShardID)},
+			host.ConstructP2pMessage(
+				byte(0),
+				proto_node.ConstructSlashMessage(witness)),
+		)
+	} else {
+		node.Blockchain().AddPendingSlashingCandidate(witness)
+	}
 }
 
 // BroadcastCrossLink is called by consensus leader to send the new header as cross link to beacon chain.

node/node_handler_test.go

@@ -41,7 +41,9 @@ func TestAddNewBlock(t *testing.T) {
 		txs, stks, common.Address{},
 		func([]staking.RPCTransactionError) {},
 	)
-	block, _ := node.Worker.FinalizeNewBlock([]byte{}, []byte{}, 0, common.Address{}, nil, nil)
+	block, _ := node.Worker.FinalizeNewBlock(
+		[]byte{}, []byte{}, 0, common.Address{}, nil, nil, nil,
+	)
 
 	_, err = node.Blockchain().InsertChain([]*types.Block{block}, true)
 	if err != nil {
@@ -77,7 +79,9 @@ func TestVerifyNewBlock(t *testing.T) {
 		txs, stks, common.Address{},
 		func([]staking.RPCTransactionError) {},
 	)
-	block, _ := node.Worker.FinalizeNewBlock([]byte{}, []byte{}, 0, common.Address{}, nil, nil)
+	block, _ := node.Worker.FinalizeNewBlock(
+		[]byte{}, []byte{}, 0, common.Address{}, nil, nil, nil,
+	)
 
 	if err := node.VerifyNewBlock(block); err != nil {
 		t.Error("New block is not verified successfully:", err)

node/node_newblock.go

@@ -10,6 +10,7 @@ import (
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/shard"
+	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
 )
 
@@ -144,7 +145,10 @@ func (node *Node) proposeNewBlock() (*types.Block, error) {
 	}
 
 	// Prepare cross links
-	var crossLinksToPropose types.CrossLinks
+	var (
+		slashingToPropose   []slash.Record
+		crossLinksToPropose types.CrossLinks
+	)
 
 	if node.NodeConfig.ShardID == shard.BeaconChainShardID &&
 		node.Blockchain().Config().IsCrossLink(node.Worker.GetCurrentHeader().Epoch()) {
@@ -182,7 +186,9 @@ func (node *Node) proposeNewBlock() (*types.Block, error) {
 		return nil, err
 	}
 	return node.Worker.FinalizeNewBlock(
-		sig, mask, node.Consensus.GetViewID(), coinbase, crossLinksToPropose, shardState,
+		sig, mask, node.Consensus.GetViewID(),
+		coinbase, crossLinksToPropose, shardState,
+		slashingToPropose,
 	)
 }
 
@@ -204,8 +210,13 @@ func (node *Node) proposeReceiptsProof() []*types.CXReceiptsProof {
 		pendingCXReceipts = append(pendingCXReceipts, v)
 	}
 
-	sort.Slice(pendingCXReceipts, func(i, j int) bool {
-		return pendingCXReceipts[i].MerkleProof.ShardID < pendingCXReceipts[j].MerkleProof.ShardID || (pendingCXReceipts[i].MerkleProof.ShardID == pendingCXReceipts[j].MerkleProof.ShardID && pendingCXReceipts[i].MerkleProof.BlockNum.Cmp(pendingCXReceipts[j].MerkleProof.BlockNum) < 0)
+	sort.SliceStable(pendingCXReceipts, func(i, j int) bool {
+		shardCMP := pendingCXReceipts[i].MerkleProof.ShardID < pendingCXReceipts[j].MerkleProof.ShardID
+		shardEQ := pendingCXReceipts[i].MerkleProof.ShardID == pendingCXReceipts[j].MerkleProof.ShardID
+		blockCMP := pendingCXReceipts[i].MerkleProof.BlockNum.Cmp(
+			pendingCXReceipts[j].MerkleProof.BlockNum,
+		) == -1
+		return shardCMP || (shardEQ && blockCMP)
 	})
 
 	m := make(map[common.Hash]bool)

node/worker/worker.go

@@ -1,8 +1,10 @@
 package worker
 
 import (
+	"bytes"
 	"fmt"
 	"math/big"
+	"sort"
 	"time"
 
 	common2 "github.com/harmony-one/harmony/internal/common"
@@ -20,6 +22,7 @@ import (
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/shard"
+	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
 )
 
@@ -310,7 +313,7 @@ func (w *Worker) IncomingReceipts() []*types.CXReceiptsProof {
 // FinalizeNewBlock generate a new block for the next consensus round.
 func (w *Worker) FinalizeNewBlock(
 	sig []byte, signers []byte, viewID uint64, coinbase common.Address,
-	crossLinks types.CrossLinks, shardState *shard.State,
+	crossLinks types.CrossLinks, shardState *shard.State, doubleSigners []slash.Record,
 ) (*types.Block, error) {
 	if len(sig) > 0 && len(signers) > 0 {
 		sig2 := w.current.header.LastCommitSignature()
@@ -321,6 +324,21 @@ func (w *Worker) FinalizeNewBlock(
 	w.current.header.SetCoinbase(coinbase)
 	w.current.header.SetViewID(new(big.Int).SetUint64(viewID))
 
+	// Slashes
+	if d := doubleSigners; d != nil && len(d) != 0 {
+		// Enforce order, reproducibility
+		sort.SliceStable(d,
+			func(i, j int) bool {
+				return bytes.Compare(
+					d[i].Beneficiary.Bytes(), d[j].Beneficiary.Bytes(),
+				) == -1
+			},
+		)
+		if rlpBytes, err := rlp.EncodeToBytes(d); err == nil {
+			w.current.header.SetSlashes(rlpBytes)
+		}
+	}
+
 	// Cross Links
 	if crossLinks != nil && len(crossLinks) != 0 {
 		crossLinks.Sort()
@@ -360,7 +378,6 @@ func (w *Worker) FinalizeNewBlock(
 	}
 
 	state := w.current.state.Copy()
-
 	copyHeader := types.CopyHeader(w.current.header)
 	// TODO: feed coinbase into here so the proposer gets extra rewards.
 	block, _, err := w.engine.Finalize(

shard/shard_state.go

@@ -23,7 +23,7 @@ var (
 // PublicKeySizeInBytes ..
 const (
 	PublicKeySizeInBytes    = 48
-	BlsSignatureSizeInBytes = 96
+	BLSSignatureSizeInBytes = 96
 )
 
 // State is the collection of all committees
@@ -35,8 +35,8 @@ type State struct {
 // BlsPublicKey defines the bls public key
 type BlsPublicKey [PublicKeySizeInBytes]byte
 
-// BlsSignature defines the bls signature
-type BlsSignature [BlsSignatureSizeInBytes]byte
+// BLSSignature defines the bls signature
+type BLSSignature [BLSSignatureSizeInBytes]byte
 
 // Slot represents node id (BLS address)
 type Slot struct {
@@ -135,6 +135,31 @@ func EncodeWrapper(shardState State, isStaking bool) ([]byte, error) {
 	return data, err
 }
 
+// ExternalValidators returns only the staking era,
+// external validators aka non-harmony nodes
+func (ss *State) ExternalValidators() []common.Address {
+	processed := make(map[common.Address]struct{})
+	for i := range ss.Shards {
+		shard := ss.Shards[i]
+		for j := range shard.Slots {
+			slot := shard.Slots[j]
+			if slot.EffectiveStake != nil { // For external validator
+				_, ok := processed[slot.EcdsaAddress]
+				if !ok {
+					processed[slot.EcdsaAddress] = struct{}{}
+
+				}
+			}
+		}
+	}
+	slice, i := make([]common.Address, len(processed)), 0
+	for key := range processed {
+		slice[i] = key
+		i++
+	}
+	return slice
+}
+
 // JSON produces a non-pretty printed JSON string of the SuperCommittee
 func (ss *State) JSON() string {
 	type t struct {

staking/slash/double-sign.go

@@ -1,30 +1,52 @@
 package slash
 
 import (
-	"math/big"
-
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/harmony-one/harmony/consensus/quorum"
+	"github.com/harmony-one/bls/ffi/go/bls"
+	"github.com/harmony-one/harmony/block"
+	"github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/shard"
 )
 
-// Slasher ..
-type Slasher interface {
-	ShouldSlash(shard.BlsPublicKey) bool
-}
-
 // Record is an proof of a slashing made by a witness of a double-signing event
 type Record struct {
-	BlockHash   common.Hash
-	BlockNumber *big.Int
-	Signature   [96]byte // (aggregated) signature
-	Bitmap      []byte   // corresponding bitmap mask for agg signature
-	ShardID     uint32
-	Epoch       *big.Int
+	Offender shard.BlsPublicKey
+	Signed   struct {
+		Header    *block.Header
+		Signature *bls.Sign
+	} `json:"signed"`
+	DoubleSigned struct {
+		Header    *block.Header
+		Signature *bls.Sign
+	} `json:"double-signed"`
 	Beneficiary common.Address // the reporter who will get rewarded
 }
 
-// DidAnyoneDoubleSign ..
-func DidAnyoneDoubleSign(d quorum.Decider) bool {
-	return false
+// NewRecord ..
+func NewRecord(
+	offender shard.BlsPublicKey,
+	signedHeader, doubleSignedHeader *block.Header,
+	signedSignature, doubleSignedSignature *bls.Sign,
+	beneficiary common.Address,
+) Record {
+	r := Record{}
+	r.Offender = offender
+	r.Signed.Header = signedHeader
+	r.Signed.Signature = signedSignature
+	r.DoubleSigned.Header = doubleSignedHeader
+	r.DoubleSigned.Signature = doubleSignedSignature
+	r.Beneficiary = beneficiary
+	return r
+}
+
+// TODO(Edgar) Implement Verify and Apply
+
+// Verify checks that the signature is valid
+func Verify(candidate *Record) error {
+	return nil
+}
+
+// Apply ..
+func Apply(state *state.DB, slashes []byte) error {
+	return nil
 }

staking/slash/report.go

@@ -0,0 +1,28 @@
+package slash
+
+import (
+	"io/ioutil"
+
+	"gopkg.in/yaml.v2"
+)
+
+// DoubleSignWebHooks ..
+type DoubleSignWebHooks struct {
+	WebHooks struct {
+		OnNoticeDoubleSign     string `yaml:"notice-double-sign"`
+		OnThisNodeDoubleSigned string `yaml:"this-node-double-signed"`
+	} `yaml:"web-hooks"`
+}
+
+// NewDoubleSignWebHooksFromPath ..
+func NewDoubleSignWebHooksFromPath(yamlPath string) (*DoubleSignWebHooks, error) {
+	rawYAML, err := ioutil.ReadFile(yamlPath)
+	if err != nil {
+		return nil, err
+	}
+	t := DoubleSignWebHooks{}
+	if err := yaml.UnmarshalStrict(rawYAML, &t); err != nil {
+		return nil, err
+	}
+	return &t, nil
+}

staking/types/messages.go

@@ -60,7 +60,7 @@ type CreateValidator struct {
 	MinSelfDelegation  *big.Int             `json:"min_self_delegation"`
 	MaxTotalDelegation *big.Int             `json:"max_total_delegation"`
 	SlotPubKeys        []shard.BlsPublicKey `json:"slot_pub_keys"`
-	SlotKeySigs        []shard.BlsSignature `json:"slot_key_sigs"`
+	SlotKeySigs        []shard.BLSSignature `json:"slot_key_sigs"`
 	Amount             *big.Int             `json:"amount"`
 }
 
@@ -73,7 +73,7 @@ type EditValidator struct {
 	MaxTotalDelegation *big.Int            `json:"max_total_delegation" rlp:"nil"`
 	SlotKeyToRemove    *shard.BlsPublicKey `json:"slot_key_to_remove" rlp:"nil"`
 	SlotKeyToAdd       *shard.BlsPublicKey `json:"slot_key_to_add" rlp:"nil"`
-	SlotKeyToAddSig    *shard.BlsSignature `json:"slot_key_to_add_sig" rlp:"nil"`
+	SlotKeyToAddSig    *shard.BLSSignature `json:"slot_key_to_add_sig" rlp:"nil"`
 	Active             *bool               `json:"active" rlp:"nil"`
 }
 

staking/types/validator.go

@@ -350,7 +350,7 @@ func (v *Validator) GetCommissionRate() numeric.Dec { return v.Commission.Rate }
 // GetMinSelfDelegation returns the minimum amount the validator must stake
 func (v *Validator) GetMinSelfDelegation() *big.Int { return v.MinSelfDelegation }
 
-func verifyBLSKeys(pubKeys []shard.BlsPublicKey, pubKeySigs []shard.BlsSignature) error {
+func verifyBLSKeys(pubKeys []shard.BlsPublicKey, pubKeySigs []shard.BLSSignature) error {
 	if len(pubKeys) != len(pubKeySigs) {
 		return errBLSKeysNotMatchSigs
 	}
@@ -364,7 +364,7 @@ func verifyBLSKeys(pubKeys []shard.BlsPublicKey, pubKeySigs []shard.BlsSignature
 	return nil
 }
 
-func verifyBLSKey(pubKey *shard.BlsPublicKey, pubKeySig *shard.BlsSignature) error {
+func verifyBLSKey(pubKey *shard.BlsPublicKey, pubKeySig *shard.BLSSignature) error {
 	if len(pubKeySig) == 0 {
 		return errBLSKeysNotMatchSigs
 	}

test/chain/main.go

@@ -134,7 +134,8 @@ func fundFaucetContract(chain *core.BlockChain) {
 	if err != nil {
 		fmt.Println(err)
 	}
-	block, _ := contractworker.FinalizeNewBlock([]byte{}, []byte{}, 0, common.Address{}, nil, nil)
+	block, _ := contractworker.
+		FinalizeNewBlock([]byte{}, []byte{}, 0, common.Address{}, nil, nil, nil)
 	_, err = chain.InsertChain(types.Blocks{block}, true /* verifyHeaders */)
 	if err != nil {
 		fmt.Println(err)
@@ -179,7 +180,9 @@ func callFaucetContractToFundAnAddress(chain *core.BlockChain) {
 	if err != nil {
 		fmt.Println(err)
 	}
-	block, _ := contractworker.FinalizeNewBlock([]byte{}, []byte{}, 0, common.Address{}, nil, nil)
+	block, _ := contractworker.FinalizeNewBlock(
+		[]byte{}, []byte{}, 0, common.Address{}, nil, nil, nil,
+	)
 	_, err = chain.InsertChain(types.Blocks{block}, true /* verifyHeaders */)
 	if err != nil {
 		fmt.Println(err)