multi-bls key support (#2249)

refactoring to multibls package to avoid cyclic dependency and better structure fixing import issue fixing more imports fixing a bug that causes nil pointer for explorer node fixing a bug in the super majority vote changing config to have same epoch for cross link and staking fixing the compilation error addressing PR comments leaving todo for assuming order between pub/pri keys pairs fixing goimports removing unwanted warning message
5 years ago · ae362c24c6
parent b7a8c506eb
commit ae362c24c6
28 changed files with 459 additions and 236 deletions
--- a/cmd/harmony/main.go
+++ b/cmd/harmony/main.go
@ -1,7 +1,6 @@
 package main

 import (
-	"encoding/hex"
 	"flag"
 	"fmt"
 	"io/ioutil"
@ -10,6 +9,7 @@ import (
 	"os"
 	"os/signal"
 	"path"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@ -32,6 +32,7 @@ import (
 	"github.com/harmony-one/harmony/internal/memprofiling"
 	"github.com/harmony-one/harmony/internal/shardchain"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/node"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
@ -101,6 +102,7 @@ var (
 	enableMemProfiling = flag.Bool("enableMemProfiling", false, "Enable memsize logging.")
 	enableGC           = flag.Bool("enableGC", true, "Enable calling garbage collector manually .")
 	blsKeyFile         = flag.String("blskey_file", "", "The encrypted file of bls serialized private key by passphrase.")
+	blsFolder          = flag.String("blsfolder", ".hmy/blskeys", "The folder that stores the bls keys; same blspass is used to decrypt all bls keys; all bls keys mapped to same shard")
 	blsPass            = flag.String("blspass", "", "The file containing passphrase to decrypt the encrypted bls file.")
 	blsPassphrase      string
 	// Sharding configuration parameters for devnet
@ -114,7 +116,7 @@ var (
 	// Use a separate log file to log libp2p traces
 	logP2P = flag.Bool("log_p2p", false, "log libp2p debug info")

-	initialAccount = &genesis.DeployAccount{}
+	initialAccounts = []*genesis.DeployAccount{}
 	// logging verbosity
 	verbosity = flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)")
 	// dbDir is the database directory.
@ -188,8 +190,12 @@ func passphraseForBls() {
 		return
 	}

-	if *blsKeyFile == "" || *blsPass == "" {
-		fmt.Println("Internal nodes need to have pass to decrypt blskey")
+	if *blsKeyFile == "" && *blsFolder == "" {
+		fmt.Println("blskey_file or blsfolder option must be provided")
+		os.Exit(101)
+	}
+	if *blsPass == "" {
+		fmt.Println("Internal nodes need to have blspass to decrypt blskey")
 		os.Exit(101)
 	}
 	passphrase, err := utils.GetPassphraseFromSource(*blsPass)
@ -200,27 +206,41 @@ func passphraseForBls() {
 	blsPassphrase = passphrase
 }

+func findAccountsByPubKeys(config shardingconfig.Instance, pubKeys []*bls.PublicKey) {
+	for _, key := range pubKeys {
+		keyStr := key.SerializeToHexStr()
+		_, account := config.FindAccount(keyStr)
+		if account != nil {
+			initialAccounts = append(initialAccounts, account)
+		}
+	}
+}
+
 func setupLegacyNodeAccount() error {
 	genesisShardingConfig := shard.Schedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch))
-	pubKey := setupConsensusKey(nodeconfig.GetDefaultConfig())
+	multiBlsPubKey := setupConsensusKey(nodeconfig.GetDefaultConfig())

 	reshardingEpoch := genesisShardingConfig.ReshardingEpoch()
 	if reshardingEpoch != nil && len(reshardingEpoch) > 0 {
 		for _, epoch := range reshardingEpoch {
 			config := shard.Schedule.InstanceForEpoch(epoch)
-			_, initialAccount = config.FindAccount(pubKey.SerializeToHexStr())
-			if initialAccount != nil {
+			findAccountsByPubKeys(config, multiBlsPubKey.PublicKey)
+			if len(initialAccounts) != 0 {
 				break
 			}
 		}
 	} else {
-		_, initialAccount = genesisShardingConfig.FindAccount(pubKey.SerializeToHexStr())
+		findAccountsByPubKeys(genesisShardingConfig, multiBlsPubKey.PublicKey)
+	}
+
+	if len(initialAccounts) == 0 {
+		fmt.Fprintf(os.Stderr, "ERROR cannot find your BLS key in the genesis/FN tables: %s\n", multiBlsPubKey.SerializeToHexStr())
+		os.Exit(100)
 	}

-	if initialAccount == nil {
-		return errors.Errorf("cannot find key %s in table", pubKey.SerializeToHexStr())
+	for _, account := range initialAccounts {
+		fmt.Printf("My Genesis Account: %v\n", *account)
 	}
-	fmt.Printf("My Genesis Account: %v\n", *initialAccount)
 	return nil
 }

@ -230,39 +250,82 @@ func setupStakingNodeAccount() error {
 	if err != nil {
 		return errors.Wrap(err, "cannot determine shard to join")
 	}
-	initialAccount = &genesis.DeployAccount{}
+	for _, blsKey := range pubKey.PublicKey {
+		initialAccount := &genesis.DeployAccount{}
 		initialAccount.ShardID = shardID
-	initialAccount.BlsPublicKey = pubKey.SerializeToHexStr()
+		initialAccount.BlsPublicKey = blsKey.SerializeToHexStr()
 		initialAccount.Address = ""
+		initialAccounts = append(initialAccounts, initialAccount)
+	}
+	return nil
+}
+
+func readMultiBlsKeys(consensusMultiBlsPriKey *multibls.PrivateKey, consensusMultiBlsPubKey *multibls.PublicKey) error {
+	multiBlsKeyDir := blsFolder
+	blsKeyFiles, err := ioutil.ReadDir(*multiBlsKeyDir)
+	if err != nil {
+		return err
+	}
+
+	for _, blsKeyFile := range blsKeyFiles {
+		if filepath.Ext(blsKeyFile.Name()) != ".key" {
+			fmt.Println("BLS key file should have .key file extension, found", blsKeyFile.Name())
+			continue
+		}
+		blsKeyFilePath := path.Join(*multiBlsKeyDir, blsKeyFile.Name())
+		consensusPriKey, err := blsgen.LoadBlsKeyWithPassPhrase(blsKeyFilePath, blsPassphrase) // uses the same bls passphrase for multiple bls keys
+		if err != nil {
+			return err
+		}
+		// TODO: assumes order between public/private key pairs
+		multibls.AppendPriKey(consensusMultiBlsPriKey, consensusPriKey)
+		multibls.AppendPubKey(consensusMultiBlsPubKey, consensusPriKey.GetPublicKey())
+	}
+
 	return nil
 }

-func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) *bls.PublicKey {
+func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) multibls.PublicKey {
+	consensusMultiPriKey := &multibls.PrivateKey{}
+	consensusMultiPubKey := &multibls.PublicKey{}
+
+	if *blsKeyFile != "" {
 		consensusPriKey, err := blsgen.LoadBlsKeyWithPassPhrase(*blsKeyFile, blsPassphrase)
 		if err != nil {
 			_, _ = fmt.Fprintf(os.Stderr, "ERROR when loading bls key, err :%v\n", err)
 			os.Exit(100)
 		}
-	pubKey := consensusPriKey.GetPublicKey()
-
-	// Consensus keys are the BLS12-381 keys used to sign consensus messages
-	nodeConfig.ConsensusPriKey, nodeConfig.ConsensusPubKey = consensusPriKey, consensusPriKey.GetPublicKey()
-	if nodeConfig.ConsensusPriKey == nil || nodeConfig.ConsensusPubKey == nil {
-		fmt.Println("error to get consensus keys.")
+		multibls.AppendPriKey(consensusMultiPriKey, consensusPriKey)
+		multibls.AppendPubKey(consensusMultiPubKey, consensusPriKey.GetPublicKey())
+	} else {
+		err := readMultiBlsKeys(consensusMultiPriKey, consensusMultiPubKey)
+		if err != nil {
+			_, _ = fmt.Fprintf(os.Stderr, "ERROR when loading bls keys, err :%v\n", err)
 			os.Exit(100)
 		}
-	return pubKey
+	}
+
+	// Consensus keys are the BLS12-381 keys used to sign consensus messages
+	nodeConfig.ConsensusPriKey = consensusMultiPriKey
+	nodeConfig.ConsensusPubKey = consensusMultiPubKey
+
+	return *consensusMultiPubKey
 }

 func createGlobalConfig() (*nodeconfig.ConfigType, error) {
 	var err error

-	nodeConfig := nodeconfig.GetShardConfig(initialAccount.ShardID)
+	if len(initialAccounts) == 0 {
+		initialAccounts = append(initialAccounts, &genesis.DeployAccount{ShardID: uint32(*shardID)})
+	}
+	nodeConfig := nodeconfig.GetShardConfig(initialAccounts[0].ShardID)
 	if *nodeType == "validator" {
 		// Set up consensus keys.
 		setupConsensusKey(nodeConfig)
 	} else {
-		nodeConfig.ConsensusPriKey = &bls.SecretKey{} // set dummy bls key for consensus object
+		// set dummy bls key for consensus object
+		nodeConfig.ConsensusPriKey = multibls.GetPrivateKey(&bls.SecretKey{})
+		nodeConfig.ConsensusPubKey = multibls.GetPublicKey(&bls.PublicKey{})
 	}

 	// Set network type
@ -279,7 +342,7 @@ func createGlobalConfig() (*nodeconfig.ConfigType, error) {
 			*keyFile)
 	}

-	selfPeer := p2p.Peer{IP: *ip, Port: *port, ConsensusPubKey: nodeConfig.ConsensusPubKey}
+	selfPeer := p2p.Peer{IP: *ip, Port: *port, ConsensusPubKey: nodeConfig.ConsensusPubKey.PublicKey[0]}

 	myHost, err = p2pimpl.NewHost(&selfPeer, nodeConfig.P2pPriKey)
 	if err != nil {
@ -317,12 +380,14 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node {
 	currentConsensus.Decider.SetShardIDProvider(func() (uint32, error) {
 		return currentConsensus.ShardID, nil
 	})
-	currentConsensus.Decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) {
+	currentConsensus.Decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) {
 		return currentConsensus.PubKey, nil
 	})

-	if initialAccount.Address != "" { // staking validator doesn't have to specify ECDSA address
-		currentConsensus.SelfAddress = common.ParseAddr(initialAccount.Address)
+	// staking validator doesn't have to specify ECDSA address
+	currentConsensus.SelfAddresses = map[string]ethCommon.Address{}
+	for _, initialAccount := range initialAccounts {
+		currentConsensus.SelfAddresses[initialAccount.BlsPublicKey] = common.ParseAddr(initialAccount.Address)
 	}

 	if err != nil {
@ -540,18 +605,21 @@ func main() {
 			os.Exit(1)
 		}
 	}
+	if *nodeType == "validator" {
 		fmt.Printf("%s mode; node key %s -> shard %d\n",
 			map[bool]string{false: "Legacy", true: "Staking"}[*stakingFlag],
 			nodeconfig.GetDefaultConfig().ConsensusPubKey.SerializeToHexStr(),
-		initialAccount.ShardID)
-
+			initialAccounts[0].ShardID)
+	}
 	if *nodeType != "validator" && *shardID >= 0 {
+		for _, initialAccount := range initialAccounts {
 			utils.Logger().Info().
 				Uint32("original", initialAccount.ShardID).
 				Int("override", *shardID).
 				Msg("ShardID Override")
 			initialAccount.ShardID = uint32(*shardID)
 		}
+	}

 	nodeConfig, err := createGlobalConfig()
 	if err != nil {
@ -614,7 +682,7 @@ func main() {
 	}

 	utils.Logger().Info().
-		Str("BlsPubKey", hex.EncodeToString(nodeConfig.ConsensusPubKey.Serialize())).
+		Str("BlsPubKey", nodeConfig.ConsensusPubKey.SerializeToHexStr()).
 		Uint32("ShardID", nodeConfig.ShardID).
 		Str("ShardGroupID", nodeConfig.GetShardGroupID().String()).
 		Str("BeaconGroupID", nodeConfig.GetBeaconGroupID().String()).
--- a/consensus/consensus.go
+++ b/consensus/consensus.go
@ -14,8 +14,10 @@ import (
 	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/memprofiling"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/staking/slash"
+	"github.com/pkg/errors"
 )

 const (
@ -23,6 +25,8 @@ const (
 	vdfAndSeedSize  = 548 // size of VDF/Proof and Seed
 )

+var errLeaderPriKeyNotFound = errors.New("getting leader private key from consensus public keys failed")
+
 // Consensus is the main struct with all states and data related to consensus process.
 type Consensus struct {
 	Decider quorum.Decider
@ -75,9 +79,9 @@ type Consensus struct {
 	MinPeers   int
 	pubKeyLock sync.Mutex
 	// private/public keys of current node
-	priKey      *bls.SecretKey
-	PubKey      *bls.PublicKey
-	SelfAddress common.Address
+	priKey        *multibls.PrivateKey
+	PubKey        *multibls.PublicKey
+	SelfAddresses map[string]common.Address
 	// the publickey of leader
 	LeaderPubKey *bls.PublicKey
 	viewID       uint64
@ -173,11 +177,26 @@ func (consensus *Consensus) GetBlockReward() *big.Int {
 	return consensus.lastBlockReward
 }

+// GetLeaderPrivateKey returns leader private key if node is the leader
+func (consensus *Consensus) GetLeaderPrivateKey(leaderKey *bls.PublicKey) (*bls.SecretKey, error) {
+	for i, key := range consensus.PubKey.PublicKey {
+		if key.IsEqual(leaderKey) {
+			return consensus.priKey.PrivateKey[i], nil
+		}
+	}
+	return nil, errors.Wrapf(errLeaderPriKeyNotFound, leaderKey.SerializeToHexStr())
+}
+
+// GetConsensusLeaderPrivateKey returns consensus leader private key if node is the leader
+func (consensus *Consensus) GetConsensusLeaderPrivateKey() (*bls.SecretKey, error) {
+	return consensus.GetLeaderPrivateKey(consensus.LeaderPubKey)
+}
+
 // TODO: put shardId into chain reader's chain config

 // New create a new Consensus record
 func New(
-	host p2p.Host, shard uint32, leader p2p.Peer, blsPriKey *bls.SecretKey,
+	host p2p.Host, shard uint32, leader p2p.Peer, multiBlsPriKey *multibls.PrivateKey,
 	Decider quorum.Decider,
 ) (*Consensus, error) {
 	consensus := Consensus{}
@ -194,9 +213,9 @@ func New(
 	consensus.consensusTimeout = createTimeout()
 	consensus.validators.Store(leader.ConsensusPubKey.SerializeToHexStr(), leader)

-	if blsPriKey != nil {
-		consensus.priKey = blsPriKey
-		consensus.PubKey = blsPriKey.GetPublicKey()
+	if multiBlsPriKey != nil {
+		consensus.priKey = multiBlsPriKey
+		consensus.PubKey = multiBlsPriKey.GetPublicKey()
 		utils.Logger().Info().
 			Str("publicKey", consensus.PubKey.SerializeToHexStr()).Msg("My Public Key")
 	} else {
--- a/consensus/consensus_service.go
+++ b/consensus/consensus_service.go
@ -20,6 +20,7 @@ import (
 	"github.com/harmony-one/harmony/internal/ctxerror"
 	"github.com/harmony-one/harmony/internal/profiler"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/shard/committee"
@ -61,13 +62,11 @@ var (
 )

 // Signs the consensus message and returns the marshaled message.
-func (consensus *Consensus) signAndMarshalConsensusMessage(
-	message *msg_pb.Message,
-) ([]byte, error) {
-	if err := consensus.signConsensusMessage(message); err != nil {
+func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Message,
+	priKey *bls.SecretKey) ([]byte, error) {
+	if err := consensus.signConsensusMessage(message, priKey); err != nil {
 		return empty, err
 	}
-
 	marshaledMessage, err := protobuf.Marshal(message)
 	if err != nil {
 		return empty, err
@ -121,14 +120,15 @@ func NewFaker() *Consensus {
 }

 // Sign on the hash of the message
-func (consensus *Consensus) signMessage(message []byte) []byte {
+func (consensus *Consensus) signMessage(message []byte, priKey *bls.SecretKey) []byte {
 	hash := hash.Keccak256(message)
-	signature := consensus.priKey.SignHash(hash[:])
+	signature := priKey.SignHash(hash[:])
 	return signature.Serialize()
 }

 // Sign on the consensus message signature field.
-func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message) error {
+func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message,
+	priKey *bls.SecretKey) error {
 	message.Signature = nil
 	// TODO: use custom serialization method rather than protobuf
 	marshaledMessage, err := protobuf.Marshal(message)
@ -136,7 +136,7 @@ func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message) error
 		return err
 	}
 	// 64 byte of signature on previous data
-	signature := consensus.signMessage(marshaledMessage)
+	signature := consensus.signMessage(marshaledMessage, priKey)
 	message.Signature = signature
 	return nil
 }
@ -381,7 +381,7 @@ func (consensus *Consensus) reportMetrics(block types.Block) {
 		txHashes = append(txHashes, hex.EncodeToString(txHash[:]))
 	}
 	metrics := map[string]interface{}{
-		"key":             hex.EncodeToString(consensus.PubKey.Serialize()),
+		"key":             hex.EncodeToString(consensus.LeaderPubKey.Serialize()),
 		"tps":             tps,
 		"txCount":         numOfTxs,
 		"nodeCount":       consensus.Decider.ParticipantsCount() + 1,
@ -480,7 +480,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode {
 		decider.SetShardIDProvider(func() (uint32, error) {
 			return consensus.ShardID, nil
 		})
-		decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) {
+		decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) {
 			return consensus.PubKey, nil
 		})
 		consensus.Decider = decider
@ -577,15 +577,15 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode {
 		}
 	}

-	for i := range pubKeys {
+	for _, key := range pubKeys {
 		// in committee
-		if pubKeys[i].IsEqual(consensus.PubKey) {
+		if consensus.PubKey.Contains(key) {
 			if hasError {
 				return Syncing
 			}

 			// If the leader changed and I myself become the leader
-			if !consensus.LeaderPubKey.IsEqual(oldLeader) && consensus.LeaderPubKey.IsEqual(consensus.PubKey) {
+			if !consensus.LeaderPubKey.IsEqual(oldLeader) && consensus.IsLeader() {
 				go func() {
 					utils.Logger().Debug().
 						Str("myKey", consensus.PubKey.SerializeToHexStr()).
@ -605,8 +605,10 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode {
 // IsLeader check if the node is a leader or not by comparing the public key of
 // the node with the leader public key
 func (consensus *Consensus) IsLeader() bool {
-	if consensus.PubKey != nil && consensus.LeaderPubKey != nil {
-		return consensus.PubKey.IsEqual(consensus.LeaderPubKey)
+	for _, key := range consensus.PubKey.PublicKey {
+		if key.IsEqual(consensus.LeaderPubKey) {
+			return true
+		}
 	}
 	return false
 }
--- a/consensus/consensus_service_test.go
+++ b/consensus/consensus_service_test.go
@ -8,6 +8,7 @@ import (
 	"github.com/harmony-one/harmony/consensus/quorum"
 	"github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
 	"github.com/harmony-one/harmony/shard"
@ -23,7 +24,7 @@ func TestPopulateMessageFields(t *testing.T) {
 	blsPriKey := bls.RandPrivateKey()
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
 	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, blsPriKey, decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot craeate consensus: %v", err)
@ -37,9 +38,9 @@ func TestPopulateMessageFields(t *testing.T) {
 			Consensus: &msg_pb.ConsensusRequest{},
 		},
 	}
-	consensusMsg := consensus.populateMessageFields(
-		msg.GetConsensus(), consensus.blockHash[:],
-	)
+
+	consensusMsg := consensus.populateMessageFields(msg.GetConsensus(), consensus.blockHash[:],
+		blsPriKey.GetPublicKey())

 	if consensusMsg.ViewId != 2 {
 		t.Errorf("Consensus ID is not populated correctly")
@ -60,8 +61,9 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) {
 		t.Fatalf("newhost failure: %v", err)
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
+	blsPriKey := bls.RandPrivateKey()
 	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot craeate consensus: %v", err)
@ -70,7 +72,7 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) {
 	consensus.blockHash = [32]byte{}

 	msg := &msg_pb.Message{}
-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(msg)
+	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(msg, blsPriKey)

 	if err != nil || len(marshaledMessage) == 0 {
 		t.Errorf("Failed to sign and marshal the message: %s", err)
@ -88,8 +90,9 @@ func TestSetViewID(t *testing.T) {
 		t.Fatalf("newhost failure: %v", err)
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
+	blsPriKey := bls.RandPrivateKey()
 	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot craeate consensus: %v", err)
--- a/consensus/consensus_test.go
+++ b/consensus/consensus_test.go
@ -6,6 +6,7 @@ import (
 	"github.com/harmony-one/harmony/consensus/quorum"
 	"github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
 	"github.com/harmony-one/harmony/shard"
@ -20,7 +21,7 @@ func TestNew(test *testing.T) {
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
 	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(bls.RandPrivateKey()), decider,
 	)
 	if err != nil {
 		test.Fatalf("Cannot craeate consensus: %v", err)
--- a/consensus/consensus_v2.go
+++ b/consensus/consensus_v2.go
@ -109,8 +109,13 @@ func (consensus *Consensus) finalizeCommits() {
 		Int64("NumCommits", consensus.Decider.SignersCount(quorum.Commit)).
 		Msg("[Finalizing] Finalizing Block")
 	beforeCatchupNum := consensus.blockNum
+	leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey()
+	if err != nil {
+		consensus.getLogger().Error().Err(err).Msg("[FinalizeCommits] leader not found")
+		return
+	}
 	// Construct committed message
-	network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil)
+	network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil, leaderPriKey.GetPublicKey(), leaderPriKey)
 	if err != nil {
 		consensus.getLogger().Warn().Err(err).
 			Msg("[FinalizeCommits] Unable to construct Committed message")
@ -182,7 +187,7 @@ func (consensus *Consensus) finalizeCommits() {
 		Uint64("epochNum", block.Epoch().Uint64()).
 		Uint64("ViewId", block.Header().ViewID().Uint64()).
 		Str("blockHash", block.Hash().String()).
-		Int("index", consensus.Decider.IndexOf(consensus.PubKey)).
+		Int("index", consensus.Decider.IndexOf(consensus.LeaderPubKey)).
 		Int("numTxns", len(block.Transactions())).
 		Int("numStakingTxns", len(block.StakingTransactions())).
 		Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!")
@ -491,10 +496,15 @@ func (consensus *Consensus) Start(
 }

 // GenerateVrfAndProof generates new VRF/Proof from hash of previous block
-func (consensus *Consensus) GenerateVrfAndProof(
-	newBlock *types.Block, vrfBlockNumbers []uint64,
-) []uint64 {
-	sk := vrf_bls.NewVRFSigner(consensus.priKey)
+func (consensus *Consensus) GenerateVrfAndProof(newBlock *types.Block, vrfBlockNumbers []uint64) []uint64 {
+	key, err := consensus.GetConsensusLeaderPrivateKey()
+	if err != nil {
+		consensus.getLogger().Error().
+			Err(err).
+			Msg("[GenerateVrfAndProof] VRF generation error")
+		return vrfBlockNumbers
+	}
+	sk := vrf_bls.NewVRFSigner(key)
 	blockHash := [32]byte{}
 	previousHeader := consensus.ChainReader.GetHeaderByNumber(
 		newBlock.NumberU64() - 1,
--- a/consensus/consensus_viewchange_msg.go
+++ b/consensus/consensus_viewchange_msg.go
@ -3,6 +3,7 @@ package consensus
 import (
 	"encoding/binary"

+	"github.com/harmony-one/bls/ffi/go/bls"
 	"github.com/harmony-one/harmony/api/proto"
 	msg_pb "github.com/harmony-one/harmony/api/proto/message"
 	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
@ -10,7 +11,7 @@ import (
 )

 // construct the view change message
-func (consensus *Consensus) constructViewChangeMessage() []byte {
+func (consensus *Consensus) constructViewChangeMessage(pubKey *bls.PublicKey, priKey *bls.SecretKey) []byte {
 	message := &msg_pb.Message{
 		ServiceType: msg_pb.ServiceType_CONSENSUS,
 		Type:        msg_pb.MessageType_VIEWCHANGE,
@ -24,7 +25,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte {
 	vcMsg.BlockNum = consensus.blockNum
 	vcMsg.ShardId = consensus.ShardID
 	// sender address
-	vcMsg.SenderPubkey = consensus.PubKey.Serialize()
+	vcMsg.SenderPubkey = pubKey.Serialize()

 	// next leader key already updated
 	vcMsg.LeaderPubkey = consensus.LeaderPubKey.Serialize()
@ -49,7 +50,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte {
 		Str("pubKey", consensus.PubKey.SerializeToHexStr()).
 		Msg("[constructViewChangeMessage]")

-	sign := consensus.priKey.SignHash(msgToSign)
+	sign := priKey.SignHash(msgToSign)
 	if sign != nil {
 		vcMsg.ViewchangeSig = sign.Serialize()
 	} else {
@ -58,14 +59,14 @@ func (consensus *Consensus) constructViewChangeMessage() []byte {

 	viewIDBytes := make([]byte, 8)
 	binary.LittleEndian.PutUint64(viewIDBytes, consensus.current.ViewID())
-	sign1 := consensus.priKey.SignHash(viewIDBytes)
+	sign1 := priKey.SignHash(viewIDBytes)
 	if sign1 != nil {
 		vcMsg.ViewidSig = sign1.Serialize()
 	} else {
 		utils.Logger().Error().Msg("unable to serialize viewID signature")
 	}

-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
+	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey)
 	if err != nil {
 		utils.Logger().Error().Err(err).
 			Msg("[constructViewChangeMessage] failed to sign and marshal the viewchange message")
@ -74,7 +75,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte {
 }

 // new leader construct newview message
-func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte {
+func (consensus *Consensus) constructNewViewMessage(viewID uint64, pubKey *bls.PublicKey, priKey *bls.SecretKey) []byte {
 	message := &msg_pb.Message{
 		ServiceType: msg_pb.ServiceType_CONSENSUS,
 		Type:        msg_pb.MessageType_NEWVIEW,
@ -88,7 +89,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte {
 	vcMsg.BlockNum = consensus.blockNum
 	vcMsg.ShardId = consensus.ShardID
 	// sender address
-	vcMsg.SenderPubkey = consensus.PubKey.Serialize()
+	vcMsg.SenderPubkey = pubKey.Serialize()
 	vcMsg.Payload = consensus.m1Payload

 	sig2arr := consensus.GetNilSigsArray(viewID)
@ -108,7 +109,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte {
 		vcMsg.M3Bitmap = consensus.viewIDBitmap[viewID].Bitmap
 	}

-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
+	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey)
 	if err != nil {
 		utils.Logger().Error().Err(err).
 			Msg("[constructNewViewMessage] failed to sign and marshal the new view message")
--- a/consensus/construct.go
+++ b/consensus/construct.go
@ -22,7 +22,7 @@ type NetworkMessage struct {

 // Populates the common basic fields for all consensus message.
 func (consensus *Consensus) populateMessageFields(
-	request *msg_pb.ConsensusRequest, blockHash []byte,
+	request *msg_pb.ConsensusRequest, blockHash []byte, pubKey *bls.PublicKey,
 ) *msg_pb.ConsensusRequest {
 	request.ViewId = consensus.viewID
 	request.BlockNum = consensus.blockNum
@ -30,13 +30,13 @@ func (consensus *Consensus) populateMessageFields(
 	// 32 byte block hash
 	request.BlockHash = blockHash
 	// sender address
-	request.SenderPubkey = consensus.PubKey.Serialize()
+	request.SenderPubkey = pubKey.Serialize()
 	return request
 }

 // construct is the single creation point of messages intended for the wire.
 func (consensus *Consensus) construct(
-	p msg_pb.MessageType, payloadForSign []byte,
+	p msg_pb.MessageType, payloadForSign []byte, pubKey *bls.PublicKey, priKey *bls.SecretKey,
 ) (*NetworkMessage, error) {
 	message := &msg_pb.Message{
 		ServiceType: msg_pb.ServiceType_CONSENSUS,
@ -51,7 +51,7 @@ func (consensus *Consensus) construct(
 	)

 	consensusMsg = consensus.populateMessageFields(
-		message.GetConsensus(), consensus.blockHash[:],
+		message.GetConsensus(), consensus.blockHash[:], pubKey,
 	)

 	// Do the signing, 96 byte of bls signature
@ -67,11 +67,11 @@ func (consensus *Consensus) construct(
 		buffer.Write(consensus.prepareBitmap.Bitmap)
 		consensusMsg.Payload = buffer.Bytes()
 	case msg_pb.MessageType_PREPARE:
-		if s := consensus.priKey.SignHash(consensusMsg.BlockHash); s != nil {
+		if s := priKey.SignHash(consensusMsg.BlockHash); s != nil {
 			consensusMsg.Payload = s.Serialize()
 		}
 	case msg_pb.MessageType_COMMIT:
-		if s := consensus.priKey.SignHash(payloadForSign); s != nil {
+		if s := priKey.SignHash(payloadForSign); s != nil {
 			consensusMsg.Payload = s.Serialize()
 		}
 	case msg_pb.MessageType_COMMITTED:
@ -86,7 +86,7 @@ func (consensus *Consensus) construct(
 		consensusMsg.Payload = consensus.blockHash[:]
 	}

-	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message)
+	marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey)
 	if err != nil {
 		utils.Logger().Error().Err(err).
 			Str("phase", p.String()).
--- a/consensus/construct_test.go
+++ b/consensus/construct_test.go
@ -9,6 +9,7 @@ import (
 	"github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/ctxerror"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
 	"github.com/harmony-one/harmony/shard"
@ -22,14 +23,15 @@ func TestConstructAnnounceMessage(test *testing.T) {
 		test.Fatalf("newhost failure: %v", err)
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
+	blsPriKey := bls.RandPrivateKey()
 	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider,
 	)
 	if err != nil {
 		test.Fatalf("Cannot create consensus: %v", err)
 	}
 	consensus.blockHash = [32]byte{}
-	if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil); err != nil {
+	if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, blsPriKey.GetPublicKey(), blsPriKey); err != nil {
 		test.Fatalf("could not construct announce: %v", err)
 	}
 }
@ -47,8 +49,9 @@ func TestConstructPreparedMessage(test *testing.T) {
 		test.Fatalf("newhost failure: %v", err)
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
+	blsPriKey := bls.RandPrivateKey()
 	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider,
 	)
 	if err != nil {
 		test.Fatalf("Cannot craeate consensus: %v", err)
@ -78,7 +81,7 @@ func TestConstructPreparedMessage(test *testing.T) {
 		test.Log(ctxerror.New("prepareBitmap.SetKey").WithCause(err))
 	}

-	network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil)
+	network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, blsPriKey.GetPublicKey(), blsPriKey)
 	if err != nil {
 		test.Errorf("Error when creating prepared message")
 	}
--- a/consensus/fbft_log_test.go
+++ b/consensus/fbft_log_test.go
@ -8,6 +8,7 @@ import (
 	"github.com/harmony-one/harmony/consensus/quorum"
 	"github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
 	"github.com/harmony-one/harmony/shard"
@ -21,14 +22,15 @@ func constructAnnounceMessage(t *testing.T) (*NetworkMessage, error) {
 		t.Fatalf("newhost failure: %v", err)
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
+	blsPriKey := bls.RandPrivateKey()
 	consensus, err := New(
-		host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot create consensus: %v", err)
 	}
 	consensus.blockHash = [32]byte{}
-	return consensus.construct(msg_pb.MessageType_ANNOUNCE, nil)
+	return consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, blsPriKey.GetPublicKey(), blsPriKey)
 }

 func getConsensusMessage(payload []byte) (*msg_pb.Message, error) {
--- a/consensus/leader.go
+++ b/consensus/leader.go
@ -36,7 +36,13 @@ func (consensus *Consensus) announce(block *types.Block) {

 	consensus.block = encodedBlock
 	consensus.blockHeader = encodedBlockHeader
-	networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil)
+
+	key, err := consensus.GetConsensusLeaderPrivateKey()
+	if err != nil {
+		consensus.getLogger().Warn().Err(err).Msg("[Announce] Node not a leader")
+		return
+	}
+	networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, key.GetPublicKey(), key)
 	if err != nil {
 		consensus.getLogger().Err(err).
 			Str("message-type", msg_pb.MessageType_ANNOUNCE.String()).
@ -55,21 +61,20 @@ func (consensus *Consensus) announce(block *types.Block) {
 	consensus.FBFTLog.AddBlock(block)

 	// Leader sign the block hash itself
+	for i, key := range consensus.PubKey.PublicKey {
 		consensus.Decider.SubmitVote(
 			quorum.Prepare,
-		consensus.PubKey,
-		consensus.priKey.SignHash(consensus.blockHash[:]),
+			key,
+			consensus.priKey.PrivateKey[i].SignHash(consensus.blockHash[:]),
 			common.BytesToHash(consensus.blockHash[:]),
 		)
-	if err := consensus.prepareBitmap.SetKey(
-		consensus.PubKey, true,
-	); err != nil {
+		if err := consensus.prepareBitmap.SetKey(key, true); err != nil {
 			consensus.getLogger().Warn().Err(err).Msg(
 				"[Announce] Leader prepareBitmap SetKey failed",
 			)
 			return
 		}
-
+	}
 	// Construct broadcast p2p message
 	if err := consensus.msgSender.SendWithRetry(
 		consensus.blockNum, msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{
@ -268,7 +273,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) {
 								Offender: *addr,
 							}
 							consensus.SlashChan <- proof
-						}(consensus.SelfAddress)
+						}(consensus.SelfAddresses[consensus.LeaderPubKey.SerializeToHexStr()])
 						return
 					}
 				}
--- a/consensus/quorum/one-node-one-vote.go
+++ b/consensus/quorum/one-node-one-vote.go
@ -120,12 +120,14 @@ func (v *uniformVoteWeight) AmIMemberOfCommitee() bool {
 	}
 	identity, _ := pubKeyFunc()
 	everyone := v.DumpParticipants()
-	myVoterID := identity.SerializeToHexStr()
+	for _, key := range identity.PublicKey {
+		myVoterID := key.SerializeToHexStr()
 		for i := range everyone {
 			if everyone[i] == myVoterID {
 				return true
 			}
 		}
+	}
 	return false
 }

--- a/consensus/quorum/one-node-staked-vote.go
+++ b/consensus/quorum/one-node-staked-vote.go
@ -244,10 +244,15 @@ func (v *stakedVoteWeight) AmIMemberOfCommitee() bool {
 		return false
 	}
 	identity, _ := pubKeyFunc()
+	for _, key := range identity.PublicKey {
 		w := shard.BlsPublicKey{}
-	w.FromLibBLSPublicKey(identity)
+		w.FromLibBLSPublicKey(key)
 		_, ok := v.roster.Voters[w]
-	return ok
+		if ok {
+			return true
+		}
+	}
+	return false
 }

 func newBox() *voteBox {
--- a/consensus/quorum/quorum.go
+++ b/consensus/quorum/quorum.go
@ -8,6 +8,7 @@ import (
 	"github.com/harmony-one/harmony/consensus/votepower"
 	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/numeric"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/pkg/errors"
@ -95,13 +96,13 @@ type SignatureReader interface {
 // DependencyInjectionWriter ..
 type DependencyInjectionWriter interface {
 	SetShardIDProvider(func() (uint32, error))
-	SetMyPublicKeyProvider(func() (*bls.PublicKey, error))
+	SetMyPublicKeyProvider(func() (*multibls.PublicKey, error))
 }

 // DependencyInjectionReader ..
 type DependencyInjectionReader interface {
 	ShardIDProvider() func() (uint32, error)
-	MyPublicKey() func() (*bls.PublicKey, error)
+	MyPublicKey() func() (*multibls.PublicKey, error)
 }

 // Decider ..
@ -151,7 +152,7 @@ type cIdentities struct {

 type depInject struct {
 	shardIDProvider   func() (uint32, error)
-	publicKeyProvider func() (*bls.PublicKey, error)
+	publicKeyProvider func() (*multibls.PublicKey, error)
 }

 func (s *cIdentities) AggregateVotes(p Phase) *bls.Sign {
@ -323,11 +324,11 @@ func (d *depInject) ShardIDProvider() func() (uint32, error) {
 	return d.shardIDProvider
 }

-func (d *depInject) SetMyPublicKeyProvider(p func() (*bls.PublicKey, error)) {
+func (d *depInject) SetMyPublicKeyProvider(p func() (*multibls.PublicKey, error)) {
 	d.publicKeyProvider = p
 }

-func (d *depInject) MyPublicKey() func() (*bls.PublicKey, error) {
+func (d *depInject) MyPublicKey() func() (*multibls.PublicKey, error) {
 	return d.publicKeyProvider
 }

--- a/consensus/threshold.go
+++ b/consensus/threshold.go
@ -14,8 +14,13 @@ import (
 func (consensus *Consensus) didReachPrepareQuorum() error {
 	logger := utils.Logger()
 	logger.Debug().Msg("[OnPrepare] Received Enough Prepare Signatures")
+	leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey()
+	if err != nil {
+		utils.Logger().Warn().Err(err).Msg("[OnPrepare] leader not found")
+		return err
+	}
 	// Construct and broadcast prepared message
-	networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARED, nil)
+	networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, consensus.LeaderPubKey, leaderPriKey)
 	if err != nil {
 		consensus.getLogger().Err(err).
 			Str("message-type", msg_pb.MessageType_PREPARED.String()).
@ -33,18 +38,22 @@ func (consensus *Consensus) didReachPrepareQuorum() error {
 	blockNumHash := [8]byte{}
 	binary.LittleEndian.PutUint64(blockNumHash[:], consensus.blockNum)
 	commitPayload := append(blockNumHash[:], consensus.blockHash[:]...)
+
+	// so by this point, everyone has committed to the blockhash of this block
+	// in prepare and so this is the actual block.
+	for i, key := range consensus.PubKey.PublicKey {
 		consensus.Decider.SubmitVote(
 			quorum.Commit,
-		consensus.PubKey,
-		consensus.priKey.SignHash(commitPayload),
+			key,
+			consensus.priKey.PrivateKey[i].SignHash(commitPayload),
 			common.BytesToHash(consensus.blockHash[:]),
 		)

-	if err := consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil {
+		if err := consensus.commitBitmap.SetKey(key, true); err != nil {
 			consensus.getLogger().Debug().Msg("[OnPrepare] Leader commit bitmap set failed")
 			return err
 		}
-
+	}
 	if err := consensus.msgSender.SendWithRetry(
 		consensus.blockNum,
 		msg_pb.MessageType_PREPARED, []nodeconfig.GroupID{
--- a/consensus/validator.go
+++ b/consensus/validator.go
@ -58,7 +58,9 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) {
 }

 func (consensus *Consensus) prepare() {
-	networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil)
+	groupID := []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}
+	for i, key := range consensus.PubKey.PublicKey {
+		networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil, key, consensus.priKey.PrivateKey[i])
 		if err != nil {
 			consensus.getLogger().Err(err).
 				Str("message-type", msg_pb.MessageType_PREPARE.String()).
@ -68,9 +70,7 @@ func (consensus *Consensus) prepare() {

 		// TODO: this will not return immediatey, may block
 		if err := consensus.msgSender.SendWithoutRetry(
-		[]nodeconfig.GroupID{
-			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)),
-		},
+			groupID,
 			host.ConstructP2pMessage(byte(17), networkMessage.Bytes),
 		); err != nil {
 			consensus.getLogger().Warn().Err(err).Msg("[OnAnnounce] Cannot send prepare message")
@ -79,6 +79,7 @@ func (consensus *Consensus) prepare() {
 				Str("blockHash", hex.EncodeToString(consensus.blockHash[:])).
 				Msg("[OnAnnounce] Sent Prepare Message!!")
 		}
+	}
 	consensus.getLogger().Debug().
 		Str("From", consensus.phase.String()).
 		Str("To", FBFTPrepare.String()).
@ -192,9 +193,13 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) {
 	}
 	blockNumBytes := make([]byte, 8)
 	binary.LittleEndian.PutUint64(blockNumBytes, consensus.blockNum)
+	groupID := []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}
+	for i, key := range consensus.PubKey.PublicKey {
 		networkMessage, _ := consensus.construct(
+			// TODO: should only sign on block hash
 			msg_pb.MessageType_COMMIT,
 			append(blockNumBytes, consensus.blockHash[:]...),
+			key, consensus.priKey.PrivateKey[i],
 		)
 		// TODO: genesis account node delay for 1 second,
 		// this is a temp fix for allows FN nodes to earning reward
@ -203,8 +208,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) {
 		}

 		if err := consensus.msgSender.SendWithoutRetry(
-		[]nodeconfig.GroupID{
-			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))},
+			groupID,
 			host.ConstructP2pMessage(byte(17), networkMessage.Bytes),
 		); err != nil {
 			consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!")
@ -214,7 +218,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) {
 				Hex("blockHash", consensus.blockHash[:]).
 				Msg("[OnPrepared] Sent Commit Message!!")
 		}
-
+	}
 	consensus.getLogger().Debug().
 		Str("From", consensus.phase.String()).
 		Str("To", FBFTCommit.String()).
--- a/consensus/view_change.go
+++ b/consensus/view_change.go
@ -127,12 +127,14 @@ func (consensus *Consensus) startViewChange(viewID uint64) {
 		Str("NextLeader", consensus.LeaderPubKey.SerializeToHexStr()).
 		Msg("[startViewChange]")

-	msgToSend := consensus.constructViewChangeMessage()
+	for i, key := range consensus.PubKey.PublicKey {
+		msgToSend := consensus.constructViewChangeMessage(key, consensus.priKey.PrivateKey[i])
 		consensus.host.SendMessageToGroups([]nodeconfig.GroupID{
 			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)),
 		},
 			host.ConstructP2pMessage(byte(17), msgToSend),
 		)
+	}

 	consensus.consensusTimeout[timeoutViewChange].SetDuration(duration)
 	consensus.consensusTimeout[timeoutViewChange].Start()
@ -148,7 +150,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 		return
 	}
 	newLeaderKey := recvMsg.LeaderPubkey
-	if !consensus.PubKey.IsEqual(newLeaderKey) {
+	newLeaderPriKey, err := consensus.GetLeaderPrivateKey(newLeaderKey)
+	if err != nil {
 		return
 	}

@ -214,13 +217,13 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 		preparedMsg := consensus.FBFTLog.FindMessageByMaxViewID(preparedMsgs)
 		if preparedMsg == nil {
 			consensus.getLogger().Debug().Msg("[onViewChange] add my M2(NIL) type messaage")
-			consensus.nilSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(NIL)
-			consensus.nilBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true)
+			consensus.nilSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(NIL)
+			consensus.nilBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true)
 		} else {
 			consensus.getLogger().Debug().Msg("[onViewChange] add my M1 type messaage")
 			msgToSign := append(preparedMsg.BlockHash[:], preparedMsg.Payload...)
-			consensus.bhpSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(msgToSign)
-			consensus.bhpBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true)
+			consensus.bhpSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(msgToSign)
+			consensus.bhpBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true)
 		}
 	}
 	// add self m3 type message signature and bitmap
@ -228,8 +231,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 	if !ok3 {
 		viewIDBytes := make([]byte, 8)
 		binary.LittleEndian.PutUint64(viewIDBytes, recvMsg.ViewID)
-		consensus.viewIDSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(viewIDBytes)
-		consensus.viewIDBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true)
+		consensus.viewIDSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(viewIDBytes)
+		consensus.viewIDBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true)
 	}

 	// m2 type message
@ -307,7 +310,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 				copy(preparedMsg.BlockHash[:], recvMsg.Payload[:32])
 				preparedMsg.Payload = make([]byte, len(recvMsg.Payload)-32)
 				copy(preparedMsg.Payload[:], recvMsg.Payload[32:])
-				preparedMsg.SenderPubkey = consensus.PubKey
+				preparedMsg.SenderPubkey = newLeaderKey
 				consensus.getLogger().Info().Msg("[onViewChange] New Leader Prepared Message Added")
 				consensus.FBFTLog.AddMessage(&preparedMsg)
 			}
@ -349,7 +352,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 	// received enough view change messages, change state to normal consensus
 	if consensus.Decider.IsQuorumAchievedByMask(consensus.viewIDBitmap[recvMsg.ViewID]) {
 		consensus.current.SetMode(Normal)
-		consensus.LeaderPubKey = consensus.PubKey
+		consensus.LeaderPubKey = newLeaderKey
 		consensus.ResetState()
 		if len(consensus.m1Payload) == 0 {
 			// TODO(Chao): explain why ReadySignal is sent only in this case but not the other case.
@ -379,12 +382,12 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 			commitPayload := append(blockNumBytes[:], consensus.blockHash[:]...)
 			consensus.Decider.SubmitVote(
 				quorum.Commit,
-				consensus.PubKey,
-				consensus.priKey.SignHash(commitPayload),
+				newLeaderKey,
+				newLeaderPriKey.SignHash(commitPayload),
 				common.BytesToHash(consensus.blockHash[:]),
 			)

-			if err = consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil {
+			if err = consensus.commitBitmap.SetKey(newLeaderKey, true); err != nil {
 				consensus.getLogger().Debug().
 					Msg("[OnViewChange] New Leader commit bitmap set failed")
 				return
@ -392,7 +395,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) {
 		}

 		consensus.current.SetViewID(recvMsg.ViewID)
-		msgToSend := consensus.constructNewViewMessage(recvMsg.ViewID)
+		msgToSend := consensus.constructNewViewMessage(recvMsg.ViewID, newLeaderKey, newLeaderPriKey)

 		consensus.getLogger().Warn().
 			Int("payloadSize", len(consensus.m1Payload)).
@ -541,9 +544,13 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) {
 		// Construct and send the commit message
 		blockNumHash := make([]byte, 8)
 		binary.LittleEndian.PutUint64(blockNumHash, consensus.blockNum)
+		groupID := []nodeconfig.GroupID{
+			nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}
+		for i, key := range consensus.PubKey.PublicKey {
 			network, err := consensus.construct(
 				msg_pb.MessageType_COMMIT,
 				append(blockNumHash, consensus.blockHash[:]...),
+				key, consensus.priKey.PrivateKey[i],
 			)
 			if err != nil {
 				consensus.getLogger().Err(err).Msg("could not create commit message")
@ -552,10 +559,10 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) {
 			msgToSend := network.Bytes
 			consensus.getLogger().Info().Msg("onNewView === commit")
 			consensus.host.SendMessageToGroups(
-			[]nodeconfig.GroupID{
-				nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))},
+				groupID,
 				host.ConstructP2pMessage(byte(17), msgToSend),
 			)
+		}
 		consensus.getLogger().Debug().
 			Str("From", consensus.phase.String()).
 			Str("To", FBFTCommit.String()).
--- a/internal/chain/engine.go
+++ b/internal/chain/engine.go
@ -17,6 +17,7 @@ import (
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/internal/ctxerror"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/shard/committee"
 	"github.com/harmony-one/harmony/staking/availability"
@ -212,7 +213,7 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header)
 		d.SetShardIDProvider(func() (uint32, error) {
 			return parentHeader.ShardID(), nil
 		})
-		d.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) {
+		d.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) {
 			return nil, nil
 		})
 		d.SetVoters(slotList.FindCommitteeByID(parentHeader.ShardID()).Slots)
@ -444,7 +445,7 @@ func (e *engineImpl) VerifyHeaderWithSignature(chain engine.ChainReader, header
 		d.SetShardIDProvider(func() (uint32, error) {
 			return header.ShardID(), nil
 		})
-		d.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) {
+		d.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) {
 			return nil, nil
 		})
 		d.SetVoters(slotList.FindCommitteeByID(header.ShardID()).Slots)
--- a/internal/configs/node/config.go
+++ b/internal/configs/node/config.go
@ -8,9 +8,9 @@ import (
 	"math/big"
 	"sync"

-	"github.com/harmony-one/bls/ffi/go/bls"
 	shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding"
 	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/shard"
 	"github.com/harmony-one/harmony/staking/slash"
 	p2p_crypto "github.com/libp2p/go-libp2p-crypto"
@ -83,8 +83,8 @@ type ConfigType struct {
 	PushgatewayPort string  // metrics pushgateway prometheus port
 	StringRole      string
 	P2pPriKey       p2p_crypto.PrivKey
-	ConsensusPriKey *bls.SecretKey
-	ConsensusPubKey *bls.PublicKey
+	ConsensusPriKey *multibls.PrivateKey
+	ConsensusPubKey *multibls.PublicKey
 	// Database directory
 	DBDir            string
 	networkType      NetworkType
@ -280,7 +280,8 @@ func SetShardingSchedule(schedule shardingconfig.Schedule) {
 // consensus key.
 func (conf *ConfigType) ShardIDFromConsensusKey() (uint32, error) {
 	var pubKey shard.BlsPublicKey
-	if err := pubKey.FromLibBLSPublicKey(conf.ConsensusPubKey); err != nil {
+	// all keys belong to same shard
+	if err := pubKey.FromLibBLSPublicKey(conf.ConsensusPubKey.PublicKey[0]); err != nil {
 		return 0, errors.Wrapf(err,
 			"cannot convert libbls public key %s to internal form",
 			conf.ConsensusPubKey.SerializeToHexStr())
--- a/internal/configs/node/config_test.go
+++ b/internal/configs/node/config_test.go
@ -9,6 +9,7 @@ import (

 	mock_shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding/mock"
 	"github.com/harmony-one/harmony/internal/params"
+	"github.com/harmony-one/harmony/multibls"
 )

 func TestNodeConfigSingleton(t *testing.T) {
@ -115,7 +116,7 @@ func TestConfigType_ShardIDFromConsensusKey(t *testing.T) {
 			schedule := mock_shardingconfig.NewMockSchedule(mc)
 			schedule.EXPECT().InstanceForEpoch(tt.epoch).Return(instance)
 			conf := &ConfigType{
-				ConsensusPubKey:  tt.fields.ConsensusPubKey,
+				ConsensusPubKey:  multibls.GetPublicKey(tt.fields.ConsensusPubKey),
 				networkType:      tt.fields.networkType,
 				shardingSchedule: schedule,
 			}
--- a/multibls/multibls.go
+++ b/multibls/multibls.go
@ -0,0 +1,73 @@
+package multibls
+
+import (
+	"strings"
+
+	"github.com/harmony-one/bls/ffi/go/bls"
+)
+
+// PrivateKey stores the bls secret keys that belongs to the node
+type PrivateKey struct {
+	PrivateKey []*bls.SecretKey
+}
+
+// PublicKey stores the bls public keys that belongs to the node
+type PublicKey struct {
+	PublicKey []*bls.PublicKey
+}
+
+// SerializeToHexStr wrapper
+func (multiKey PublicKey) SerializeToHexStr() string {
+	var builder strings.Builder
+	for _, pubKey := range multiKey.PublicKey {
+		builder.WriteString(pubKey.SerializeToHexStr())
+	}
+	return builder.String()
+}
+
+// Contains wrapper
+func (multiKey PublicKey) Contains(pubKey *bls.PublicKey) bool {
+	for _, key := range multiKey.PublicKey {
+		if key.IsEqual(pubKey) {
+			return true
+		}
+	}
+	return false
+}
+
+// GetPublicKey wrapper
+func (multiKey PrivateKey) GetPublicKey() *PublicKey {
+	pubKeys := make([]*bls.PublicKey, len(multiKey.PrivateKey))
+	for i, key := range multiKey.PrivateKey {
+		pubKeys[i] = key.GetPublicKey()
+	}
+	return &PublicKey{PublicKey: pubKeys}
+}
+
+// GetPrivateKey creates a multibls PrivateKey using bls.SecretKey
+func GetPrivateKey(key *bls.SecretKey) *PrivateKey {
+	return &PrivateKey{PrivateKey: []*bls.SecretKey{key}}
+}
+
+// GetPublicKey creates a multibls PublicKey using bls.PublicKey
+func GetPublicKey(key *bls.PublicKey) *PublicKey {
+	return &PublicKey{PublicKey: []*bls.PublicKey{key}}
+}
+
+// AppendPubKey appends a PublicKey to multibls PublicKey
+func AppendPubKey(multiKey *PublicKey, key *bls.PublicKey) {
+	if multiKey != nil {
+		multiKey.PublicKey = append(multiKey.PublicKey, key)
+	} else {
+		multiKey = &PublicKey{PublicKey: []*bls.PublicKey{key}}
+	}
+}
+
+// AppendPriKey appends a SecretKey to multibls PrivateKey
+func AppendPriKey(multiKey *PrivateKey, key *bls.SecretKey) {
+	if multiKey != nil {
+		multiKey.PrivateKey = append(multiKey.PrivateKey, key)
+	} else {
+		multiKey = &PrivateKey{PrivateKey: []*bls.SecretKey{key}}
+	}
+}
--- a/node/node.go
+++ b/node/node.go
@ -652,8 +652,8 @@ func (node *Node) InitConsensusWithValidators() (err error) {
 			"blockNum", blockNum)
 	}

-	for i := range pubKeys {
-		if pubKeys[i].IsEqual(node.Consensus.PubKey) {
+	for _, key := range pubKeys {
+		if node.Consensus.PubKey.Contains(key) {
 			utils.Logger().Info().
 				Uint64("blockNum", blockNum).
 				Int("numPubKeys", len(pubKeys)).
--- a/node/node_cross_link.go
+++ b/node/node_cross_link.go
@ -10,6 +10,7 @@ import (
 	bls_cosi "github.com/harmony-one/harmony/crypto/bls"
 	"github.com/harmony-one/harmony/internal/ctxerror"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/shard"
 )

@ -163,7 +164,7 @@ func (node *Node) VerifyCrossLink(cl types.CrossLink) error {
 	decider.SetShardIDProvider(func() (uint32, error) {
 		return cl.ShardID(), nil
 	})
-	decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) {
+	decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) {
 		return nil, nil
 	})
 	if _, err := decider.SetVoters(committee.Slots); err != nil {
--- a/node/node_handler.go
+++ b/node/node_handler.go
@ -417,7 +417,7 @@ func (node *Node) PostConsensusProcessing(
 	// TODO: randomly selected a few validators to broadcast messages instead of only leader broadcast
 	// TODO: refactor the asynchronous calls to separate go routine.
 	node.lastConsensusTime = time.Now().Unix()
-	if node.Consensus.PubKey.IsEqual(node.Consensus.LeaderPubKey) {
+	if node.Consensus.IsLeader() {
 		if node.NodeConfig.ShardID == shard.BeaconChainShardID {
 			node.BroadcastNewBlock(newBlock)
 		}
--- a/node/node_handler_test.go
+++ b/node/node_handler_test.go
@ -10,6 +10,7 @@ import (
 	"github.com/harmony-one/harmony/crypto/bls"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
 	"github.com/harmony-one/harmony/shard"
@ -27,7 +28,7 @@ func TestAddNewBlock(t *testing.T) {
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
 	consensus, err := consensus.New(
-		host, shard.BeaconChainShardID, leader, blsKey, decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot craeate consensus: %v", err)
@ -65,7 +66,7 @@ func TestVerifyNewBlock(t *testing.T) {
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
 	consensus, err := consensus.New(
-		host, shard.BeaconChainShardID, leader, blsKey, decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot craeate consensus: %v", err)
--- a/node/node_metrics.go
+++ b/node/node_metrics.go
@ -43,13 +43,15 @@ func (node *Node) UpdateTxPoolSizeForMetrics(txPoolSize uint64) {

 // UpdateBalanceForMetrics uppdates node balance for metrics service.
 func (node *Node) UpdateBalanceForMetrics() {
-	curBalance, err := node.GetBalanceOfAddress(node.Consensus.SelfAddress)
+	for _, addr := range node.Consensus.SelfAddresses {
+		curBalance, err := node.GetBalanceOfAddress(addr)
 		if err != nil {
 			return
 		}
 		utils.Logger().Info().Msgf("Updating metrics node balance %d", curBalance.Uint64())
 		metrics.UpdateNodeBalance(curBalance)
 	}
+}

 // UpdateLastConsensusTimeForMetrics uppdates last consensus reached time for metrics service.
 func (node *Node) UpdateLastConsensusTimeForMetrics(prevLastConsensusTime int64) int64 {
--- a/node/node_newblock.go
+++ b/node/node_newblock.go
@ -84,7 +84,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) {

 	// Update worker's current header and state data in preparation to propose/process new transactions
 	var (
-		coinbase    = node.Consensus.SelfAddress
+		coinbase    = node.Consensus.SelfAddresses[node.Consensus.LeaderPubKey.SerializeToHexStr()]
 		beneficiary = coinbase
 		err         error
 	)
@ -94,7 +94,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) {
 	// After staking, all coinbase will be the address of bls pub key
 	if header := node.Worker.GetCurrentHeader(); node.Blockchain().Config().IsStaking(header.Epoch()) {
 		addr := common.Address{}
-		blsPubKeyBytes := node.Consensus.PubKey.GetAddress()
+		blsPubKeyBytes := node.Consensus.LeaderPubKey.GetAddress()
 		addr.SetBytes(blsPubKeyBytes[:])
 		coinbase = addr // coinbase will be the bls address
 		header.SetCoinbase(coinbase)
--- a/node/node_test.go
+++ b/node/node_test.go
@ -16,6 +16,7 @@ import (
 	"github.com/harmony-one/harmony/drand"
 	"github.com/harmony-one/harmony/internal/shardchain"
 	"github.com/harmony-one/harmony/internal/utils"
+	"github.com/harmony-one/harmony/multibls"
 	"github.com/harmony-one/harmony/p2p"
 	"github.com/harmony-one/harmony/p2p/p2pimpl"
 	"github.com/harmony-one/harmony/shard"
@ -35,7 +36,7 @@ func TestNewNode(t *testing.T) {
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
 	consensus, err := consensus.New(
-		host, shard.BeaconChainShardID, leader, blsKey, decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot craeate consensus: %v", err)
@ -202,7 +203,7 @@ func TestAddPeers(t *testing.T) {
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
 	consensus, err := consensus.New(
-		host, shard.BeaconChainShardID, leader, blsKey, decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot craeate consensus: %v", err)
@ -252,7 +253,7 @@ func TestAddBeaconPeer(t *testing.T) {
 	}
 	decider := quorum.NewDecider(quorum.SuperMajorityVote)
 	consensus, err := consensus.New(
-		host, shard.BeaconChainShardID, leader, blsKey, decider,
+		host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider,
 	)
 	if err != nil {
 		t.Fatalf("Cannot craeate consensus: %v", err)