From ae362c24c62055521f859a891657364009dbe20d Mon Sep 17 00:00:00 2001 From: Ganesha Upadhyaya Date: Tue, 3 Mar 2020 18:46:11 -0800 Subject: [PATCH] multi-bls key support (#2249) refactoring to multibls package to avoid cyclic dependency and better structure fixing import issue fixing more imports fixing a bug that causes nil pointer for explorer node fixing a bug in the super majority vote changing config to have same epoch for cross link and staking fixing the compilation error addressing PR comments leaving todo for assuming order between pub/pri keys pairs fixing goimports removing unwanted warning message --- cmd/harmony/main.go | 154 ++++++++++++++++------- consensus/consensus.go | 33 +++-- consensus/consensus_service.go | 34 ++--- consensus/consensus_service_test.go | 17 +-- consensus/consensus_test.go | 3 +- consensus/consensus_v2.go | 22 +++- consensus/consensus_viewchange_msg.go | 17 +-- consensus/construct.go | 14 +-- consensus/construct_test.go | 11 +- consensus/fbft_log_test.go | 6 +- consensus/leader.go | 35 +++--- consensus/quorum/one-node-one-vote.go | 10 +- consensus/quorum/one-node-staked-vote.go | 13 +- consensus/quorum/quorum.go | 11 +- consensus/threshold.go | 31 +++-- consensus/validator.go | 84 +++++++------ consensus/view_change.go | 73 ++++++----- internal/chain/engine.go | 5 +- internal/configs/node/config.go | 9 +- internal/configs/node/config_test.go | 3 +- multibls/multibls.go | 73 +++++++++++ node/node.go | 4 +- node/node_cross_link.go | 3 +- node/node_handler.go | 2 +- node/node_handler_test.go | 5 +- node/node_metrics.go | 12 +- node/node_newblock.go | 4 +- node/node_test.go | 7 +- 28 files changed, 459 insertions(+), 236 deletions(-) create mode 100644 multibls/multibls.go diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 5b24e19ac..39e647759 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -1,7 +1,6 @@ package main import ( - "encoding/hex" "flag" "fmt" "io/ioutil" @@ -10,6 +9,7 @@ import ( "os" "os/signal" "path" + "path/filepath" "runtime" "strconv" "strings" @@ -32,6 +32,7 @@ import ( "github.com/harmony-one/harmony/internal/memprofiling" "github.com/harmony-one/harmony/internal/shardchain" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/node" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" @@ -101,6 +102,7 @@ var ( enableMemProfiling = flag.Bool("enableMemProfiling", false, "Enable memsize logging.") enableGC = flag.Bool("enableGC", true, "Enable calling garbage collector manually .") blsKeyFile = flag.String("blskey_file", "", "The encrypted file of bls serialized private key by passphrase.") + blsFolder = flag.String("blsfolder", ".hmy/blskeys", "The folder that stores the bls keys; same blspass is used to decrypt all bls keys; all bls keys mapped to same shard") blsPass = flag.String("blspass", "", "The file containing passphrase to decrypt the encrypted bls file.") blsPassphrase string // Sharding configuration parameters for devnet @@ -114,7 +116,7 @@ var ( // Use a separate log file to log libp2p traces logP2P = flag.Bool("log_p2p", false, "log libp2p debug info") - initialAccount = &genesis.DeployAccount{} + initialAccounts = []*genesis.DeployAccount{} // logging verbosity verbosity = flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)") // dbDir is the database directory. @@ -188,8 +190,12 @@ func passphraseForBls() { return } - if *blsKeyFile == "" || *blsPass == "" { - fmt.Println("Internal nodes need to have pass to decrypt blskey") + if *blsKeyFile == "" && *blsFolder == "" { + fmt.Println("blskey_file or blsfolder option must be provided") + os.Exit(101) + } + if *blsPass == "" { + fmt.Println("Internal nodes need to have blspass to decrypt blskey") os.Exit(101) } passphrase, err := utils.GetPassphraseFromSource(*blsPass) @@ -200,27 +206,41 @@ func passphraseForBls() { blsPassphrase = passphrase } +func findAccountsByPubKeys(config shardingconfig.Instance, pubKeys []*bls.PublicKey) { + for _, key := range pubKeys { + keyStr := key.SerializeToHexStr() + _, account := config.FindAccount(keyStr) + if account != nil { + initialAccounts = append(initialAccounts, account) + } + } +} + func setupLegacyNodeAccount() error { genesisShardingConfig := shard.Schedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) - pubKey := setupConsensusKey(nodeconfig.GetDefaultConfig()) + multiBlsPubKey := setupConsensusKey(nodeconfig.GetDefaultConfig()) reshardingEpoch := genesisShardingConfig.ReshardingEpoch() if reshardingEpoch != nil && len(reshardingEpoch) > 0 { for _, epoch := range reshardingEpoch { config := shard.Schedule.InstanceForEpoch(epoch) - _, initialAccount = config.FindAccount(pubKey.SerializeToHexStr()) - if initialAccount != nil { + findAccountsByPubKeys(config, multiBlsPubKey.PublicKey) + if len(initialAccounts) != 0 { break } } } else { - _, initialAccount = genesisShardingConfig.FindAccount(pubKey.SerializeToHexStr()) + findAccountsByPubKeys(genesisShardingConfig, multiBlsPubKey.PublicKey) + } + + if len(initialAccounts) == 0 { + fmt.Fprintf(os.Stderr, "ERROR cannot find your BLS key in the genesis/FN tables: %s\n", multiBlsPubKey.SerializeToHexStr()) + os.Exit(100) } - if initialAccount == nil { - return errors.Errorf("cannot find key %s in table", pubKey.SerializeToHexStr()) + for _, account := range initialAccounts { + fmt.Printf("My Genesis Account: %v\n", *account) } - fmt.Printf("My Genesis Account: %v\n", *initialAccount) return nil } @@ -230,39 +250,82 @@ func setupStakingNodeAccount() error { if err != nil { return errors.Wrap(err, "cannot determine shard to join") } - initialAccount = &genesis.DeployAccount{} - initialAccount.ShardID = shardID - initialAccount.BlsPublicKey = pubKey.SerializeToHexStr() - initialAccount.Address = "" + for _, blsKey := range pubKey.PublicKey { + initialAccount := &genesis.DeployAccount{} + initialAccount.ShardID = shardID + initialAccount.BlsPublicKey = blsKey.SerializeToHexStr() + initialAccount.Address = "" + initialAccounts = append(initialAccounts, initialAccount) + } return nil } -func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) *bls.PublicKey { - consensusPriKey, err := blsgen.LoadBlsKeyWithPassPhrase(*blsKeyFile, blsPassphrase) +func readMultiBlsKeys(consensusMultiBlsPriKey *multibls.PrivateKey, consensusMultiBlsPubKey *multibls.PublicKey) error { + multiBlsKeyDir := blsFolder + blsKeyFiles, err := ioutil.ReadDir(*multiBlsKeyDir) if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "ERROR when loading bls key, err :%v\n", err) - os.Exit(100) + return err } - pubKey := consensusPriKey.GetPublicKey() - // Consensus keys are the BLS12-381 keys used to sign consensus messages - nodeConfig.ConsensusPriKey, nodeConfig.ConsensusPubKey = consensusPriKey, consensusPriKey.GetPublicKey() - if nodeConfig.ConsensusPriKey == nil || nodeConfig.ConsensusPubKey == nil { - fmt.Println("error to get consensus keys.") - os.Exit(100) + for _, blsKeyFile := range blsKeyFiles { + if filepath.Ext(blsKeyFile.Name()) != ".key" { + fmt.Println("BLS key file should have .key file extension, found", blsKeyFile.Name()) + continue + } + blsKeyFilePath := path.Join(*multiBlsKeyDir, blsKeyFile.Name()) + consensusPriKey, err := blsgen.LoadBlsKeyWithPassPhrase(blsKeyFilePath, blsPassphrase) // uses the same bls passphrase for multiple bls keys + if err != nil { + return err + } + // TODO: assumes order between public/private key pairs + multibls.AppendPriKey(consensusMultiBlsPriKey, consensusPriKey) + multibls.AppendPubKey(consensusMultiBlsPubKey, consensusPriKey.GetPublicKey()) } - return pubKey + + return nil +} + +func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) multibls.PublicKey { + consensusMultiPriKey := &multibls.PrivateKey{} + consensusMultiPubKey := &multibls.PublicKey{} + + if *blsKeyFile != "" { + consensusPriKey, err := blsgen.LoadBlsKeyWithPassPhrase(*blsKeyFile, blsPassphrase) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "ERROR when loading bls key, err :%v\n", err) + os.Exit(100) + } + multibls.AppendPriKey(consensusMultiPriKey, consensusPriKey) + multibls.AppendPubKey(consensusMultiPubKey, consensusPriKey.GetPublicKey()) + } else { + err := readMultiBlsKeys(consensusMultiPriKey, consensusMultiPubKey) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "ERROR when loading bls keys, err :%v\n", err) + os.Exit(100) + } + } + + // Consensus keys are the BLS12-381 keys used to sign consensus messages + nodeConfig.ConsensusPriKey = consensusMultiPriKey + nodeConfig.ConsensusPubKey = consensusMultiPubKey + + return *consensusMultiPubKey } func createGlobalConfig() (*nodeconfig.ConfigType, error) { var err error - nodeConfig := nodeconfig.GetShardConfig(initialAccount.ShardID) + if len(initialAccounts) == 0 { + initialAccounts = append(initialAccounts, &genesis.DeployAccount{ShardID: uint32(*shardID)}) + } + nodeConfig := nodeconfig.GetShardConfig(initialAccounts[0].ShardID) if *nodeType == "validator" { // Set up consensus keys. setupConsensusKey(nodeConfig) } else { - nodeConfig.ConsensusPriKey = &bls.SecretKey{} // set dummy bls key for consensus object + // set dummy bls key for consensus object + nodeConfig.ConsensusPriKey = multibls.GetPrivateKey(&bls.SecretKey{}) + nodeConfig.ConsensusPubKey = multibls.GetPublicKey(&bls.PublicKey{}) } // Set network type @@ -279,7 +342,7 @@ func createGlobalConfig() (*nodeconfig.ConfigType, error) { *keyFile) } - selfPeer := p2p.Peer{IP: *ip, Port: *port, ConsensusPubKey: nodeConfig.ConsensusPubKey} + selfPeer := p2p.Peer{IP: *ip, Port: *port, ConsensusPubKey: nodeConfig.ConsensusPubKey.PublicKey[0]} myHost, err = p2pimpl.NewHost(&selfPeer, nodeConfig.P2pPriKey) if err != nil { @@ -317,12 +380,14 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentConsensus.Decider.SetShardIDProvider(func() (uint32, error) { return currentConsensus.ShardID, nil }) - currentConsensus.Decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + currentConsensus.Decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return currentConsensus.PubKey, nil }) - if initialAccount.Address != "" { // staking validator doesn't have to specify ECDSA address - currentConsensus.SelfAddress = common.ParseAddr(initialAccount.Address) + // staking validator doesn't have to specify ECDSA address + currentConsensus.SelfAddresses = map[string]ethCommon.Address{} + for _, initialAccount := range initialAccounts { + currentConsensus.SelfAddresses[initialAccount.BlsPublicKey] = common.ParseAddr(initialAccount.Address) } if err != nil { @@ -540,17 +605,20 @@ func main() { os.Exit(1) } } - fmt.Printf("%s mode; node key %s -> shard %d\n", - map[bool]string{false: "Legacy", true: "Staking"}[*stakingFlag], - nodeconfig.GetDefaultConfig().ConsensusPubKey.SerializeToHexStr(), - initialAccount.ShardID) - + if *nodeType == "validator" { + fmt.Printf("%s mode; node key %s -> shard %d\n", + map[bool]string{false: "Legacy", true: "Staking"}[*stakingFlag], + nodeconfig.GetDefaultConfig().ConsensusPubKey.SerializeToHexStr(), + initialAccounts[0].ShardID) + } if *nodeType != "validator" && *shardID >= 0 { - utils.Logger().Info(). - Uint32("original", initialAccount.ShardID). - Int("override", *shardID). - Msg("ShardID Override") - initialAccount.ShardID = uint32(*shardID) + for _, initialAccount := range initialAccounts { + utils.Logger().Info(). + Uint32("original", initialAccount.ShardID). + Int("override", *shardID). + Msg("ShardID Override") + initialAccount.ShardID = uint32(*shardID) + } } nodeConfig, err := createGlobalConfig() @@ -614,7 +682,7 @@ func main() { } utils.Logger().Info(). - Str("BlsPubKey", hex.EncodeToString(nodeConfig.ConsensusPubKey.Serialize())). + Str("BlsPubKey", nodeConfig.ConsensusPubKey.SerializeToHexStr()). Uint32("ShardID", nodeConfig.ShardID). Str("ShardGroupID", nodeConfig.GetShardGroupID().String()). Str("BeaconGroupID", nodeConfig.GetBeaconGroupID().String()). diff --git a/consensus/consensus.go b/consensus/consensus.go index 8b734d924..9a7b4bb95 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -14,8 +14,10 @@ import ( bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/memprofiling" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/staking/slash" + "github.com/pkg/errors" ) const ( @@ -23,6 +25,8 @@ const ( vdfAndSeedSize = 548 // size of VDF/Proof and Seed ) +var errLeaderPriKeyNotFound = errors.New("getting leader private key from consensus public keys failed") + // Consensus is the main struct with all states and data related to consensus process. type Consensus struct { Decider quorum.Decider @@ -75,9 +79,9 @@ type Consensus struct { MinPeers int pubKeyLock sync.Mutex // private/public keys of current node - priKey *bls.SecretKey - PubKey *bls.PublicKey - SelfAddress common.Address + priKey *multibls.PrivateKey + PubKey *multibls.PublicKey + SelfAddresses map[string]common.Address // the publickey of leader LeaderPubKey *bls.PublicKey viewID uint64 @@ -173,11 +177,26 @@ func (consensus *Consensus) GetBlockReward() *big.Int { return consensus.lastBlockReward } +// GetLeaderPrivateKey returns leader private key if node is the leader +func (consensus *Consensus) GetLeaderPrivateKey(leaderKey *bls.PublicKey) (*bls.SecretKey, error) { + for i, key := range consensus.PubKey.PublicKey { + if key.IsEqual(leaderKey) { + return consensus.priKey.PrivateKey[i], nil + } + } + return nil, errors.Wrapf(errLeaderPriKeyNotFound, leaderKey.SerializeToHexStr()) +} + +// GetConsensusLeaderPrivateKey returns consensus leader private key if node is the leader +func (consensus *Consensus) GetConsensusLeaderPrivateKey() (*bls.SecretKey, error) { + return consensus.GetLeaderPrivateKey(consensus.LeaderPubKey) +} + // TODO: put shardId into chain reader's chain config // New create a new Consensus record func New( - host p2p.Host, shard uint32, leader p2p.Peer, blsPriKey *bls.SecretKey, + host p2p.Host, shard uint32, leader p2p.Peer, multiBlsPriKey *multibls.PrivateKey, Decider quorum.Decider, ) (*Consensus, error) { consensus := Consensus{} @@ -194,9 +213,9 @@ func New( consensus.consensusTimeout = createTimeout() consensus.validators.Store(leader.ConsensusPubKey.SerializeToHexStr(), leader) - if blsPriKey != nil { - consensus.priKey = blsPriKey - consensus.PubKey = blsPriKey.GetPublicKey() + if multiBlsPriKey != nil { + consensus.priKey = multiBlsPriKey + consensus.PubKey = multiBlsPriKey.GetPublicKey() utils.Logger().Info(). Str("publicKey", consensus.PubKey.SerializeToHexStr()).Msg("My Public Key") } else { diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index c8a94ee4e..8d0eb611b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -20,6 +20,7 @@ import ( "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/profiler" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" @@ -61,13 +62,11 @@ var ( ) // Signs the consensus message and returns the marshaled message. -func (consensus *Consensus) signAndMarshalConsensusMessage( - message *msg_pb.Message, -) ([]byte, error) { - if err := consensus.signConsensusMessage(message); err != nil { +func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Message, + priKey *bls.SecretKey) ([]byte, error) { + if err := consensus.signConsensusMessage(message, priKey); err != nil { return empty, err } - marshaledMessage, err := protobuf.Marshal(message) if err != nil { return empty, err @@ -121,14 +120,15 @@ func NewFaker() *Consensus { } // Sign on the hash of the message -func (consensus *Consensus) signMessage(message []byte) []byte { +func (consensus *Consensus) signMessage(message []byte, priKey *bls.SecretKey) []byte { hash := hash.Keccak256(message) - signature := consensus.priKey.SignHash(hash[:]) + signature := priKey.SignHash(hash[:]) return signature.Serialize() } // Sign on the consensus message signature field. -func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message) error { +func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message, + priKey *bls.SecretKey) error { message.Signature = nil // TODO: use custom serialization method rather than protobuf marshaledMessage, err := protobuf.Marshal(message) @@ -136,7 +136,7 @@ func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message) error return err } // 64 byte of signature on previous data - signature := consensus.signMessage(marshaledMessage) + signature := consensus.signMessage(marshaledMessage, priKey) message.Signature = signature return nil } @@ -381,7 +381,7 @@ func (consensus *Consensus) reportMetrics(block types.Block) { txHashes = append(txHashes, hex.EncodeToString(txHash[:])) } metrics := map[string]interface{}{ - "key": hex.EncodeToString(consensus.PubKey.Serialize()), + "key": hex.EncodeToString(consensus.LeaderPubKey.Serialize()), "tps": tps, "txCount": numOfTxs, "nodeCount": consensus.Decider.ParticipantsCount() + 1, @@ -480,7 +480,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { decider.SetShardIDProvider(func() (uint32, error) { return consensus.ShardID, nil }) - decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return consensus.PubKey, nil }) consensus.Decider = decider @@ -577,15 +577,15 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { } } - for i := range pubKeys { + for _, key := range pubKeys { // in committee - if pubKeys[i].IsEqual(consensus.PubKey) { + if consensus.PubKey.Contains(key) { if hasError { return Syncing } // If the leader changed and I myself become the leader - if !consensus.LeaderPubKey.IsEqual(oldLeader) && consensus.LeaderPubKey.IsEqual(consensus.PubKey) { + if !consensus.LeaderPubKey.IsEqual(oldLeader) && consensus.IsLeader() { go func() { utils.Logger().Debug(). Str("myKey", consensus.PubKey.SerializeToHexStr()). @@ -605,8 +605,10 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - if consensus.PubKey != nil && consensus.LeaderPubKey != nil { - return consensus.PubKey.IsEqual(consensus.LeaderPubKey) + for _, key := range consensus.PubKey.PublicKey { + if key.IsEqual(consensus.LeaderPubKey) { + return true + } } return false } diff --git a/consensus/consensus_service_test.go b/consensus/consensus_service_test.go index d3d77ddc0..f152569ef 100644 --- a/consensus/consensus_service_test.go +++ b/consensus/consensus_service_test.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -23,7 +24,7 @@ func TestPopulateMessageFields(t *testing.T) { blsPriKey := bls.RandPrivateKey() decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, shard.BeaconChainShardID, leader, blsPriKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -37,9 +38,9 @@ func TestPopulateMessageFields(t *testing.T) { Consensus: &msg_pb.ConsensusRequest{}, }, } - consensusMsg := consensus.populateMessageFields( - msg.GetConsensus(), consensus.blockHash[:], - ) + + consensusMsg := consensus.populateMessageFields(msg.GetConsensus(), consensus.blockHash[:], + blsPriKey.GetPublicKey()) if consensusMsg.ViewId != 2 { t.Errorf("Consensus ID is not populated correctly") @@ -60,8 +61,9 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) { t.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -70,7 +72,7 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) { consensus.blockHash = [32]byte{} msg := &msg_pb.Message{} - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(msg) + marshaledMessage, err := consensus.signAndMarshalConsensusMessage(msg, blsPriKey) if err != nil || len(marshaledMessage) == 0 { t.Errorf("Failed to sign and marshal the message: %s", err) @@ -88,8 +90,9 @@ func TestSetViewID(t *testing.T) { t.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index 87a1ee11f..f642c6c92 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -20,7 +21,7 @@ func TestNew(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(bls.RandPrivateKey()), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 765ffb475..d8e5fa32d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -109,8 +109,13 @@ func (consensus *Consensus) finalizeCommits() { Int64("NumCommits", consensus.Decider.SignersCount(quorum.Commit)). Msg("[Finalizing] Finalizing Block") beforeCatchupNum := consensus.blockNum + leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + if err != nil { + consensus.getLogger().Error().Err(err).Msg("[FinalizeCommits] leader not found") + return + } // Construct committed message - network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil) + network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil, leaderPriKey.GetPublicKey(), leaderPriKey) if err != nil { consensus.getLogger().Warn().Err(err). Msg("[FinalizeCommits] Unable to construct Committed message") @@ -182,7 +187,7 @@ func (consensus *Consensus) finalizeCommits() { Uint64("epochNum", block.Epoch().Uint64()). Uint64("ViewId", block.Header().ViewID().Uint64()). Str("blockHash", block.Hash().String()). - Int("index", consensus.Decider.IndexOf(consensus.PubKey)). + Int("index", consensus.Decider.IndexOf(consensus.LeaderPubKey)). Int("numTxns", len(block.Transactions())). Int("numStakingTxns", len(block.StakingTransactions())). Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!") @@ -491,10 +496,15 @@ func (consensus *Consensus) Start( } // GenerateVrfAndProof generates new VRF/Proof from hash of previous block -func (consensus *Consensus) GenerateVrfAndProof( - newBlock *types.Block, vrfBlockNumbers []uint64, -) []uint64 { - sk := vrf_bls.NewVRFSigner(consensus.priKey) +func (consensus *Consensus) GenerateVrfAndProof(newBlock *types.Block, vrfBlockNumbers []uint64) []uint64 { + key, err := consensus.GetConsensusLeaderPrivateKey() + if err != nil { + consensus.getLogger().Error(). + Err(err). + Msg("[GenerateVrfAndProof] VRF generation error") + return vrfBlockNumbers + } + sk := vrf_bls.NewVRFSigner(key) blockHash := [32]byte{} previousHeader := consensus.ChainReader.GetHeaderByNumber( newBlock.NumberU64() - 1, diff --git a/consensus/consensus_viewchange_msg.go b/consensus/consensus_viewchange_msg.go index e1d7c7864..2409de44a 100644 --- a/consensus/consensus_viewchange_msg.go +++ b/consensus/consensus_viewchange_msg.go @@ -3,6 +3,7 @@ package consensus import ( "encoding/binary" + "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/api/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" bls_cosi "github.com/harmony-one/harmony/crypto/bls" @@ -10,7 +11,7 @@ import ( ) // construct the view change message -func (consensus *Consensus) constructViewChangeMessage() []byte { +func (consensus *Consensus) constructViewChangeMessage(pubKey *bls.PublicKey, priKey *bls.SecretKey) []byte { message := &msg_pb.Message{ ServiceType: msg_pb.ServiceType_CONSENSUS, Type: msg_pb.MessageType_VIEWCHANGE, @@ -24,7 +25,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { vcMsg.BlockNum = consensus.blockNum vcMsg.ShardId = consensus.ShardID // sender address - vcMsg.SenderPubkey = consensus.PubKey.Serialize() + vcMsg.SenderPubkey = pubKey.Serialize() // next leader key already updated vcMsg.LeaderPubkey = consensus.LeaderPubKey.Serialize() @@ -49,7 +50,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { Str("pubKey", consensus.PubKey.SerializeToHexStr()). Msg("[constructViewChangeMessage]") - sign := consensus.priKey.SignHash(msgToSign) + sign := priKey.SignHash(msgToSign) if sign != nil { vcMsg.ViewchangeSig = sign.Serialize() } else { @@ -58,14 +59,14 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, consensus.current.ViewID()) - sign1 := consensus.priKey.SignHash(viewIDBytes) + sign1 := priKey.SignHash(viewIDBytes) if sign1 != nil { vcMsg.ViewidSig = sign1.Serialize() } else { utils.Logger().Error().Msg("unable to serialize viewID signature") } - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message) + marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey) if err != nil { utils.Logger().Error().Err(err). Msg("[constructViewChangeMessage] failed to sign and marshal the viewchange message") @@ -74,7 +75,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { } // new leader construct newview message -func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte { +func (consensus *Consensus) constructNewViewMessage(viewID uint64, pubKey *bls.PublicKey, priKey *bls.SecretKey) []byte { message := &msg_pb.Message{ ServiceType: msg_pb.ServiceType_CONSENSUS, Type: msg_pb.MessageType_NEWVIEW, @@ -88,7 +89,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte { vcMsg.BlockNum = consensus.blockNum vcMsg.ShardId = consensus.ShardID // sender address - vcMsg.SenderPubkey = consensus.PubKey.Serialize() + vcMsg.SenderPubkey = pubKey.Serialize() vcMsg.Payload = consensus.m1Payload sig2arr := consensus.GetNilSigsArray(viewID) @@ -108,7 +109,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte { vcMsg.M3Bitmap = consensus.viewIDBitmap[viewID].Bitmap } - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message) + marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey) if err != nil { utils.Logger().Error().Err(err). Msg("[constructNewViewMessage] failed to sign and marshal the new view message") diff --git a/consensus/construct.go b/consensus/construct.go index 76d85d804..5565f336d 100644 --- a/consensus/construct.go +++ b/consensus/construct.go @@ -22,7 +22,7 @@ type NetworkMessage struct { // Populates the common basic fields for all consensus message. func (consensus *Consensus) populateMessageFields( - request *msg_pb.ConsensusRequest, blockHash []byte, + request *msg_pb.ConsensusRequest, blockHash []byte, pubKey *bls.PublicKey, ) *msg_pb.ConsensusRequest { request.ViewId = consensus.viewID request.BlockNum = consensus.blockNum @@ -30,13 +30,13 @@ func (consensus *Consensus) populateMessageFields( // 32 byte block hash request.BlockHash = blockHash // sender address - request.SenderPubkey = consensus.PubKey.Serialize() + request.SenderPubkey = pubKey.Serialize() return request } // construct is the single creation point of messages intended for the wire. func (consensus *Consensus) construct( - p msg_pb.MessageType, payloadForSign []byte, + p msg_pb.MessageType, payloadForSign []byte, pubKey *bls.PublicKey, priKey *bls.SecretKey, ) (*NetworkMessage, error) { message := &msg_pb.Message{ ServiceType: msg_pb.ServiceType_CONSENSUS, @@ -51,7 +51,7 @@ func (consensus *Consensus) construct( ) consensusMsg = consensus.populateMessageFields( - message.GetConsensus(), consensus.blockHash[:], + message.GetConsensus(), consensus.blockHash[:], pubKey, ) // Do the signing, 96 byte of bls signature @@ -67,11 +67,11 @@ func (consensus *Consensus) construct( buffer.Write(consensus.prepareBitmap.Bitmap) consensusMsg.Payload = buffer.Bytes() case msg_pb.MessageType_PREPARE: - if s := consensus.priKey.SignHash(consensusMsg.BlockHash); s != nil { + if s := priKey.SignHash(consensusMsg.BlockHash); s != nil { consensusMsg.Payload = s.Serialize() } case msg_pb.MessageType_COMMIT: - if s := consensus.priKey.SignHash(payloadForSign); s != nil { + if s := priKey.SignHash(payloadForSign); s != nil { consensusMsg.Payload = s.Serialize() } case msg_pb.MessageType_COMMITTED: @@ -86,7 +86,7 @@ func (consensus *Consensus) construct( consensusMsg.Payload = consensus.blockHash[:] } - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message) + marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey) if err != nil { utils.Logger().Error().Err(err). Str("phase", p.String()). diff --git a/consensus/construct_test.go b/consensus/construct_test.go index 5c9c22fe4..88043bb61 100644 --- a/consensus/construct_test.go +++ b/consensus/construct_test.go @@ -9,6 +9,7 @@ import ( "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -22,14 +23,15 @@ func TestConstructAnnounceMessage(test *testing.T) { test.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { test.Fatalf("Cannot create consensus: %v", err) } consensus.blockHash = [32]byte{} - if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil); err != nil { + if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, blsPriKey.GetPublicKey(), blsPriKey); err != nil { test.Fatalf("could not construct announce: %v", err) } } @@ -47,8 +49,9 @@ func TestConstructPreparedMessage(test *testing.T) { test.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) @@ -78,7 +81,7 @@ func TestConstructPreparedMessage(test *testing.T) { test.Log(ctxerror.New("prepareBitmap.SetKey").WithCause(err)) } - network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil) + network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, blsPriKey.GetPublicKey(), blsPriKey) if err != nil { test.Errorf("Error when creating prepared message") } diff --git a/consensus/fbft_log_test.go b/consensus/fbft_log_test.go index e1a8269ff..5d95e2a3f 100644 --- a/consensus/fbft_log_test.go +++ b/consensus/fbft_log_test.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -21,14 +22,15 @@ func constructAnnounceMessage(t *testing.T) (*NetworkMessage, error) { t.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { t.Fatalf("Cannot create consensus: %v", err) } consensus.blockHash = [32]byte{} - return consensus.construct(msg_pb.MessageType_ANNOUNCE, nil) + return consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, blsPriKey.GetPublicKey(), blsPriKey) } func getConsensusMessage(payload []byte) (*msg_pb.Message, error) { diff --git a/consensus/leader.go b/consensus/leader.go index 9e73bf995..c536c77db 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -36,7 +36,13 @@ func (consensus *Consensus) announce(block *types.Block) { consensus.block = encodedBlock consensus.blockHeader = encodedBlockHeader - networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil) + + key, err := consensus.GetConsensusLeaderPrivateKey() + if err != nil { + consensus.getLogger().Warn().Err(err).Msg("[Announce] Node not a leader") + return + } + networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, key.GetPublicKey(), key) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_ANNOUNCE.String()). @@ -55,21 +61,20 @@ func (consensus *Consensus) announce(block *types.Block) { consensus.FBFTLog.AddBlock(block) // Leader sign the block hash itself - consensus.Decider.SubmitVote( - quorum.Prepare, - consensus.PubKey, - consensus.priKey.SignHash(consensus.blockHash[:]), - common.BytesToHash(consensus.blockHash[:]), - ) - if err := consensus.prepareBitmap.SetKey( - consensus.PubKey, true, - ); err != nil { - consensus.getLogger().Warn().Err(err).Msg( - "[Announce] Leader prepareBitmap SetKey failed", + for i, key := range consensus.PubKey.PublicKey { + consensus.Decider.SubmitVote( + quorum.Prepare, + key, + consensus.priKey.PrivateKey[i].SignHash(consensus.blockHash[:]), + common.BytesToHash(consensus.blockHash[:]), ) - return + if err := consensus.prepareBitmap.SetKey(key, true); err != nil { + consensus.getLogger().Warn().Err(err).Msg( + "[Announce] Leader prepareBitmap SetKey failed", + ) + return + } } - // Construct broadcast p2p message if err := consensus.msgSender.SendWithRetry( consensus.blockNum, msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{ @@ -268,7 +273,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { Offender: *addr, } consensus.SlashChan <- proof - }(consensus.SelfAddress) + }(consensus.SelfAddresses[consensus.LeaderPubKey.SerializeToHexStr()]) return } } diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index 63c0b2c27..b865b3bc2 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -120,10 +120,12 @@ func (v *uniformVoteWeight) AmIMemberOfCommitee() bool { } identity, _ := pubKeyFunc() everyone := v.DumpParticipants() - myVoterID := identity.SerializeToHexStr() - for i := range everyone { - if everyone[i] == myVoterID { - return true + for _, key := range identity.PublicKey { + myVoterID := key.SerializeToHexStr() + for i := range everyone { + if everyone[i] == myVoterID { + return true + } } } return false diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index eae22386f..87480ba15 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -244,10 +244,15 @@ func (v *stakedVoteWeight) AmIMemberOfCommitee() bool { return false } identity, _ := pubKeyFunc() - w := shard.BlsPublicKey{} - w.FromLibBLSPublicKey(identity) - _, ok := v.roster.Voters[w] - return ok + for _, key := range identity.PublicKey { + w := shard.BlsPublicKey{} + w.FromLibBLSPublicKey(key) + _, ok := v.roster.Voters[w] + if ok { + return true + } + } + return false } func newBox() *voteBox { diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 18b611b37..e24f777a5 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/harmony/consensus/votepower" bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" "github.com/pkg/errors" @@ -95,13 +96,13 @@ type SignatureReader interface { // DependencyInjectionWriter .. type DependencyInjectionWriter interface { SetShardIDProvider(func() (uint32, error)) - SetMyPublicKeyProvider(func() (*bls.PublicKey, error)) + SetMyPublicKeyProvider(func() (*multibls.PublicKey, error)) } // DependencyInjectionReader .. type DependencyInjectionReader interface { ShardIDProvider() func() (uint32, error) - MyPublicKey() func() (*bls.PublicKey, error) + MyPublicKey() func() (*multibls.PublicKey, error) } // Decider .. @@ -151,7 +152,7 @@ type cIdentities struct { type depInject struct { shardIDProvider func() (uint32, error) - publicKeyProvider func() (*bls.PublicKey, error) + publicKeyProvider func() (*multibls.PublicKey, error) } func (s *cIdentities) AggregateVotes(p Phase) *bls.Sign { @@ -323,11 +324,11 @@ func (d *depInject) ShardIDProvider() func() (uint32, error) { return d.shardIDProvider } -func (d *depInject) SetMyPublicKeyProvider(p func() (*bls.PublicKey, error)) { +func (d *depInject) SetMyPublicKeyProvider(p func() (*multibls.PublicKey, error)) { d.publicKeyProvider = p } -func (d *depInject) MyPublicKey() func() (*bls.PublicKey, error) { +func (d *depInject) MyPublicKey() func() (*multibls.PublicKey, error) { return d.publicKeyProvider } diff --git a/consensus/threshold.go b/consensus/threshold.go index f44d57d5b..f87695eb4 100644 --- a/consensus/threshold.go +++ b/consensus/threshold.go @@ -14,8 +14,13 @@ import ( func (consensus *Consensus) didReachPrepareQuorum() error { logger := utils.Logger() logger.Debug().Msg("[OnPrepare] Received Enough Prepare Signatures") + leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + if err != nil { + utils.Logger().Warn().Err(err).Msg("[OnPrepare] leader not found") + return err + } // Construct and broadcast prepared message - networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARED, nil) + networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, consensus.LeaderPubKey, leaderPriKey) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_PREPARED.String()). @@ -33,18 +38,22 @@ func (consensus *Consensus) didReachPrepareQuorum() error { blockNumHash := [8]byte{} binary.LittleEndian.PutUint64(blockNumHash[:], consensus.blockNum) commitPayload := append(blockNumHash[:], consensus.blockHash[:]...) - consensus.Decider.SubmitVote( - quorum.Commit, - consensus.PubKey, - consensus.priKey.SignHash(commitPayload), - common.BytesToHash(consensus.blockHash[:]), - ) - if err := consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil { - consensus.getLogger().Debug().Msg("[OnPrepare] Leader commit bitmap set failed") - return err - } + // so by this point, everyone has committed to the blockhash of this block + // in prepare and so this is the actual block. + for i, key := range consensus.PubKey.PublicKey { + consensus.Decider.SubmitVote( + quorum.Commit, + key, + consensus.priKey.PrivateKey[i].SignHash(commitPayload), + common.BytesToHash(consensus.blockHash[:]), + ) + if err := consensus.commitBitmap.SetKey(key, true); err != nil { + consensus.getLogger().Debug().Msg("[OnPrepare] Leader commit bitmap set failed") + return err + } + } if err := consensus.msgSender.SendWithRetry( consensus.blockNum, msg_pb.MessageType_PREPARED, []nodeconfig.GroupID{ diff --git a/consensus/validator.go b/consensus/validator.go index a0ac0acad..d47107988 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -58,26 +58,27 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } func (consensus *Consensus) prepare() { - networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil) - if err != nil { - consensus.getLogger().Err(err). - Str("message-type", msg_pb.MessageType_PREPARE.String()). - Msg("could not construct message") - return - } + groupID := []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} + for i, key := range consensus.PubKey.PublicKey { + networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil, key, consensus.priKey.PrivateKey[i]) + if err != nil { + consensus.getLogger().Err(err). + Str("message-type", msg_pb.MessageType_PREPARE.String()). + Msg("could not construct message") + return + } - // TODO: this will not return immediatey, may block - if err := consensus.msgSender.SendWithoutRetry( - []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), - }, - host.ConstructP2pMessage(byte(17), networkMessage.Bytes), - ); err != nil { - consensus.getLogger().Warn().Err(err).Msg("[OnAnnounce] Cannot send prepare message") - } else { - consensus.getLogger().Info(). - Str("blockHash", hex.EncodeToString(consensus.blockHash[:])). - Msg("[OnAnnounce] Sent Prepare Message!!") + // TODO: this will not return immediatey, may block + if err := consensus.msgSender.SendWithoutRetry( + groupID, + host.ConstructP2pMessage(byte(17), networkMessage.Bytes), + ); err != nil { + consensus.getLogger().Warn().Err(err).Msg("[OnAnnounce] Cannot send prepare message") + } else { + consensus.getLogger().Info(). + Str("blockHash", hex.EncodeToString(consensus.blockHash[:])). + Msg("[OnAnnounce] Sent Prepare Message!!") + } } consensus.getLogger().Debug(). Str("From", consensus.phase.String()). @@ -192,29 +193,32 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { } blockNumBytes := make([]byte, 8) binary.LittleEndian.PutUint64(blockNumBytes, consensus.blockNum) - networkMessage, _ := consensus.construct( - msg_pb.MessageType_COMMIT, - append(blockNumBytes, consensus.blockHash[:]...), - ) - // TODO: genesis account node delay for 1 second, - // this is a temp fix for allows FN nodes to earning reward - if consensus.delayCommit > 0 { - time.Sleep(consensus.delayCommit) - } + groupID := []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} + for i, key := range consensus.PubKey.PublicKey { + networkMessage, _ := consensus.construct( + // TODO: should only sign on block hash + msg_pb.MessageType_COMMIT, + append(blockNumBytes, consensus.blockHash[:]...), + key, consensus.priKey.PrivateKey[i], + ) + // TODO: genesis account node delay for 1 second, + // this is a temp fix for allows FN nodes to earning reward + if consensus.delayCommit > 0 { + time.Sleep(consensus.delayCommit) + } - if err := consensus.msgSender.SendWithoutRetry( - []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, - host.ConstructP2pMessage(byte(17), networkMessage.Bytes), - ); err != nil { - consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!") - } else { - consensus.getLogger().Info(). - Uint64("blockNum", consensus.blockNum). - Hex("blockHash", consensus.blockHash[:]). - Msg("[OnPrepared] Sent Commit Message!!") + if err := consensus.msgSender.SendWithoutRetry( + groupID, + host.ConstructP2pMessage(byte(17), networkMessage.Bytes), + ); err != nil { + consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!") + } else { + consensus.getLogger().Info(). + Uint64("blockNum", consensus.blockNum). + Hex("blockHash", consensus.blockHash[:]). + Msg("[OnPrepared] Sent Commit Message!!") + } } - consensus.getLogger().Debug(). Str("From", consensus.phase.String()). Str("To", FBFTCommit.String()). diff --git a/consensus/view_change.go b/consensus/view_change.go index 1faf6793a..5970b3d6f 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -127,12 +127,14 @@ func (consensus *Consensus) startViewChange(viewID uint64) { Str("NextLeader", consensus.LeaderPubKey.SerializeToHexStr()). Msg("[startViewChange]") - msgToSend := consensus.constructViewChangeMessage() - consensus.host.SendMessageToGroups([]nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), - }, - host.ConstructP2pMessage(byte(17), msgToSend), - ) + for i, key := range consensus.PubKey.PublicKey { + msgToSend := consensus.constructViewChangeMessage(key, consensus.priKey.PrivateKey[i]) + consensus.host.SendMessageToGroups([]nodeconfig.GroupID{ + nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), + }, + host.ConstructP2pMessage(byte(17), msgToSend), + ) + } consensus.consensusTimeout[timeoutViewChange].SetDuration(duration) consensus.consensusTimeout[timeoutViewChange].Start() @@ -148,7 +150,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { return } newLeaderKey := recvMsg.LeaderPubkey - if !consensus.PubKey.IsEqual(newLeaderKey) { + newLeaderPriKey, err := consensus.GetLeaderPrivateKey(newLeaderKey) + if err != nil { return } @@ -214,13 +217,13 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { preparedMsg := consensus.FBFTLog.FindMessageByMaxViewID(preparedMsgs) if preparedMsg == nil { consensus.getLogger().Debug().Msg("[onViewChange] add my M2(NIL) type messaage") - consensus.nilSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(NIL) - consensus.nilBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true) + consensus.nilSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(NIL) + consensus.nilBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true) } else { consensus.getLogger().Debug().Msg("[onViewChange] add my M1 type messaage") msgToSign := append(preparedMsg.BlockHash[:], preparedMsg.Payload...) - consensus.bhpSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(msgToSign) - consensus.bhpBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true) + consensus.bhpSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(msgToSign) + consensus.bhpBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true) } } // add self m3 type message signature and bitmap @@ -228,8 +231,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { if !ok3 { viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, recvMsg.ViewID) - consensus.viewIDSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(viewIDBytes) - consensus.viewIDBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true) + consensus.viewIDSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(viewIDBytes) + consensus.viewIDBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true) } // m2 type message @@ -307,7 +310,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { copy(preparedMsg.BlockHash[:], recvMsg.Payload[:32]) preparedMsg.Payload = make([]byte, len(recvMsg.Payload)-32) copy(preparedMsg.Payload[:], recvMsg.Payload[32:]) - preparedMsg.SenderPubkey = consensus.PubKey + preparedMsg.SenderPubkey = newLeaderKey consensus.getLogger().Info().Msg("[onViewChange] New Leader Prepared Message Added") consensus.FBFTLog.AddMessage(&preparedMsg) } @@ -349,7 +352,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { // received enough view change messages, change state to normal consensus if consensus.Decider.IsQuorumAchievedByMask(consensus.viewIDBitmap[recvMsg.ViewID]) { consensus.current.SetMode(Normal) - consensus.LeaderPubKey = consensus.PubKey + consensus.LeaderPubKey = newLeaderKey consensus.ResetState() if len(consensus.m1Payload) == 0 { // TODO(Chao): explain why ReadySignal is sent only in this case but not the other case. @@ -379,12 +382,12 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { commitPayload := append(blockNumBytes[:], consensus.blockHash[:]...) consensus.Decider.SubmitVote( quorum.Commit, - consensus.PubKey, - consensus.priKey.SignHash(commitPayload), + newLeaderKey, + newLeaderPriKey.SignHash(commitPayload), common.BytesToHash(consensus.blockHash[:]), ) - if err = consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil { + if err = consensus.commitBitmap.SetKey(newLeaderKey, true); err != nil { consensus.getLogger().Debug(). Msg("[OnViewChange] New Leader commit bitmap set failed") return @@ -392,7 +395,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { } consensus.current.SetViewID(recvMsg.ViewID) - msgToSend := consensus.constructNewViewMessage(recvMsg.ViewID) + msgToSend := consensus.constructNewViewMessage(recvMsg.ViewID, newLeaderKey, newLeaderPriKey) consensus.getLogger().Warn(). Int("payloadSize", len(consensus.m1Payload)). @@ -541,21 +544,25 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { // Construct and send the commit message blockNumHash := make([]byte, 8) binary.LittleEndian.PutUint64(blockNumHash, consensus.blockNum) - network, err := consensus.construct( - msg_pb.MessageType_COMMIT, - append(blockNumHash, consensus.blockHash[:]...), - ) - if err != nil { - consensus.getLogger().Err(err).Msg("could not create commit message") - return + groupID := []nodeconfig.GroupID{ + nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} + for i, key := range consensus.PubKey.PublicKey { + network, err := consensus.construct( + msg_pb.MessageType_COMMIT, + append(blockNumHash, consensus.blockHash[:]...), + key, consensus.priKey.PrivateKey[i], + ) + if err != nil { + consensus.getLogger().Err(err).Msg("could not create commit message") + return + } + msgToSend := network.Bytes + consensus.getLogger().Info().Msg("onNewView === commit") + consensus.host.SendMessageToGroups( + groupID, + host.ConstructP2pMessage(byte(17), msgToSend), + ) } - msgToSend := network.Bytes - consensus.getLogger().Info().Msg("onNewView === commit") - consensus.host.SendMessageToGroups( - []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, - host.ConstructP2pMessage(byte(17), msgToSend), - ) consensus.getLogger().Debug(). Str("From", consensus.phase.String()). Str("To", FBFTCommit.String()). diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 9135333b3..743735eab 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -17,6 +17,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" "github.com/harmony-one/harmony/staking/availability" @@ -212,7 +213,7 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header) d.SetShardIDProvider(func() (uint32, error) { return parentHeader.ShardID(), nil }) - d.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + d.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return nil, nil }) d.SetVoters(slotList.FindCommitteeByID(parentHeader.ShardID()).Slots) @@ -444,7 +445,7 @@ func (e *engineImpl) VerifyHeaderWithSignature(chain engine.ChainReader, header d.SetShardIDProvider(func() (uint32, error) { return header.ShardID(), nil }) - d.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + d.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return nil, nil }) d.SetVoters(slotList.FindCommitteeByID(header.ShardID()).Slots) diff --git a/internal/configs/node/config.go b/internal/configs/node/config.go index 7f4c0829e..7d1edd9ce 100644 --- a/internal/configs/node/config.go +++ b/internal/configs/node/config.go @@ -8,9 +8,9 @@ import ( "math/big" "sync" - "github.com/harmony-one/bls/ffi/go/bls" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/staking/slash" p2p_crypto "github.com/libp2p/go-libp2p-crypto" @@ -83,8 +83,8 @@ type ConfigType struct { PushgatewayPort string // metrics pushgateway prometheus port StringRole string P2pPriKey p2p_crypto.PrivKey - ConsensusPriKey *bls.SecretKey - ConsensusPubKey *bls.PublicKey + ConsensusPriKey *multibls.PrivateKey + ConsensusPubKey *multibls.PublicKey // Database directory DBDir string networkType NetworkType @@ -280,7 +280,8 @@ func SetShardingSchedule(schedule shardingconfig.Schedule) { // consensus key. func (conf *ConfigType) ShardIDFromConsensusKey() (uint32, error) { var pubKey shard.BlsPublicKey - if err := pubKey.FromLibBLSPublicKey(conf.ConsensusPubKey); err != nil { + // all keys belong to same shard + if err := pubKey.FromLibBLSPublicKey(conf.ConsensusPubKey.PublicKey[0]); err != nil { return 0, errors.Wrapf(err, "cannot convert libbls public key %s to internal form", conf.ConsensusPubKey.SerializeToHexStr()) diff --git a/internal/configs/node/config_test.go b/internal/configs/node/config_test.go index 421935e9d..ef47d3178 100644 --- a/internal/configs/node/config_test.go +++ b/internal/configs/node/config_test.go @@ -9,6 +9,7 @@ import ( mock_shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding/mock" "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/multibls" ) func TestNodeConfigSingleton(t *testing.T) { @@ -115,7 +116,7 @@ func TestConfigType_ShardIDFromConsensusKey(t *testing.T) { schedule := mock_shardingconfig.NewMockSchedule(mc) schedule.EXPECT().InstanceForEpoch(tt.epoch).Return(instance) conf := &ConfigType{ - ConsensusPubKey: tt.fields.ConsensusPubKey, + ConsensusPubKey: multibls.GetPublicKey(tt.fields.ConsensusPubKey), networkType: tt.fields.networkType, shardingSchedule: schedule, } diff --git a/multibls/multibls.go b/multibls/multibls.go new file mode 100644 index 000000000..43c266c14 --- /dev/null +++ b/multibls/multibls.go @@ -0,0 +1,73 @@ +package multibls + +import ( + "strings" + + "github.com/harmony-one/bls/ffi/go/bls" +) + +// PrivateKey stores the bls secret keys that belongs to the node +type PrivateKey struct { + PrivateKey []*bls.SecretKey +} + +// PublicKey stores the bls public keys that belongs to the node +type PublicKey struct { + PublicKey []*bls.PublicKey +} + +// SerializeToHexStr wrapper +func (multiKey PublicKey) SerializeToHexStr() string { + var builder strings.Builder + for _, pubKey := range multiKey.PublicKey { + builder.WriteString(pubKey.SerializeToHexStr()) + } + return builder.String() +} + +// Contains wrapper +func (multiKey PublicKey) Contains(pubKey *bls.PublicKey) bool { + for _, key := range multiKey.PublicKey { + if key.IsEqual(pubKey) { + return true + } + } + return false +} + +// GetPublicKey wrapper +func (multiKey PrivateKey) GetPublicKey() *PublicKey { + pubKeys := make([]*bls.PublicKey, len(multiKey.PrivateKey)) + for i, key := range multiKey.PrivateKey { + pubKeys[i] = key.GetPublicKey() + } + return &PublicKey{PublicKey: pubKeys} +} + +// GetPrivateKey creates a multibls PrivateKey using bls.SecretKey +func GetPrivateKey(key *bls.SecretKey) *PrivateKey { + return &PrivateKey{PrivateKey: []*bls.SecretKey{key}} +} + +// GetPublicKey creates a multibls PublicKey using bls.PublicKey +func GetPublicKey(key *bls.PublicKey) *PublicKey { + return &PublicKey{PublicKey: []*bls.PublicKey{key}} +} + +// AppendPubKey appends a PublicKey to multibls PublicKey +func AppendPubKey(multiKey *PublicKey, key *bls.PublicKey) { + if multiKey != nil { + multiKey.PublicKey = append(multiKey.PublicKey, key) + } else { + multiKey = &PublicKey{PublicKey: []*bls.PublicKey{key}} + } +} + +// AppendPriKey appends a SecretKey to multibls PrivateKey +func AppendPriKey(multiKey *PrivateKey, key *bls.SecretKey) { + if multiKey != nil { + multiKey.PrivateKey = append(multiKey.PrivateKey, key) + } else { + multiKey = &PrivateKey{PrivateKey: []*bls.SecretKey{key}} + } +} diff --git a/node/node.go b/node/node.go index a366a8c95..b8c6560c2 100644 --- a/node/node.go +++ b/node/node.go @@ -652,8 +652,8 @@ func (node *Node) InitConsensusWithValidators() (err error) { "blockNum", blockNum) } - for i := range pubKeys { - if pubKeys[i].IsEqual(node.Consensus.PubKey) { + for _, key := range pubKeys { + if node.Consensus.PubKey.Contains(key) { utils.Logger().Info(). Uint64("blockNum", blockNum). Int("numPubKeys", len(pubKeys)). diff --git a/node/node_cross_link.go b/node/node_cross_link.go index b601c19f8..36016ea64 100644 --- a/node/node_cross_link.go +++ b/node/node_cross_link.go @@ -10,6 +10,7 @@ import ( bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/shard" ) @@ -163,7 +164,7 @@ func (node *Node) VerifyCrossLink(cl types.CrossLink) error { decider.SetShardIDProvider(func() (uint32, error) { return cl.ShardID(), nil }) - decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return nil, nil }) if _, err := decider.SetVoters(committee.Slots); err != nil { diff --git a/node/node_handler.go b/node/node_handler.go index 866b3e6f7..0bb6b119c 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -417,7 +417,7 @@ func (node *Node) PostConsensusProcessing( // TODO: randomly selected a few validators to broadcast messages instead of only leader broadcast // TODO: refactor the asynchronous calls to separate go routine. node.lastConsensusTime = time.Now().Unix() - if node.Consensus.PubKey.IsEqual(node.Consensus.LeaderPubKey) { + if node.Consensus.IsLeader() { if node.NodeConfig.ShardID == shard.BeaconChainShardID { node.BroadcastNewBlock(newBlock) } diff --git a/node/node_handler_test.go b/node/node_handler_test.go index 2bf7188b2..ddc34fc15 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -10,6 +10,7 @@ import ( "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -27,7 +28,7 @@ func TestAddNewBlock(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -65,7 +66,7 @@ func TestVerifyNewBlock(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/node/node_metrics.go b/node/node_metrics.go index 3b56e59aa..44f51c5fc 100644 --- a/node/node_metrics.go +++ b/node/node_metrics.go @@ -43,12 +43,14 @@ func (node *Node) UpdateTxPoolSizeForMetrics(txPoolSize uint64) { // UpdateBalanceForMetrics uppdates node balance for metrics service. func (node *Node) UpdateBalanceForMetrics() { - curBalance, err := node.GetBalanceOfAddress(node.Consensus.SelfAddress) - if err != nil { - return + for _, addr := range node.Consensus.SelfAddresses { + curBalance, err := node.GetBalanceOfAddress(addr) + if err != nil { + return + } + utils.Logger().Info().Msgf("Updating metrics node balance %d", curBalance.Uint64()) + metrics.UpdateNodeBalance(curBalance) } - utils.Logger().Info().Msgf("Updating metrics node balance %d", curBalance.Uint64()) - metrics.UpdateNodeBalance(curBalance) } // UpdateLastConsensusTimeForMetrics uppdates last consensus reached time for metrics service. diff --git a/node/node_newblock.go b/node/node_newblock.go index 5084bbd42..fa18672bd 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -84,7 +84,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { // Update worker's current header and state data in preparation to propose/process new transactions var ( - coinbase = node.Consensus.SelfAddress + coinbase = node.Consensus.SelfAddresses[node.Consensus.LeaderPubKey.SerializeToHexStr()] beneficiary = coinbase err error ) @@ -94,7 +94,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { // After staking, all coinbase will be the address of bls pub key if header := node.Worker.GetCurrentHeader(); node.Blockchain().Config().IsStaking(header.Epoch()) { addr := common.Address{} - blsPubKeyBytes := node.Consensus.PubKey.GetAddress() + blsPubKeyBytes := node.Consensus.LeaderPubKey.GetAddress() addr.SetBytes(blsPubKeyBytes[:]) coinbase = addr // coinbase will be the bls address header.SetCoinbase(coinbase) diff --git a/node/node_test.go b/node/node_test.go index 90d59afb9..e31a65fe3 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -16,6 +16,7 @@ import ( "github.com/harmony-one/harmony/drand" "github.com/harmony-one/harmony/internal/shardchain" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -35,7 +36,7 @@ func TestNewNode(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -202,7 +203,7 @@ func TestAddPeers(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -252,7 +253,7 @@ func TestAddBeaconPeer(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err)