diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 5b24e19ac..39e647759 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -1,7 +1,6 @@ package main import ( - "encoding/hex" "flag" "fmt" "io/ioutil" @@ -10,6 +9,7 @@ import ( "os" "os/signal" "path" + "path/filepath" "runtime" "strconv" "strings" @@ -32,6 +32,7 @@ import ( "github.com/harmony-one/harmony/internal/memprofiling" "github.com/harmony-one/harmony/internal/shardchain" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/node" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" @@ -101,6 +102,7 @@ var ( enableMemProfiling = flag.Bool("enableMemProfiling", false, "Enable memsize logging.") enableGC = flag.Bool("enableGC", true, "Enable calling garbage collector manually .") blsKeyFile = flag.String("blskey_file", "", "The encrypted file of bls serialized private key by passphrase.") + blsFolder = flag.String("blsfolder", ".hmy/blskeys", "The folder that stores the bls keys; same blspass is used to decrypt all bls keys; all bls keys mapped to same shard") blsPass = flag.String("blspass", "", "The file containing passphrase to decrypt the encrypted bls file.") blsPassphrase string // Sharding configuration parameters for devnet @@ -114,7 +116,7 @@ var ( // Use a separate log file to log libp2p traces logP2P = flag.Bool("log_p2p", false, "log libp2p debug info") - initialAccount = &genesis.DeployAccount{} + initialAccounts = []*genesis.DeployAccount{} // logging verbosity verbosity = flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)") // dbDir is the database directory. @@ -188,8 +190,12 @@ func passphraseForBls() { return } - if *blsKeyFile == "" || *blsPass == "" { - fmt.Println("Internal nodes need to have pass to decrypt blskey") + if *blsKeyFile == "" && *blsFolder == "" { + fmt.Println("blskey_file or blsfolder option must be provided") + os.Exit(101) + } + if *blsPass == "" { + fmt.Println("Internal nodes need to have blspass to decrypt blskey") os.Exit(101) } passphrase, err := utils.GetPassphraseFromSource(*blsPass) @@ -200,27 +206,41 @@ func passphraseForBls() { blsPassphrase = passphrase } +func findAccountsByPubKeys(config shardingconfig.Instance, pubKeys []*bls.PublicKey) { + for _, key := range pubKeys { + keyStr := key.SerializeToHexStr() + _, account := config.FindAccount(keyStr) + if account != nil { + initialAccounts = append(initialAccounts, account) + } + } +} + func setupLegacyNodeAccount() error { genesisShardingConfig := shard.Schedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) - pubKey := setupConsensusKey(nodeconfig.GetDefaultConfig()) + multiBlsPubKey := setupConsensusKey(nodeconfig.GetDefaultConfig()) reshardingEpoch := genesisShardingConfig.ReshardingEpoch() if reshardingEpoch != nil && len(reshardingEpoch) > 0 { for _, epoch := range reshardingEpoch { config := shard.Schedule.InstanceForEpoch(epoch) - _, initialAccount = config.FindAccount(pubKey.SerializeToHexStr()) - if initialAccount != nil { + findAccountsByPubKeys(config, multiBlsPubKey.PublicKey) + if len(initialAccounts) != 0 { break } } } else { - _, initialAccount = genesisShardingConfig.FindAccount(pubKey.SerializeToHexStr()) + findAccountsByPubKeys(genesisShardingConfig, multiBlsPubKey.PublicKey) + } + + if len(initialAccounts) == 0 { + fmt.Fprintf(os.Stderr, "ERROR cannot find your BLS key in the genesis/FN tables: %s\n", multiBlsPubKey.SerializeToHexStr()) + os.Exit(100) } - if initialAccount == nil { - return errors.Errorf("cannot find key %s in table", pubKey.SerializeToHexStr()) + for _, account := range initialAccounts { + fmt.Printf("My Genesis Account: %v\n", *account) } - fmt.Printf("My Genesis Account: %v\n", *initialAccount) return nil } @@ -230,39 +250,82 @@ func setupStakingNodeAccount() error { if err != nil { return errors.Wrap(err, "cannot determine shard to join") } - initialAccount = &genesis.DeployAccount{} - initialAccount.ShardID = shardID - initialAccount.BlsPublicKey = pubKey.SerializeToHexStr() - initialAccount.Address = "" + for _, blsKey := range pubKey.PublicKey { + initialAccount := &genesis.DeployAccount{} + initialAccount.ShardID = shardID + initialAccount.BlsPublicKey = blsKey.SerializeToHexStr() + initialAccount.Address = "" + initialAccounts = append(initialAccounts, initialAccount) + } return nil } -func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) *bls.PublicKey { - consensusPriKey, err := blsgen.LoadBlsKeyWithPassPhrase(*blsKeyFile, blsPassphrase) +func readMultiBlsKeys(consensusMultiBlsPriKey *multibls.PrivateKey, consensusMultiBlsPubKey *multibls.PublicKey) error { + multiBlsKeyDir := blsFolder + blsKeyFiles, err := ioutil.ReadDir(*multiBlsKeyDir) if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "ERROR when loading bls key, err :%v\n", err) - os.Exit(100) + return err } - pubKey := consensusPriKey.GetPublicKey() - // Consensus keys are the BLS12-381 keys used to sign consensus messages - nodeConfig.ConsensusPriKey, nodeConfig.ConsensusPubKey = consensusPriKey, consensusPriKey.GetPublicKey() - if nodeConfig.ConsensusPriKey == nil || nodeConfig.ConsensusPubKey == nil { - fmt.Println("error to get consensus keys.") - os.Exit(100) + for _, blsKeyFile := range blsKeyFiles { + if filepath.Ext(blsKeyFile.Name()) != ".key" { + fmt.Println("BLS key file should have .key file extension, found", blsKeyFile.Name()) + continue + } + blsKeyFilePath := path.Join(*multiBlsKeyDir, blsKeyFile.Name()) + consensusPriKey, err := blsgen.LoadBlsKeyWithPassPhrase(blsKeyFilePath, blsPassphrase) // uses the same bls passphrase for multiple bls keys + if err != nil { + return err + } + // TODO: assumes order between public/private key pairs + multibls.AppendPriKey(consensusMultiBlsPriKey, consensusPriKey) + multibls.AppendPubKey(consensusMultiBlsPubKey, consensusPriKey.GetPublicKey()) } - return pubKey + + return nil +} + +func setupConsensusKey(nodeConfig *nodeconfig.ConfigType) multibls.PublicKey { + consensusMultiPriKey := &multibls.PrivateKey{} + consensusMultiPubKey := &multibls.PublicKey{} + + if *blsKeyFile != "" { + consensusPriKey, err := blsgen.LoadBlsKeyWithPassPhrase(*blsKeyFile, blsPassphrase) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "ERROR when loading bls key, err :%v\n", err) + os.Exit(100) + } + multibls.AppendPriKey(consensusMultiPriKey, consensusPriKey) + multibls.AppendPubKey(consensusMultiPubKey, consensusPriKey.GetPublicKey()) + } else { + err := readMultiBlsKeys(consensusMultiPriKey, consensusMultiPubKey) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "ERROR when loading bls keys, err :%v\n", err) + os.Exit(100) + } + } + + // Consensus keys are the BLS12-381 keys used to sign consensus messages + nodeConfig.ConsensusPriKey = consensusMultiPriKey + nodeConfig.ConsensusPubKey = consensusMultiPubKey + + return *consensusMultiPubKey } func createGlobalConfig() (*nodeconfig.ConfigType, error) { var err error - nodeConfig := nodeconfig.GetShardConfig(initialAccount.ShardID) + if len(initialAccounts) == 0 { + initialAccounts = append(initialAccounts, &genesis.DeployAccount{ShardID: uint32(*shardID)}) + } + nodeConfig := nodeconfig.GetShardConfig(initialAccounts[0].ShardID) if *nodeType == "validator" { // Set up consensus keys. setupConsensusKey(nodeConfig) } else { - nodeConfig.ConsensusPriKey = &bls.SecretKey{} // set dummy bls key for consensus object + // set dummy bls key for consensus object + nodeConfig.ConsensusPriKey = multibls.GetPrivateKey(&bls.SecretKey{}) + nodeConfig.ConsensusPubKey = multibls.GetPublicKey(&bls.PublicKey{}) } // Set network type @@ -279,7 +342,7 @@ func createGlobalConfig() (*nodeconfig.ConfigType, error) { *keyFile) } - selfPeer := p2p.Peer{IP: *ip, Port: *port, ConsensusPubKey: nodeConfig.ConsensusPubKey} + selfPeer := p2p.Peer{IP: *ip, Port: *port, ConsensusPubKey: nodeConfig.ConsensusPubKey.PublicKey[0]} myHost, err = p2pimpl.NewHost(&selfPeer, nodeConfig.P2pPriKey) if err != nil { @@ -317,12 +380,14 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentConsensus.Decider.SetShardIDProvider(func() (uint32, error) { return currentConsensus.ShardID, nil }) - currentConsensus.Decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + currentConsensus.Decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return currentConsensus.PubKey, nil }) - if initialAccount.Address != "" { // staking validator doesn't have to specify ECDSA address - currentConsensus.SelfAddress = common.ParseAddr(initialAccount.Address) + // staking validator doesn't have to specify ECDSA address + currentConsensus.SelfAddresses = map[string]ethCommon.Address{} + for _, initialAccount := range initialAccounts { + currentConsensus.SelfAddresses[initialAccount.BlsPublicKey] = common.ParseAddr(initialAccount.Address) } if err != nil { @@ -540,17 +605,20 @@ func main() { os.Exit(1) } } - fmt.Printf("%s mode; node key %s -> shard %d\n", - map[bool]string{false: "Legacy", true: "Staking"}[*stakingFlag], - nodeconfig.GetDefaultConfig().ConsensusPubKey.SerializeToHexStr(), - initialAccount.ShardID) - + if *nodeType == "validator" { + fmt.Printf("%s mode; node key %s -> shard %d\n", + map[bool]string{false: "Legacy", true: "Staking"}[*stakingFlag], + nodeconfig.GetDefaultConfig().ConsensusPubKey.SerializeToHexStr(), + initialAccounts[0].ShardID) + } if *nodeType != "validator" && *shardID >= 0 { - utils.Logger().Info(). - Uint32("original", initialAccount.ShardID). - Int("override", *shardID). - Msg("ShardID Override") - initialAccount.ShardID = uint32(*shardID) + for _, initialAccount := range initialAccounts { + utils.Logger().Info(). + Uint32("original", initialAccount.ShardID). + Int("override", *shardID). + Msg("ShardID Override") + initialAccount.ShardID = uint32(*shardID) + } } nodeConfig, err := createGlobalConfig() @@ -614,7 +682,7 @@ func main() { } utils.Logger().Info(). - Str("BlsPubKey", hex.EncodeToString(nodeConfig.ConsensusPubKey.Serialize())). + Str("BlsPubKey", nodeConfig.ConsensusPubKey.SerializeToHexStr()). Uint32("ShardID", nodeConfig.ShardID). Str("ShardGroupID", nodeConfig.GetShardGroupID().String()). Str("BeaconGroupID", nodeConfig.GetBeaconGroupID().String()). diff --git a/consensus/consensus.go b/consensus/consensus.go index 8b734d924..9a7b4bb95 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -14,8 +14,10 @@ import ( bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/memprofiling" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/staking/slash" + "github.com/pkg/errors" ) const ( @@ -23,6 +25,8 @@ const ( vdfAndSeedSize = 548 // size of VDF/Proof and Seed ) +var errLeaderPriKeyNotFound = errors.New("getting leader private key from consensus public keys failed") + // Consensus is the main struct with all states and data related to consensus process. type Consensus struct { Decider quorum.Decider @@ -75,9 +79,9 @@ type Consensus struct { MinPeers int pubKeyLock sync.Mutex // private/public keys of current node - priKey *bls.SecretKey - PubKey *bls.PublicKey - SelfAddress common.Address + priKey *multibls.PrivateKey + PubKey *multibls.PublicKey + SelfAddresses map[string]common.Address // the publickey of leader LeaderPubKey *bls.PublicKey viewID uint64 @@ -173,11 +177,26 @@ func (consensus *Consensus) GetBlockReward() *big.Int { return consensus.lastBlockReward } +// GetLeaderPrivateKey returns leader private key if node is the leader +func (consensus *Consensus) GetLeaderPrivateKey(leaderKey *bls.PublicKey) (*bls.SecretKey, error) { + for i, key := range consensus.PubKey.PublicKey { + if key.IsEqual(leaderKey) { + return consensus.priKey.PrivateKey[i], nil + } + } + return nil, errors.Wrapf(errLeaderPriKeyNotFound, leaderKey.SerializeToHexStr()) +} + +// GetConsensusLeaderPrivateKey returns consensus leader private key if node is the leader +func (consensus *Consensus) GetConsensusLeaderPrivateKey() (*bls.SecretKey, error) { + return consensus.GetLeaderPrivateKey(consensus.LeaderPubKey) +} + // TODO: put shardId into chain reader's chain config // New create a new Consensus record func New( - host p2p.Host, shard uint32, leader p2p.Peer, blsPriKey *bls.SecretKey, + host p2p.Host, shard uint32, leader p2p.Peer, multiBlsPriKey *multibls.PrivateKey, Decider quorum.Decider, ) (*Consensus, error) { consensus := Consensus{} @@ -194,9 +213,9 @@ func New( consensus.consensusTimeout = createTimeout() consensus.validators.Store(leader.ConsensusPubKey.SerializeToHexStr(), leader) - if blsPriKey != nil { - consensus.priKey = blsPriKey - consensus.PubKey = blsPriKey.GetPublicKey() + if multiBlsPriKey != nil { + consensus.priKey = multiBlsPriKey + consensus.PubKey = multiBlsPriKey.GetPublicKey() utils.Logger().Info(). Str("publicKey", consensus.PubKey.SerializeToHexStr()).Msg("My Public Key") } else { diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index c8a94ee4e..8d0eb611b 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -20,6 +20,7 @@ import ( "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/profiler" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" @@ -61,13 +62,11 @@ var ( ) // Signs the consensus message and returns the marshaled message. -func (consensus *Consensus) signAndMarshalConsensusMessage( - message *msg_pb.Message, -) ([]byte, error) { - if err := consensus.signConsensusMessage(message); err != nil { +func (consensus *Consensus) signAndMarshalConsensusMessage(message *msg_pb.Message, + priKey *bls.SecretKey) ([]byte, error) { + if err := consensus.signConsensusMessage(message, priKey); err != nil { return empty, err } - marshaledMessage, err := protobuf.Marshal(message) if err != nil { return empty, err @@ -121,14 +120,15 @@ func NewFaker() *Consensus { } // Sign on the hash of the message -func (consensus *Consensus) signMessage(message []byte) []byte { +func (consensus *Consensus) signMessage(message []byte, priKey *bls.SecretKey) []byte { hash := hash.Keccak256(message) - signature := consensus.priKey.SignHash(hash[:]) + signature := priKey.SignHash(hash[:]) return signature.Serialize() } // Sign on the consensus message signature field. -func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message) error { +func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message, + priKey *bls.SecretKey) error { message.Signature = nil // TODO: use custom serialization method rather than protobuf marshaledMessage, err := protobuf.Marshal(message) @@ -136,7 +136,7 @@ func (consensus *Consensus) signConsensusMessage(message *msg_pb.Message) error return err } // 64 byte of signature on previous data - signature := consensus.signMessage(marshaledMessage) + signature := consensus.signMessage(marshaledMessage, priKey) message.Signature = signature return nil } @@ -381,7 +381,7 @@ func (consensus *Consensus) reportMetrics(block types.Block) { txHashes = append(txHashes, hex.EncodeToString(txHash[:])) } metrics := map[string]interface{}{ - "key": hex.EncodeToString(consensus.PubKey.Serialize()), + "key": hex.EncodeToString(consensus.LeaderPubKey.Serialize()), "tps": tps, "txCount": numOfTxs, "nodeCount": consensus.Decider.ParticipantsCount() + 1, @@ -480,7 +480,7 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { decider.SetShardIDProvider(func() (uint32, error) { return consensus.ShardID, nil }) - decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return consensus.PubKey, nil }) consensus.Decider = decider @@ -577,15 +577,15 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { } } - for i := range pubKeys { + for _, key := range pubKeys { // in committee - if pubKeys[i].IsEqual(consensus.PubKey) { + if consensus.PubKey.Contains(key) { if hasError { return Syncing } // If the leader changed and I myself become the leader - if !consensus.LeaderPubKey.IsEqual(oldLeader) && consensus.LeaderPubKey.IsEqual(consensus.PubKey) { + if !consensus.LeaderPubKey.IsEqual(oldLeader) && consensus.IsLeader() { go func() { utils.Logger().Debug(). Str("myKey", consensus.PubKey.SerializeToHexStr()). @@ -605,8 +605,10 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { // IsLeader check if the node is a leader or not by comparing the public key of // the node with the leader public key func (consensus *Consensus) IsLeader() bool { - if consensus.PubKey != nil && consensus.LeaderPubKey != nil { - return consensus.PubKey.IsEqual(consensus.LeaderPubKey) + for _, key := range consensus.PubKey.PublicKey { + if key.IsEqual(consensus.LeaderPubKey) { + return true + } } return false } diff --git a/consensus/consensus_service_test.go b/consensus/consensus_service_test.go index d3d77ddc0..f152569ef 100644 --- a/consensus/consensus_service_test.go +++ b/consensus/consensus_service_test.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -23,7 +24,7 @@ func TestPopulateMessageFields(t *testing.T) { blsPriKey := bls.RandPrivateKey() decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, shard.BeaconChainShardID, leader, blsPriKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -37,9 +38,9 @@ func TestPopulateMessageFields(t *testing.T) { Consensus: &msg_pb.ConsensusRequest{}, }, } - consensusMsg := consensus.populateMessageFields( - msg.GetConsensus(), consensus.blockHash[:], - ) + + consensusMsg := consensus.populateMessageFields(msg.GetConsensus(), consensus.blockHash[:], + blsPriKey.GetPublicKey()) if consensusMsg.ViewId != 2 { t.Errorf("Consensus ID is not populated correctly") @@ -60,8 +61,9 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) { t.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -70,7 +72,7 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) { consensus.blockHash = [32]byte{} msg := &msg_pb.Message{} - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(msg) + marshaledMessage, err := consensus.signAndMarshalConsensusMessage(msg, blsPriKey) if err != nil || len(marshaledMessage) == 0 { t.Errorf("Failed to sign and marshal the message: %s", err) @@ -88,8 +90,9 @@ func TestSetViewID(t *testing.T) { t.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index 87a1ee11f..f642c6c92 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -6,6 +6,7 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -20,7 +21,7 @@ func TestNew(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(bls.RandPrivateKey()), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 765ffb475..d8e5fa32d 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -109,8 +109,13 @@ func (consensus *Consensus) finalizeCommits() { Int64("NumCommits", consensus.Decider.SignersCount(quorum.Commit)). Msg("[Finalizing] Finalizing Block") beforeCatchupNum := consensus.blockNum + leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + if err != nil { + consensus.getLogger().Error().Err(err).Msg("[FinalizeCommits] leader not found") + return + } // Construct committed message - network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil) + network, err := consensus.construct(msg_pb.MessageType_COMMITTED, nil, leaderPriKey.GetPublicKey(), leaderPriKey) if err != nil { consensus.getLogger().Warn().Err(err). Msg("[FinalizeCommits] Unable to construct Committed message") @@ -182,7 +187,7 @@ func (consensus *Consensus) finalizeCommits() { Uint64("epochNum", block.Epoch().Uint64()). Uint64("ViewId", block.Header().ViewID().Uint64()). Str("blockHash", block.Hash().String()). - Int("index", consensus.Decider.IndexOf(consensus.PubKey)). + Int("index", consensus.Decider.IndexOf(consensus.LeaderPubKey)). Int("numTxns", len(block.Transactions())). Int("numStakingTxns", len(block.StakingTransactions())). Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!") @@ -491,10 +496,15 @@ func (consensus *Consensus) Start( } // GenerateVrfAndProof generates new VRF/Proof from hash of previous block -func (consensus *Consensus) GenerateVrfAndProof( - newBlock *types.Block, vrfBlockNumbers []uint64, -) []uint64 { - sk := vrf_bls.NewVRFSigner(consensus.priKey) +func (consensus *Consensus) GenerateVrfAndProof(newBlock *types.Block, vrfBlockNumbers []uint64) []uint64 { + key, err := consensus.GetConsensusLeaderPrivateKey() + if err != nil { + consensus.getLogger().Error(). + Err(err). + Msg("[GenerateVrfAndProof] VRF generation error") + return vrfBlockNumbers + } + sk := vrf_bls.NewVRFSigner(key) blockHash := [32]byte{} previousHeader := consensus.ChainReader.GetHeaderByNumber( newBlock.NumberU64() - 1, diff --git a/consensus/consensus_viewchange_msg.go b/consensus/consensus_viewchange_msg.go index e1d7c7864..2409de44a 100644 --- a/consensus/consensus_viewchange_msg.go +++ b/consensus/consensus_viewchange_msg.go @@ -3,6 +3,7 @@ package consensus import ( "encoding/binary" + "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/api/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" bls_cosi "github.com/harmony-one/harmony/crypto/bls" @@ -10,7 +11,7 @@ import ( ) // construct the view change message -func (consensus *Consensus) constructViewChangeMessage() []byte { +func (consensus *Consensus) constructViewChangeMessage(pubKey *bls.PublicKey, priKey *bls.SecretKey) []byte { message := &msg_pb.Message{ ServiceType: msg_pb.ServiceType_CONSENSUS, Type: msg_pb.MessageType_VIEWCHANGE, @@ -24,7 +25,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { vcMsg.BlockNum = consensus.blockNum vcMsg.ShardId = consensus.ShardID // sender address - vcMsg.SenderPubkey = consensus.PubKey.Serialize() + vcMsg.SenderPubkey = pubKey.Serialize() // next leader key already updated vcMsg.LeaderPubkey = consensus.LeaderPubKey.Serialize() @@ -49,7 +50,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { Str("pubKey", consensus.PubKey.SerializeToHexStr()). Msg("[constructViewChangeMessage]") - sign := consensus.priKey.SignHash(msgToSign) + sign := priKey.SignHash(msgToSign) if sign != nil { vcMsg.ViewchangeSig = sign.Serialize() } else { @@ -58,14 +59,14 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, consensus.current.ViewID()) - sign1 := consensus.priKey.SignHash(viewIDBytes) + sign1 := priKey.SignHash(viewIDBytes) if sign1 != nil { vcMsg.ViewidSig = sign1.Serialize() } else { utils.Logger().Error().Msg("unable to serialize viewID signature") } - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message) + marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey) if err != nil { utils.Logger().Error().Err(err). Msg("[constructViewChangeMessage] failed to sign and marshal the viewchange message") @@ -74,7 +75,7 @@ func (consensus *Consensus) constructViewChangeMessage() []byte { } // new leader construct newview message -func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte { +func (consensus *Consensus) constructNewViewMessage(viewID uint64, pubKey *bls.PublicKey, priKey *bls.SecretKey) []byte { message := &msg_pb.Message{ ServiceType: msg_pb.ServiceType_CONSENSUS, Type: msg_pb.MessageType_NEWVIEW, @@ -88,7 +89,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte { vcMsg.BlockNum = consensus.blockNum vcMsg.ShardId = consensus.ShardID // sender address - vcMsg.SenderPubkey = consensus.PubKey.Serialize() + vcMsg.SenderPubkey = pubKey.Serialize() vcMsg.Payload = consensus.m1Payload sig2arr := consensus.GetNilSigsArray(viewID) @@ -108,7 +109,7 @@ func (consensus *Consensus) constructNewViewMessage(viewID uint64) []byte { vcMsg.M3Bitmap = consensus.viewIDBitmap[viewID].Bitmap } - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message) + marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey) if err != nil { utils.Logger().Error().Err(err). Msg("[constructNewViewMessage] failed to sign and marshal the new view message") diff --git a/consensus/construct.go b/consensus/construct.go index 76d85d804..5565f336d 100644 --- a/consensus/construct.go +++ b/consensus/construct.go @@ -22,7 +22,7 @@ type NetworkMessage struct { // Populates the common basic fields for all consensus message. func (consensus *Consensus) populateMessageFields( - request *msg_pb.ConsensusRequest, blockHash []byte, + request *msg_pb.ConsensusRequest, blockHash []byte, pubKey *bls.PublicKey, ) *msg_pb.ConsensusRequest { request.ViewId = consensus.viewID request.BlockNum = consensus.blockNum @@ -30,13 +30,13 @@ func (consensus *Consensus) populateMessageFields( // 32 byte block hash request.BlockHash = blockHash // sender address - request.SenderPubkey = consensus.PubKey.Serialize() + request.SenderPubkey = pubKey.Serialize() return request } // construct is the single creation point of messages intended for the wire. func (consensus *Consensus) construct( - p msg_pb.MessageType, payloadForSign []byte, + p msg_pb.MessageType, payloadForSign []byte, pubKey *bls.PublicKey, priKey *bls.SecretKey, ) (*NetworkMessage, error) { message := &msg_pb.Message{ ServiceType: msg_pb.ServiceType_CONSENSUS, @@ -51,7 +51,7 @@ func (consensus *Consensus) construct( ) consensusMsg = consensus.populateMessageFields( - message.GetConsensus(), consensus.blockHash[:], + message.GetConsensus(), consensus.blockHash[:], pubKey, ) // Do the signing, 96 byte of bls signature @@ -67,11 +67,11 @@ func (consensus *Consensus) construct( buffer.Write(consensus.prepareBitmap.Bitmap) consensusMsg.Payload = buffer.Bytes() case msg_pb.MessageType_PREPARE: - if s := consensus.priKey.SignHash(consensusMsg.BlockHash); s != nil { + if s := priKey.SignHash(consensusMsg.BlockHash); s != nil { consensusMsg.Payload = s.Serialize() } case msg_pb.MessageType_COMMIT: - if s := consensus.priKey.SignHash(payloadForSign); s != nil { + if s := priKey.SignHash(payloadForSign); s != nil { consensusMsg.Payload = s.Serialize() } case msg_pb.MessageType_COMMITTED: @@ -86,7 +86,7 @@ func (consensus *Consensus) construct( consensusMsg.Payload = consensus.blockHash[:] } - marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message) + marshaledMessage, err := consensus.signAndMarshalConsensusMessage(message, priKey) if err != nil { utils.Logger().Error().Err(err). Str("phase", p.String()). diff --git a/consensus/construct_test.go b/consensus/construct_test.go index 5c9c22fe4..88043bb61 100644 --- a/consensus/construct_test.go +++ b/consensus/construct_test.go @@ -9,6 +9,7 @@ import ( "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -22,14 +23,15 @@ func TestConstructAnnounceMessage(test *testing.T) { test.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { test.Fatalf("Cannot create consensus: %v", err) } consensus.blockHash = [32]byte{} - if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil); err != nil { + if _, err = consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, blsPriKey.GetPublicKey(), blsPriKey); err != nil { test.Fatalf("could not construct announce: %v", err) } } @@ -47,8 +49,9 @@ func TestConstructPreparedMessage(test *testing.T) { test.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) @@ -78,7 +81,7 @@ func TestConstructPreparedMessage(test *testing.T) { test.Log(ctxerror.New("prepareBitmap.SetKey").WithCause(err)) } - network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil) + network, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, blsPriKey.GetPublicKey(), blsPriKey) if err != nil { test.Errorf("Error when creating prepared message") } diff --git a/consensus/fbft_log_test.go b/consensus/fbft_log_test.go index e1a8269ff..5d95e2a3f 100644 --- a/consensus/fbft_log_test.go +++ b/consensus/fbft_log_test.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -21,14 +22,15 @@ func constructAnnounceMessage(t *testing.T) (*NetworkMessage, error) { t.Fatalf("newhost failure: %v", err) } decider := quorum.NewDecider(quorum.SuperMajorityVote) + blsPriKey := bls.RandPrivateKey() consensus, err := New( - host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsPriKey), decider, ) if err != nil { t.Fatalf("Cannot create consensus: %v", err) } consensus.blockHash = [32]byte{} - return consensus.construct(msg_pb.MessageType_ANNOUNCE, nil) + return consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, blsPriKey.GetPublicKey(), blsPriKey) } func getConsensusMessage(payload []byte) (*msg_pb.Message, error) { diff --git a/consensus/leader.go b/consensus/leader.go index 9e73bf995..c536c77db 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -36,7 +36,13 @@ func (consensus *Consensus) announce(block *types.Block) { consensus.block = encodedBlock consensus.blockHeader = encodedBlockHeader - networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil) + + key, err := consensus.GetConsensusLeaderPrivateKey() + if err != nil { + consensus.getLogger().Warn().Err(err).Msg("[Announce] Node not a leader") + return + } + networkMessage, err := consensus.construct(msg_pb.MessageType_ANNOUNCE, nil, key.GetPublicKey(), key) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_ANNOUNCE.String()). @@ -55,21 +61,20 @@ func (consensus *Consensus) announce(block *types.Block) { consensus.FBFTLog.AddBlock(block) // Leader sign the block hash itself - consensus.Decider.SubmitVote( - quorum.Prepare, - consensus.PubKey, - consensus.priKey.SignHash(consensus.blockHash[:]), - common.BytesToHash(consensus.blockHash[:]), - ) - if err := consensus.prepareBitmap.SetKey( - consensus.PubKey, true, - ); err != nil { - consensus.getLogger().Warn().Err(err).Msg( - "[Announce] Leader prepareBitmap SetKey failed", + for i, key := range consensus.PubKey.PublicKey { + consensus.Decider.SubmitVote( + quorum.Prepare, + key, + consensus.priKey.PrivateKey[i].SignHash(consensus.blockHash[:]), + common.BytesToHash(consensus.blockHash[:]), ) - return + if err := consensus.prepareBitmap.SetKey(key, true); err != nil { + consensus.getLogger().Warn().Err(err).Msg( + "[Announce] Leader prepareBitmap SetKey failed", + ) + return + } } - // Construct broadcast p2p message if err := consensus.msgSender.SendWithRetry( consensus.blockNum, msg_pb.MessageType_ANNOUNCE, []nodeconfig.GroupID{ @@ -268,7 +273,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { Offender: *addr, } consensus.SlashChan <- proof - }(consensus.SelfAddress) + }(consensus.SelfAddresses[consensus.LeaderPubKey.SerializeToHexStr()]) return } } diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index 63c0b2c27..b865b3bc2 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -120,10 +120,12 @@ func (v *uniformVoteWeight) AmIMemberOfCommitee() bool { } identity, _ := pubKeyFunc() everyone := v.DumpParticipants() - myVoterID := identity.SerializeToHexStr() - for i := range everyone { - if everyone[i] == myVoterID { - return true + for _, key := range identity.PublicKey { + myVoterID := key.SerializeToHexStr() + for i := range everyone { + if everyone[i] == myVoterID { + return true + } } } return false diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index eae22386f..87480ba15 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -244,10 +244,15 @@ func (v *stakedVoteWeight) AmIMemberOfCommitee() bool { return false } identity, _ := pubKeyFunc() - w := shard.BlsPublicKey{} - w.FromLibBLSPublicKey(identity) - _, ok := v.roster.Voters[w] - return ok + for _, key := range identity.PublicKey { + w := shard.BlsPublicKey{} + w.FromLibBLSPublicKey(key) + _, ok := v.roster.Voters[w] + if ok { + return true + } + } + return false } func newBox() *voteBox { diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 18b611b37..e24f777a5 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/harmony/consensus/votepower" bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" "github.com/pkg/errors" @@ -95,13 +96,13 @@ type SignatureReader interface { // DependencyInjectionWriter .. type DependencyInjectionWriter interface { SetShardIDProvider(func() (uint32, error)) - SetMyPublicKeyProvider(func() (*bls.PublicKey, error)) + SetMyPublicKeyProvider(func() (*multibls.PublicKey, error)) } // DependencyInjectionReader .. type DependencyInjectionReader interface { ShardIDProvider() func() (uint32, error) - MyPublicKey() func() (*bls.PublicKey, error) + MyPublicKey() func() (*multibls.PublicKey, error) } // Decider .. @@ -151,7 +152,7 @@ type cIdentities struct { type depInject struct { shardIDProvider func() (uint32, error) - publicKeyProvider func() (*bls.PublicKey, error) + publicKeyProvider func() (*multibls.PublicKey, error) } func (s *cIdentities) AggregateVotes(p Phase) *bls.Sign { @@ -323,11 +324,11 @@ func (d *depInject) ShardIDProvider() func() (uint32, error) { return d.shardIDProvider } -func (d *depInject) SetMyPublicKeyProvider(p func() (*bls.PublicKey, error)) { +func (d *depInject) SetMyPublicKeyProvider(p func() (*multibls.PublicKey, error)) { d.publicKeyProvider = p } -func (d *depInject) MyPublicKey() func() (*bls.PublicKey, error) { +func (d *depInject) MyPublicKey() func() (*multibls.PublicKey, error) { return d.publicKeyProvider } diff --git a/consensus/threshold.go b/consensus/threshold.go index f44d57d5b..f87695eb4 100644 --- a/consensus/threshold.go +++ b/consensus/threshold.go @@ -14,8 +14,13 @@ import ( func (consensus *Consensus) didReachPrepareQuorum() error { logger := utils.Logger() logger.Debug().Msg("[OnPrepare] Received Enough Prepare Signatures") + leaderPriKey, err := consensus.GetConsensusLeaderPrivateKey() + if err != nil { + utils.Logger().Warn().Err(err).Msg("[OnPrepare] leader not found") + return err + } // Construct and broadcast prepared message - networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARED, nil) + networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARED, nil, consensus.LeaderPubKey, leaderPriKey) if err != nil { consensus.getLogger().Err(err). Str("message-type", msg_pb.MessageType_PREPARED.String()). @@ -33,18 +38,22 @@ func (consensus *Consensus) didReachPrepareQuorum() error { blockNumHash := [8]byte{} binary.LittleEndian.PutUint64(blockNumHash[:], consensus.blockNum) commitPayload := append(blockNumHash[:], consensus.blockHash[:]...) - consensus.Decider.SubmitVote( - quorum.Commit, - consensus.PubKey, - consensus.priKey.SignHash(commitPayload), - common.BytesToHash(consensus.blockHash[:]), - ) - if err := consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil { - consensus.getLogger().Debug().Msg("[OnPrepare] Leader commit bitmap set failed") - return err - } + // so by this point, everyone has committed to the blockhash of this block + // in prepare and so this is the actual block. + for i, key := range consensus.PubKey.PublicKey { + consensus.Decider.SubmitVote( + quorum.Commit, + key, + consensus.priKey.PrivateKey[i].SignHash(commitPayload), + common.BytesToHash(consensus.blockHash[:]), + ) + if err := consensus.commitBitmap.SetKey(key, true); err != nil { + consensus.getLogger().Debug().Msg("[OnPrepare] Leader commit bitmap set failed") + return err + } + } if err := consensus.msgSender.SendWithRetry( consensus.blockNum, msg_pb.MessageType_PREPARED, []nodeconfig.GroupID{ diff --git a/consensus/validator.go b/consensus/validator.go index a0ac0acad..d47107988 100644 --- a/consensus/validator.go +++ b/consensus/validator.go @@ -58,26 +58,27 @@ func (consensus *Consensus) onAnnounce(msg *msg_pb.Message) { } func (consensus *Consensus) prepare() { - networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil) - if err != nil { - consensus.getLogger().Err(err). - Str("message-type", msg_pb.MessageType_PREPARE.String()). - Msg("could not construct message") - return - } + groupID := []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} + for i, key := range consensus.PubKey.PublicKey { + networkMessage, err := consensus.construct(msg_pb.MessageType_PREPARE, nil, key, consensus.priKey.PrivateKey[i]) + if err != nil { + consensus.getLogger().Err(err). + Str("message-type", msg_pb.MessageType_PREPARE.String()). + Msg("could not construct message") + return + } - // TODO: this will not return immediatey, may block - if err := consensus.msgSender.SendWithoutRetry( - []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), - }, - host.ConstructP2pMessage(byte(17), networkMessage.Bytes), - ); err != nil { - consensus.getLogger().Warn().Err(err).Msg("[OnAnnounce] Cannot send prepare message") - } else { - consensus.getLogger().Info(). - Str("blockHash", hex.EncodeToString(consensus.blockHash[:])). - Msg("[OnAnnounce] Sent Prepare Message!!") + // TODO: this will not return immediatey, may block + if err := consensus.msgSender.SendWithoutRetry( + groupID, + host.ConstructP2pMessage(byte(17), networkMessage.Bytes), + ); err != nil { + consensus.getLogger().Warn().Err(err).Msg("[OnAnnounce] Cannot send prepare message") + } else { + consensus.getLogger().Info(). + Str("blockHash", hex.EncodeToString(consensus.blockHash[:])). + Msg("[OnAnnounce] Sent Prepare Message!!") + } } consensus.getLogger().Debug(). Str("From", consensus.phase.String()). @@ -192,29 +193,32 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { } blockNumBytes := make([]byte, 8) binary.LittleEndian.PutUint64(blockNumBytes, consensus.blockNum) - networkMessage, _ := consensus.construct( - msg_pb.MessageType_COMMIT, - append(blockNumBytes, consensus.blockHash[:]...), - ) - // TODO: genesis account node delay for 1 second, - // this is a temp fix for allows FN nodes to earning reward - if consensus.delayCommit > 0 { - time.Sleep(consensus.delayCommit) - } + groupID := []nodeconfig.GroupID{nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} + for i, key := range consensus.PubKey.PublicKey { + networkMessage, _ := consensus.construct( + // TODO: should only sign on block hash + msg_pb.MessageType_COMMIT, + append(blockNumBytes, consensus.blockHash[:]...), + key, consensus.priKey.PrivateKey[i], + ) + // TODO: genesis account node delay for 1 second, + // this is a temp fix for allows FN nodes to earning reward + if consensus.delayCommit > 0 { + time.Sleep(consensus.delayCommit) + } - if err := consensus.msgSender.SendWithoutRetry( - []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, - host.ConstructP2pMessage(byte(17), networkMessage.Bytes), - ); err != nil { - consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!") - } else { - consensus.getLogger().Info(). - Uint64("blockNum", consensus.blockNum). - Hex("blockHash", consensus.blockHash[:]). - Msg("[OnPrepared] Sent Commit Message!!") + if err := consensus.msgSender.SendWithoutRetry( + groupID, + host.ConstructP2pMessage(byte(17), networkMessage.Bytes), + ); err != nil { + consensus.getLogger().Warn().Msg("[OnPrepared] Cannot send commit message!!") + } else { + consensus.getLogger().Info(). + Uint64("blockNum", consensus.blockNum). + Hex("blockHash", consensus.blockHash[:]). + Msg("[OnPrepared] Sent Commit Message!!") + } } - consensus.getLogger().Debug(). Str("From", consensus.phase.String()). Str("To", FBFTCommit.String()). diff --git a/consensus/view_change.go b/consensus/view_change.go index 1faf6793a..5970b3d6f 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -127,12 +127,14 @@ func (consensus *Consensus) startViewChange(viewID uint64) { Str("NextLeader", consensus.LeaderPubKey.SerializeToHexStr()). Msg("[startViewChange]") - msgToSend := consensus.constructViewChangeMessage() - consensus.host.SendMessageToGroups([]nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), - }, - host.ConstructP2pMessage(byte(17), msgToSend), - ) + for i, key := range consensus.PubKey.PublicKey { + msgToSend := consensus.constructViewChangeMessage(key, consensus.priKey.PrivateKey[i]) + consensus.host.SendMessageToGroups([]nodeconfig.GroupID{ + nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID)), + }, + host.ConstructP2pMessage(byte(17), msgToSend), + ) + } consensus.consensusTimeout[timeoutViewChange].SetDuration(duration) consensus.consensusTimeout[timeoutViewChange].Start() @@ -148,7 +150,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { return } newLeaderKey := recvMsg.LeaderPubkey - if !consensus.PubKey.IsEqual(newLeaderKey) { + newLeaderPriKey, err := consensus.GetLeaderPrivateKey(newLeaderKey) + if err != nil { return } @@ -214,13 +217,13 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { preparedMsg := consensus.FBFTLog.FindMessageByMaxViewID(preparedMsgs) if preparedMsg == nil { consensus.getLogger().Debug().Msg("[onViewChange] add my M2(NIL) type messaage") - consensus.nilSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(NIL) - consensus.nilBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true) + consensus.nilSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(NIL) + consensus.nilBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true) } else { consensus.getLogger().Debug().Msg("[onViewChange] add my M1 type messaage") msgToSign := append(preparedMsg.BlockHash[:], preparedMsg.Payload...) - consensus.bhpSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(msgToSign) - consensus.bhpBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true) + consensus.bhpSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(msgToSign) + consensus.bhpBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true) } } // add self m3 type message signature and bitmap @@ -228,8 +231,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { if !ok3 { viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, recvMsg.ViewID) - consensus.viewIDSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = consensus.priKey.SignHash(viewIDBytes) - consensus.viewIDBitmap[recvMsg.ViewID].SetKey(consensus.PubKey, true) + consensus.viewIDSigs[recvMsg.ViewID][consensus.PubKey.SerializeToHexStr()] = newLeaderPriKey.SignHash(viewIDBytes) + consensus.viewIDBitmap[recvMsg.ViewID].SetKey(newLeaderKey, true) } // m2 type message @@ -307,7 +310,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { copy(preparedMsg.BlockHash[:], recvMsg.Payload[:32]) preparedMsg.Payload = make([]byte, len(recvMsg.Payload)-32) copy(preparedMsg.Payload[:], recvMsg.Payload[32:]) - preparedMsg.SenderPubkey = consensus.PubKey + preparedMsg.SenderPubkey = newLeaderKey consensus.getLogger().Info().Msg("[onViewChange] New Leader Prepared Message Added") consensus.FBFTLog.AddMessage(&preparedMsg) } @@ -349,7 +352,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { // received enough view change messages, change state to normal consensus if consensus.Decider.IsQuorumAchievedByMask(consensus.viewIDBitmap[recvMsg.ViewID]) { consensus.current.SetMode(Normal) - consensus.LeaderPubKey = consensus.PubKey + consensus.LeaderPubKey = newLeaderKey consensus.ResetState() if len(consensus.m1Payload) == 0 { // TODO(Chao): explain why ReadySignal is sent only in this case but not the other case. @@ -379,12 +382,12 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { commitPayload := append(blockNumBytes[:], consensus.blockHash[:]...) consensus.Decider.SubmitVote( quorum.Commit, - consensus.PubKey, - consensus.priKey.SignHash(commitPayload), + newLeaderKey, + newLeaderPriKey.SignHash(commitPayload), common.BytesToHash(consensus.blockHash[:]), ) - if err = consensus.commitBitmap.SetKey(consensus.PubKey, true); err != nil { + if err = consensus.commitBitmap.SetKey(newLeaderKey, true); err != nil { consensus.getLogger().Debug(). Msg("[OnViewChange] New Leader commit bitmap set failed") return @@ -392,7 +395,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { } consensus.current.SetViewID(recvMsg.ViewID) - msgToSend := consensus.constructNewViewMessage(recvMsg.ViewID) + msgToSend := consensus.constructNewViewMessage(recvMsg.ViewID, newLeaderKey, newLeaderPriKey) consensus.getLogger().Warn(). Int("payloadSize", len(consensus.m1Payload)). @@ -541,21 +544,25 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { // Construct and send the commit message blockNumHash := make([]byte, 8) binary.LittleEndian.PutUint64(blockNumHash, consensus.blockNum) - network, err := consensus.construct( - msg_pb.MessageType_COMMIT, - append(blockNumHash, consensus.blockHash[:]...), - ) - if err != nil { - consensus.getLogger().Err(err).Msg("could not create commit message") - return + groupID := []nodeconfig.GroupID{ + nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))} + for i, key := range consensus.PubKey.PublicKey { + network, err := consensus.construct( + msg_pb.MessageType_COMMIT, + append(blockNumHash, consensus.blockHash[:]...), + key, consensus.priKey.PrivateKey[i], + ) + if err != nil { + consensus.getLogger().Err(err).Msg("could not create commit message") + return + } + msgToSend := network.Bytes + consensus.getLogger().Info().Msg("onNewView === commit") + consensus.host.SendMessageToGroups( + groupID, + host.ConstructP2pMessage(byte(17), msgToSend), + ) } - msgToSend := network.Bytes - consensus.getLogger().Info().Msg("onNewView === commit") - consensus.host.SendMessageToGroups( - []nodeconfig.GroupID{ - nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(consensus.ShardID))}, - host.ConstructP2pMessage(byte(17), msgToSend), - ) consensus.getLogger().Debug(). Str("From", consensus.phase.String()). Str("To", FBFTCommit.String()). diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 9135333b3..743735eab 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -17,6 +17,7 @@ import ( "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/shard/committee" "github.com/harmony-one/harmony/staking/availability" @@ -212,7 +213,7 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header) d.SetShardIDProvider(func() (uint32, error) { return parentHeader.ShardID(), nil }) - d.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + d.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return nil, nil }) d.SetVoters(slotList.FindCommitteeByID(parentHeader.ShardID()).Slots) @@ -444,7 +445,7 @@ func (e *engineImpl) VerifyHeaderWithSignature(chain engine.ChainReader, header d.SetShardIDProvider(func() (uint32, error) { return header.ShardID(), nil }) - d.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + d.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return nil, nil }) d.SetVoters(slotList.FindCommitteeByID(header.ShardID()).Slots) diff --git a/internal/configs/node/config.go b/internal/configs/node/config.go index 7f4c0829e..7d1edd9ce 100644 --- a/internal/configs/node/config.go +++ b/internal/configs/node/config.go @@ -8,9 +8,9 @@ import ( "math/big" "sync" - "github.com/harmony-one/bls/ffi/go/bls" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/staking/slash" p2p_crypto "github.com/libp2p/go-libp2p-crypto" @@ -83,8 +83,8 @@ type ConfigType struct { PushgatewayPort string // metrics pushgateway prometheus port StringRole string P2pPriKey p2p_crypto.PrivKey - ConsensusPriKey *bls.SecretKey - ConsensusPubKey *bls.PublicKey + ConsensusPriKey *multibls.PrivateKey + ConsensusPubKey *multibls.PublicKey // Database directory DBDir string networkType NetworkType @@ -280,7 +280,8 @@ func SetShardingSchedule(schedule shardingconfig.Schedule) { // consensus key. func (conf *ConfigType) ShardIDFromConsensusKey() (uint32, error) { var pubKey shard.BlsPublicKey - if err := pubKey.FromLibBLSPublicKey(conf.ConsensusPubKey); err != nil { + // all keys belong to same shard + if err := pubKey.FromLibBLSPublicKey(conf.ConsensusPubKey.PublicKey[0]); err != nil { return 0, errors.Wrapf(err, "cannot convert libbls public key %s to internal form", conf.ConsensusPubKey.SerializeToHexStr()) diff --git a/internal/configs/node/config_test.go b/internal/configs/node/config_test.go index 421935e9d..ef47d3178 100644 --- a/internal/configs/node/config_test.go +++ b/internal/configs/node/config_test.go @@ -9,6 +9,7 @@ import ( mock_shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding/mock" "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/multibls" ) func TestNodeConfigSingleton(t *testing.T) { @@ -115,7 +116,7 @@ func TestConfigType_ShardIDFromConsensusKey(t *testing.T) { schedule := mock_shardingconfig.NewMockSchedule(mc) schedule.EXPECT().InstanceForEpoch(tt.epoch).Return(instance) conf := &ConfigType{ - ConsensusPubKey: tt.fields.ConsensusPubKey, + ConsensusPubKey: multibls.GetPublicKey(tt.fields.ConsensusPubKey), networkType: tt.fields.networkType, shardingSchedule: schedule, } diff --git a/multibls/multibls.go b/multibls/multibls.go new file mode 100644 index 000000000..43c266c14 --- /dev/null +++ b/multibls/multibls.go @@ -0,0 +1,73 @@ +package multibls + +import ( + "strings" + + "github.com/harmony-one/bls/ffi/go/bls" +) + +// PrivateKey stores the bls secret keys that belongs to the node +type PrivateKey struct { + PrivateKey []*bls.SecretKey +} + +// PublicKey stores the bls public keys that belongs to the node +type PublicKey struct { + PublicKey []*bls.PublicKey +} + +// SerializeToHexStr wrapper +func (multiKey PublicKey) SerializeToHexStr() string { + var builder strings.Builder + for _, pubKey := range multiKey.PublicKey { + builder.WriteString(pubKey.SerializeToHexStr()) + } + return builder.String() +} + +// Contains wrapper +func (multiKey PublicKey) Contains(pubKey *bls.PublicKey) bool { + for _, key := range multiKey.PublicKey { + if key.IsEqual(pubKey) { + return true + } + } + return false +} + +// GetPublicKey wrapper +func (multiKey PrivateKey) GetPublicKey() *PublicKey { + pubKeys := make([]*bls.PublicKey, len(multiKey.PrivateKey)) + for i, key := range multiKey.PrivateKey { + pubKeys[i] = key.GetPublicKey() + } + return &PublicKey{PublicKey: pubKeys} +} + +// GetPrivateKey creates a multibls PrivateKey using bls.SecretKey +func GetPrivateKey(key *bls.SecretKey) *PrivateKey { + return &PrivateKey{PrivateKey: []*bls.SecretKey{key}} +} + +// GetPublicKey creates a multibls PublicKey using bls.PublicKey +func GetPublicKey(key *bls.PublicKey) *PublicKey { + return &PublicKey{PublicKey: []*bls.PublicKey{key}} +} + +// AppendPubKey appends a PublicKey to multibls PublicKey +func AppendPubKey(multiKey *PublicKey, key *bls.PublicKey) { + if multiKey != nil { + multiKey.PublicKey = append(multiKey.PublicKey, key) + } else { + multiKey = &PublicKey{PublicKey: []*bls.PublicKey{key}} + } +} + +// AppendPriKey appends a SecretKey to multibls PrivateKey +func AppendPriKey(multiKey *PrivateKey, key *bls.SecretKey) { + if multiKey != nil { + multiKey.PrivateKey = append(multiKey.PrivateKey, key) + } else { + multiKey = &PrivateKey{PrivateKey: []*bls.SecretKey{key}} + } +} diff --git a/node/node.go b/node/node.go index a366a8c95..b8c6560c2 100644 --- a/node/node.go +++ b/node/node.go @@ -652,8 +652,8 @@ func (node *Node) InitConsensusWithValidators() (err error) { "blockNum", blockNum) } - for i := range pubKeys { - if pubKeys[i].IsEqual(node.Consensus.PubKey) { + for _, key := range pubKeys { + if node.Consensus.PubKey.Contains(key) { utils.Logger().Info(). Uint64("blockNum", blockNum). Int("numPubKeys", len(pubKeys)). diff --git a/node/node_cross_link.go b/node/node_cross_link.go index b601c19f8..36016ea64 100644 --- a/node/node_cross_link.go +++ b/node/node_cross_link.go @@ -10,6 +10,7 @@ import ( bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/shard" ) @@ -163,7 +164,7 @@ func (node *Node) VerifyCrossLink(cl types.CrossLink) error { decider.SetShardIDProvider(func() (uint32, error) { return cl.ShardID(), nil }) - decider.SetMyPublicKeyProvider(func() (*bls.PublicKey, error) { + decider.SetMyPublicKeyProvider(func() (*multibls.PublicKey, error) { return nil, nil }) if _, err := decider.SetVoters(committee.Slots); err != nil { diff --git a/node/node_handler.go b/node/node_handler.go index 866b3e6f7..0bb6b119c 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -417,7 +417,7 @@ func (node *Node) PostConsensusProcessing( // TODO: randomly selected a few validators to broadcast messages instead of only leader broadcast // TODO: refactor the asynchronous calls to separate go routine. node.lastConsensusTime = time.Now().Unix() - if node.Consensus.PubKey.IsEqual(node.Consensus.LeaderPubKey) { + if node.Consensus.IsLeader() { if node.NodeConfig.ShardID == shard.BeaconChainShardID { node.BroadcastNewBlock(newBlock) } diff --git a/node/node_handler_test.go b/node/node_handler_test.go index 2bf7188b2..ddc34fc15 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -10,6 +10,7 @@ import ( "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -27,7 +28,7 @@ func TestAddNewBlock(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -65,7 +66,7 @@ func TestVerifyNewBlock(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/node/node_metrics.go b/node/node_metrics.go index 3b56e59aa..44f51c5fc 100644 --- a/node/node_metrics.go +++ b/node/node_metrics.go @@ -43,12 +43,14 @@ func (node *Node) UpdateTxPoolSizeForMetrics(txPoolSize uint64) { // UpdateBalanceForMetrics uppdates node balance for metrics service. func (node *Node) UpdateBalanceForMetrics() { - curBalance, err := node.GetBalanceOfAddress(node.Consensus.SelfAddress) - if err != nil { - return + for _, addr := range node.Consensus.SelfAddresses { + curBalance, err := node.GetBalanceOfAddress(addr) + if err != nil { + return + } + utils.Logger().Info().Msgf("Updating metrics node balance %d", curBalance.Uint64()) + metrics.UpdateNodeBalance(curBalance) } - utils.Logger().Info().Msgf("Updating metrics node balance %d", curBalance.Uint64()) - metrics.UpdateNodeBalance(curBalance) } // UpdateLastConsensusTimeForMetrics uppdates last consensus reached time for metrics service. diff --git a/node/node_newblock.go b/node/node_newblock.go index 5084bbd42..fa18672bd 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -84,7 +84,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { // Update worker's current header and state data in preparation to propose/process new transactions var ( - coinbase = node.Consensus.SelfAddress + coinbase = node.Consensus.SelfAddresses[node.Consensus.LeaderPubKey.SerializeToHexStr()] beneficiary = coinbase err error ) @@ -94,7 +94,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { // After staking, all coinbase will be the address of bls pub key if header := node.Worker.GetCurrentHeader(); node.Blockchain().Config().IsStaking(header.Epoch()) { addr := common.Address{} - blsPubKeyBytes := node.Consensus.PubKey.GetAddress() + blsPubKeyBytes := node.Consensus.LeaderPubKey.GetAddress() addr.SetBytes(blsPubKeyBytes[:]) coinbase = addr // coinbase will be the bls address header.SetCoinbase(coinbase) diff --git a/node/node_test.go b/node/node_test.go index 90d59afb9..e31a65fe3 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -16,6 +16,7 @@ import ( "github.com/harmony-one/harmony/drand" "github.com/harmony-one/harmony/internal/shardchain" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/multibls" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" "github.com/harmony-one/harmony/shard" @@ -35,7 +36,7 @@ func TestNewNode(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -202,7 +203,7 @@ func TestAddPeers(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -252,7 +253,7 @@ func TestAddBeaconPeer(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, shard.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, multibls.GetPrivateKey(blsKey), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err)