From 31ebf121df4d0f4f293e8bd967ab13d3f53fe389 Mon Sep 17 00:00:00 2001 From: flicker-harmony Date: Wed, 30 Oct 2019 20:08:16 +0300 Subject: [PATCH 01/10] Increase log max size to avoid log rotation --- cmd/harmony/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 5f890c3f6..70c417b73 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -76,7 +76,7 @@ var ( ip = flag.String("ip", "127.0.0.1", "ip of the node") port = flag.String("port", "9000", "port of the node.") logFolder = flag.String("log_folder", "latest", "the folder collecting the logs of this execution") - logMaxSize = flag.Int("log_max_size", 100, "the max size in megabytes of the log file before it gets rotated") + logMaxSize = flag.Int("log_max_size", 100000, "the max size in megabytes of the log file before it gets rotated") freshDB = flag.Bool("fresh_db", false, "true means the existing disk based db will be removed") profile = flag.Bool("profile", false, "Turn on profiling (CPU, Memory).") metricsReportURL = flag.String("metrics_report_url", "", "If set, reports metrics to this URL.") From a90968d379f010e26487d7ece38620ed3fc47677 Mon Sep 17 00:00:00 2001 From: flicker-harmony Date: Sat, 2 Nov 2019 00:57:25 +0300 Subject: [PATCH 02/10] Replace ethereum log with zerolog and some ctxerrors --- accounts/abi/bind/util.go | 7 +- accounts/keystore/account_cache.go | 19 +- accounts/keystore/file_cache.go | 8 +- api/service/explorer/service.go | 6 +- api/service/explorer/storage.go | 3 +- cmd/client/txgen/main.go | 4 +- cmd/harmony/main.go | 15 +- consensus/consensus_v2.go | 6 +- contracts/contract_caller.go | 6 +- node/Find Results | 448 +++++++++++++++++++++++++++++ node/node.go | 15 +- node/node_handler.go | 23 +- node/node_newblock.go | 13 +- node/node_resharding.go | 19 +- node/node_syncing.go | 5 +- 15 files changed, 543 insertions(+), 54 deletions(-) create mode 100644 node/Find Results diff --git a/accounts/abi/bind/util.go b/accounts/abi/bind/util.go index 37bbabd81..d7cbb945a 100644 --- a/accounts/abi/bind/util.go +++ b/accounts/abi/bind/util.go @@ -32,17 +32,16 @@ import ( func WaitMined(ctx context.Context, b DeployBackend, tx *types.Transaction) (*types.Receipt, error) { queryTicker := time.NewTicker(time.Second) defer queryTicker.Stop() - - logger := utils.GetLogInstance().New("hash", tx.Hash()) + utils.Logger().Info().Str("hash", tx.Hash().Hex()) for { receipt, err := b.TransactionReceipt(ctx, tx.Hash()) if receipt != nil { return receipt, nil } if err != nil { - logger.Trace("Receipt retrieval failed", "err", err) + utils.Logger().Debug().Err(err).Msg("Receipt retrieval failed") } else { - logger.Trace("Transaction not yet mined") + utils.Logger().Debug().Msg("Transaction not yet mined") } // Wait for the next round. select { diff --git a/accounts/keystore/account_cache.go b/accounts/keystore/account_cache.go index d3e8c9f59..019433f09 100644 --- a/accounts/keystore/account_cache.go +++ b/accounts/keystore/account_cache.go @@ -236,7 +236,7 @@ func (ac *accountCache) scanAccounts() error { // Scan the entire folder metadata for file changes creates, deletes, updates, err := ac.fileC.scan(ac.keydir) if err != nil { - utils.GetLogger().Debug("Failed to reload keystore contents", "err", err) + utils.Logger().Debug().Err(err).Msg("Failed to reload keystore contents") return err } if creates.Cardinality() == 0 && deletes.Cardinality() == 0 && updates.Cardinality() == 0 { @@ -252,7 +252,10 @@ func (ac *accountCache) scanAccounts() error { readAccount := func(path string) *accounts.Account { fd, err := os.Open(path) if err != nil { - utils.GetLogger().Trace("Failed to open keystore file", "path", path, "err", err) + utils.Logger().Debug(). + Str("path", path). + Err(err). + Msg("Failed to open keystore file") return nil } defer fd.Close() @@ -263,9 +266,15 @@ func (ac *accountCache) scanAccounts() error { addr := common2.ParseAddr(key.Address) switch { case err != nil: - utils.GetLogger().Debug("Failed to decode keystore key", "path", path, "err", err) + utils.Logger().Debug(). + Str("path", path). + Err(err). + Msg("Failed to decode keystore key") case (addr == common.Address{}): - utils.GetLogger().Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address") + utils.Logger().Debug(). + Str("path", path). + Err(err). + Msg("Failed to decode keystore key, missing or zero address") default: return &accounts.Account{ Address: addr, @@ -298,6 +307,6 @@ func (ac *accountCache) scanAccounts() error { case ac.notify <- struct{}{}: default: } - utils.GetLogger().Trace("Handled keystore changes", "time", end.Sub(start)) + utils.Logger().Debug().Uint64("time", uint64(end.Sub(start))).Msg("Handled keystore changes") return nil } diff --git a/accounts/keystore/file_cache.go b/accounts/keystore/file_cache.go index f0d47f02e..bd22a9f51 100644 --- a/accounts/keystore/file_cache.go +++ b/accounts/keystore/file_cache.go @@ -60,7 +60,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er path := filepath.Join(keyDir, fi.Name()) // Skip any non-key files from the folder if nonKeyFile(fi) { - utils.GetLogger().Trace("Ignoring file on account scan", "path", path) + utils.Logger().Debug().Str("path", path).Msg("Ignoring file on account scan") continue } // Gather the set of all and fresly modified files @@ -85,7 +85,11 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er t3 := time.Now() // Report on the scanning stats and return - utils.GetLogger().Debug("FS scan times", "list", t1.Sub(t0), "set", t2.Sub(t1), "diff", t3.Sub(t2)) + utils.Logger().Debug(). + Uint64("list", uint64(t1.Sub(t0))). + Uint64("set", uint64(t2.Sub(t1))). + Uint64("diff", uint64(t3.Sub(t2))). + Msg("FS scan times") return creates, deletes, updates, nil } diff --git a/api/service/explorer/service.go b/api/service/explorer/service.go index 80960aede..dce59d85b 100644 --- a/api/service/explorer/service.go +++ b/api/service/explorer/service.go @@ -25,7 +25,6 @@ import ( bls2 "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/bech32" common2 "github.com/harmony-one/harmony/internal/common" - "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" @@ -669,7 +668,7 @@ func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) { parsedAddr := common2.ParseAddr(id) oneAddr, err := common2.AddressToBech32(parsedAddr) if err != nil { - utils.Logger().Warn().Msg("unrecognized address format") + utils.Logger().Warn().Err(err).Msg("unrecognized address format") w.WriteHeader(http.StatusBadRequest) return } @@ -688,8 +687,7 @@ func (s *Service) GetExplorerAddress(w http.ResponseWriter, r *http.Request) { data := &Data{} defer func() { if err := json.NewEncoder(w).Encode(data.Address); err != nil { - ctxerror.Warn(utils.WithCallerSkip(utils.GetLogInstance(), 1), err, - "cannot JSON-encode Address") + utils.Logger().Warn().Err(err).Msg("cannot JSON-encode Address") } }() if id == "" { diff --git a/api/service/explorer/storage.go b/api/service/explorer/storage.go index 85b669719..79e12316d 100644 --- a/api/service/explorer/storage.go +++ b/api/service/explorer/storage.go @@ -10,7 +10,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" ) @@ -117,7 +116,7 @@ func (storage *Storage) Dump(block *types.Block, height uint64) { storage.UpdateAddress(batch, explorerTransaction, tx) } if err := batch.Write(); err != nil { - ctxerror.Warn(utils.GetLogger(), err, "cannot write batch") + utils.Logger().Warn().Err(err).Msg("cannot write batch") } } diff --git a/cmd/client/txgen/main.go b/cmd/client/txgen/main.go index 5363d193d..43a5641a4 100644 --- a/cmd/client/txgen/main.go +++ b/cmd/client/txgen/main.go @@ -13,7 +13,6 @@ import ( "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/quorum" "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/shardchain" "github.com/ethereum/go-ethereum/crypto" @@ -222,8 +221,7 @@ syncLoop: } stateMutex.Lock() if err := txGen.Worker.UpdateCurrent(block.Coinbase()); err != nil { - ctxerror.Warn(utils.GetLogger(), err, - "(*Worker).UpdateCurrent failed") + utils.Logger().Warn().Err(err).Msg("(*Worker).UpdateCurrent failed") } stateMutex.Unlock() readySignal <- shardID diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 70c417b73..1427ed616 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -25,7 +25,6 @@ import ( "github.com/harmony-one/harmony/internal/common" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" - "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/genesis" hmykey "github.com/harmony-one/harmony/internal/keystore" "github.com/harmony-one/harmony/internal/memprofiling" @@ -280,7 +279,7 @@ func createGlobalConfig() *nodeconfig.ConfigType { myHost, err = p2pimpl.NewHost(&selfPeer, nodeConfig.P2pPriKey) if *logConn && nodeConfig.GetNetworkType() != nodeconfig.Mainnet { - myHost.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogInstance())) + myHost.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogger())) } if err != nil { panic("unable to new host in harmony") @@ -382,8 +381,10 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { // This needs to be executed after consensus and drand are setup if err := currentNode.CalculateInitShardState(); err != nil { - ctxerror.Crit(utils.GetLogger(), err, "CalculateInitShardState failed", - "shardID", *shardID) + utils.Logger().Warn(). + Int("shardID", *shardID). + Err(err). + Msg("CalculateInitShardState failed") } // Set the consensus ID to be the current block number @@ -477,7 +478,7 @@ func main() { currentNode.SetBeaconSyncFreq(*beaconSyncFreq) if nodeConfig.ShardID != 0 && currentNode.NodeConfig.Role() != nodeconfig.ExplorerNode { - utils.GetLogInstance().Info("SupportBeaconSyncing", "shardID", currentNode.Blockchain().ShardID(), "shardID", nodeConfig.ShardID) + utils.Logger().Info().Uint32("shardID", currentNode.Blockchain().ShardID()).Uint32("shardID", nodeConfig.ShardID).Msg("SupportBeaconSyncing") go currentNode.SupportBeaconSyncing() } @@ -505,7 +506,9 @@ func main() { currentNode.RunServices() // RPC for SDK not supported for mainnet. if err := currentNode.StartRPC(*port); err != nil { - ctxerror.Warn(utils.GetLogger(), err, "StartRPC failed") + utils.Logger().Warn(). + Err(err). + Msg("StartRPC failed") } // Run additional node collectors diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 58c314b3b..10ddcbc0f 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -832,9 +832,6 @@ func (consensus *Consensus) finalizeCommits() { Str("blockHash", block.Hash().String()). Int("index", consensus.Decider.IndexOf(consensus.PubKey)). Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!") - // Print to normal log too - utils.GetLogInstance().Info("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!", "BlockNum", block.NumberU64()) - // Send signal to Node so the new block can be added and new round of consensus can be triggered consensus.ReadySignal <- struct{}{} } @@ -965,6 +962,9 @@ func (consensus *Consensus) LastCommitSig() ([]byte, []byte, error) { if err != nil || len(lastCommits) < 96 { msgs := consensus.FBFTLog.GetMessagesByTypeSeq(msg_pb.MessageType_COMMITTED, consensus.blockNum-1) if len(msgs) != 1 { + utils.Logger().Error(). + Int("numCommittedMsg", len(msgs)). + Msg("GetLastCommitSig failed with wrong number of committed message") return nil, nil, ctxerror.New("GetLastCommitSig failed with wrong number of committed message", "numCommittedMsg", len(msgs)) } lastCommits = msgs[0].Payload diff --git a/contracts/contract_caller.go b/contracts/contract_caller.go index 577a5cd12..975080387 100644 --- a/contracts/contract_caller.go +++ b/contracts/contract_caller.go @@ -35,7 +35,7 @@ func (cc *ContractCaller) CallContract(tx *types.Transaction) ([]byte, error) { currBlock := cc.blockchain.CurrentBlock() msg, err := tx.AsMessage(types.MakeSigner(cc.config, currBlock.Header().Epoch())) if err != nil { - utils.GetLogInstance().Error("[ABI] Failed to convert transaction to message", "error", err) + utils.Logger().Error().Err(err).Msg("[ABI] Failed to convert transaction to message") return []byte{}, err } evmContext := core.NewEVMContext(msg, currBlock.Header(), cc.blockchain, nil) @@ -43,7 +43,7 @@ func (cc *ContractCaller) CallContract(tx *types.Transaction) ([]byte, error) { // about the transaction and calling mechanisms. stateDB, err := cc.blockchain.State() if err != nil { - utils.GetLogInstance().Error("[ABI] Failed to retrieve state db", "error", err) + utils.Logger().Error().Err(err).Msg("[ABI] Failed to retrieve state db") return []byte{}, err } vmenv := vm.NewEVM(evmContext, stateDB, cc.config, vm.Config{}) @@ -51,7 +51,7 @@ func (cc *ContractCaller) CallContract(tx *types.Transaction) ([]byte, error) { returnValue, _, failed, err := core.NewStateTransition(vmenv, msg, gaspool).TransitionDb() if err != nil || failed { - utils.GetLogInstance().Error("[ABI] Failed executing the transaction", "error", err) + utils.Logger().Error().Err(err).Msg("[ABI] Failed executing the transaction") return []byte{}, err } return returnValue, nil diff --git a/node/Find Results b/node/Find Results new file mode 100644 index 000000000..31fc312e7 --- /dev/null +++ b/node/Find Results @@ -0,0 +1,448 @@ +Searching 2456 files for "utils.GetLogInstance(" + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/accounts/abi/bind/util.go: + 34 defer queryTicker.Stop() + 35 + 36: logger := utils.GetLogInstance().New("hash", tx.Hash()) + 37 for { + 38 receipt, err := b.TransactionReceipt(ctx, tx.Hash()) + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/api/service/explorer/service.go: + 689 defer func() { + 690 if err := json.NewEncoder(w).Encode(data.Address); err != nil { + 691: ctxerror.Warn(utils.WithCallerSkip(utils.GetLogInstance(), 1), err, + 692 "cannot JSON-encode Address") + 693 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/cmd/bootnode/main.go: + 68 + 69 if *logConn { + 70: host.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogInstance())) + 71 } + 72 + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/cmd/harmony/main.go: + 155 + 156 if *onlyLogTps { + 157: matchFilterHandler := log.MatchFilterHandler("msg", "TPS Report", utils.GetLogInstance().GetHandler()) + 158: utils.GetLogInstance().SetHandler(matchFilterHandler) + 159 } + 160 + ... + 281 myHost, err = p2pimpl.NewHost(&selfPeer, nodeConfig.P2pPriKey) + 282 if *logConn && nodeConfig.GetNetworkType() != nodeconfig.Mainnet { + 283: myHost.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogInstance())) + 284 } + 285 if err != nil { + ... + 479 if nodeConfig.ShardID != 0 && currentNode.NodeConfig.Role() != nodeconfig.ExplorerNode { + 480 utils.Logger().Info().Uint32("shardID", currentNode.Blockchain().ShardID()).Uint32("shardID", nodeConfig.ShardID).Msg("SupportBeaconSyncing") + 481: //utils.GetLogInstance().Info("SupportBeaconSyncing", "shardID", currentNode.Blockchain().ShardID(), "shardID", nodeConfig.ShardID) + 482 go currentNode.SupportBeaconSyncing() + 483 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/consensus/consensus_v2.go: + 834 Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!") + 835 // Print to normal log too + 836: utils.GetLogInstance().Info("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!", "BlockNum", block.NumberU64()) + 837 + 838 // Send signal to Node so the new block can be added and new round of consensus can be triggered + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/contracts/contract_caller.go: + 36 msg, err := tx.AsMessage(types.MakeSigner(cc.config, currBlock.Header().Epoch())) + 37 if err != nil { + 38: utils.GetLogInstance().Error("[ABI] Failed to convert transaction to message", "error", err) + 39 return []byte{}, err + 40 } + .. + 44 stateDB, err := cc.blockchain.State() + 45 if err != nil { + 46: utils.GetLogInstance().Error("[ABI] Failed to retrieve state db", "error", err) + 47 return []byte{}, err + 48 } + .. + 52 returnValue, _, failed, err := core.NewStateTransition(vmenv, msg, gaspool).TransitionDb() + 53 if err != nil || failed { + 54: utils.GetLogInstance().Error("[ABI] Failed executing the transaction", "error", err) + 55 return []byte{}, err + 56 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/internal/utils/testing.go: + 18 // + 19 // func TestMyFunc(t *testing.T) { + 20: // l := utils.GetLogInstance() + 21 // lrd := NewTestLogRedirector(l, t) + 22 // defer lrd.Close() + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_handler.go: + 337 Msg("BINGO !!! Reached Consensus") + 338 // Print to normal log too + 339: utils.GetLogInstance().Info("BINGO !!! Reached Consensus", "BlockNum", newBlock.NumberU64()) + 340 + 341 // 15% of the validator also need to do broadcasting + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_syncing.go: + 431 } else if len(node.peerRegistrationRecord) >= maxBroadcastNodes { + 432 response.Type = downloader_pb.DownloaderResponse_FAIL + 433: utils.GetLogInstance().Debug("[SYNC] maximum registration limit exceeds", "ip", ip, "port", port) + 434 return response, nil + 435 } else { + +14 matches across 9 files + + +Searching 2433 files for "GetLogger" + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/accounts/keystore/account_cache.go: + 237 creates, deletes, updates, err := ac.fileC.scan(ac.keydir) + 238 if err != nil { + 239: utils.GetLogger().Debug("Failed to reload keystore contents", "err", err) + 240 return err + 241 } + ... + 253 fd, err := os.Open(path) + 254 if err != nil { + 255: utils.GetLogger().Trace("Failed to open keystore file", "path", path, "err", err) + 256 return nil + 257 } + ... + 264 switch { + 265 case err != nil: + 266: utils.GetLogger().Debug("Failed to decode keystore key", "path", path, "err", err) + 267 case (addr == common.Address{}): + 268: utils.GetLogger().Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address") + 269 default: + 270 return &accounts.Account{ + ... + 299 default: + 300 } + 301: utils.GetLogger().Trace("Handled keystore changes", "time", end.Sub(start)) + 302 return nil + 303 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/accounts/keystore/file_cache.go: + 61 // Skip any non-key files from the folder + 62 if nonKeyFile(fi) { + 63: utils.GetLogger().Trace("Ignoring file on account scan", "path", path) + 64 continue + 65 } + .. + 86 + 87 // Report on the scanning stats and return + 88: utils.GetLogger().Debug("FS scan times", "list", t1.Sub(t0), "set", t2.Sub(t1), "diff", t3.Sub(t2)) + 89 return creates, deletes, updates, nil + 90 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/api/service/explorer/service.go: + 689 if err := json.NewEncoder(w).Encode(data.Address); err != nil { + 690 utils.Logger().Warn().Err(err).Msg("cannot JSON-encode Address") + 691: //ctxerror.Warn(utils.WithCallerSkip(utils.GetLogger(), 1), err, "cannot JSON-encode Address") + 692 } + 693 }() + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/api/service/explorer/storage.go: + 118 } + 119 if err := batch.Write(); err != nil { + 120: ctxerror.Warn(utils.GetLogger(), err, "cannot write batch") + 121 } + 122 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/0: + + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/bootnode: + + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/harmony: + + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/staking-standalone: + + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/txgen: + + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/wallet: + + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/wallet_stress_test: + + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/cmd/client/txgen/main.go: + 223 stateMutex.Lock() + 224 if err := txGen.Worker.UpdateCurrent(block.Coinbase()); err != nil { + 225: ctxerror.Warn(utils.GetLogger(), err, + 226 "(*Worker).UpdateCurrent failed") + 227 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/cmd/harmony/main.go: + 281 myHost, err = p2pimpl.NewHost(&selfPeer, nodeConfig.P2pPriKey) + 282 if *logConn && nodeConfig.GetNetworkType() != nodeconfig.Mainnet { + 283: myHost.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogger())) + 284 } + 285 if err != nil { + ... + 383 // This needs to be executed after consensus and drand are setup + 384 if err := currentNode.CalculateInitShardState(); err != nil { + 385: ctxerror.Crit(utils.GetLogger(), err, "CalculateInitShardState failed", + 386 "shardID", *shardID) + 387 } + ... + 507 // RPC for SDK not supported for mainnet. + 508 if err := currentNode.StartRPC(*port); err != nil { + 509: ctxerror.Warn(utils.GetLogger(), err, "StartRPC failed") + 510 } + 511 + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/consensus/consensus_service.go: + 67 // sender address + 68 request.SenderPubkey = consensus.PubKey.Serialize() + 69: consensus.getLogger().Debug(). + 70 Str("senderKey", consensus.PubKey.SerializeToHexStr()). + 71 Msg("[populateMessageFields]") + .. + 205 // ResetState resets the state of the consensus + 206 func (consensus *Consensus) ResetState() { + 207: consensus.getLogger().Debug(). + 208 Str("Phase", consensus.phase.String()). + 209 Msg("[ResetState] Resetting consensus state") + ... + 382 numOfTxs := len(block.Transactions()) + 383 tps := float64(numOfTxs) / timeElapsed.Seconds() + 384: consensus.getLogger().Info(). + 385 Int("numOfTXs", numOfTxs). + 386 Time("startTime", startTime). + ... + 413 } + 414 + 415: // getLogger returns logger for consensus contexts added + 416: func (consensus *Consensus) getLogger() *zerolog.Logger { + 417 logger := utils.Logger().With(). + 418 Uint64("myEpoch", consensus.epoch). + ... + 478 consensus.numPrevPubKeys = len(curPubKeys) + 479 + 480: consensus.getLogger().Info().Msg("[UpdateConsensusInformation] Updating.....") + 481 + 482 if core.IsEpochLastBlockByHeader(header) { + 483 // increase epoch by one if it's the last block + 484 consensus.SetEpochNum(epoch.Uint64() + 1) + 485: consensus.getLogger().Info().Uint64("headerNum", header.Number().Uint64()). + 486 Msg("[UpdateConsensusInformation] Epoch updated for next epoch") + 487 nextEpoch := new(big.Int).Add(epoch, common.Big1) + ... + 493 + 494 if len(pubKeys) == 0 { + 495: consensus.getLogger().Warn(). + 496 Msg("[UpdateConsensusInformation] PublicKeys is Nil") + 497 hasError = true + ... + 499 + 500 // update public keys committee + 501: consensus.getLogger().Info(). + 502 Int("numPubKeys", len(pubKeys)). + 503 Msg("[UpdateConsensusInformation] Successfully updated public keys") + ... + 508 leaderPubKey, err := consensus.getLeaderPubKeyFromCoinbase(header) + 509 if err != nil || leaderPubKey == nil { + 510: consensus.getLogger().Debug().Err(err). + 511 Msg("[SYNC] Unable to get leaderPubKey from coinbase") + 512 consensus.ignoreViewIDCheck = true + 513 hasError = true + 514 } else { + 515: consensus.getLogger().Debug(). + 516 Str("leaderPubKey", leaderPubKey.SerializeToHexStr()). + 517 Msg("[SYNC] Most Recent LeaderPubKey Updated Based on BlockChain") + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/consensus/consensus_v2.go: + 34 err := protobuf.Unmarshal(payload, msg) + 35 if err != nil { + 36: consensus.getLogger().Error().Err(err).Msg("Failed to unmarshal message payload.") + 37 return + 38 } + .. + 1138 vrfBlockNumbers, err := consensus.ChainReader.ReadEpochVrfBlockNums(newBlock.Header().Epoch()) + 1139 if err != nil { + 1140: consensus.getLogger().Info(). + 1141 Uint64("MsgBlockNum", newBlock.NumberU64()). + 1142 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). + .... + 1148 for _, v := range vrfBlockNumbers { + 1149 if v == newBlock.NumberU64() { + 1150: consensus.getLogger().Info(). + 1151 Uint64("MsgBlockNum", newBlock.NumberU64()). + 1152 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). + .... + 1182 vdfObject := vdf_go.New(core.ShardingSchedule.VdfDifficulty(), seed) + 1183 if !vdfObject.Verify(vdfOutput) { + 1184: consensus.getLogger().Warn(). + 1185 Uint64("MsgBlockNum", newBlock.NumberU64()). + 1186 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). + .... + 1190 _, err := consensus.ChainReader.ReadEpochVdfBlockNum(newBlock.Header().Epoch()) + 1191 if err == nil { + 1192: consensus.getLogger().Info(). + 1193 Uint64("MsgBlockNum", newBlock.NumberU64()). + 1194 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). + 1195 Msg("[ConsensusMainLoop] VDF has already been generated previously") + 1196 } else { + 1197: consensus.getLogger().Info(). + 1198 Uint64("MsgBlockNum", newBlock.NumberU64()). + 1199 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). + .... + 1204 } + 1205 } else { + 1206: //consensus.getLogger().Error().Err(err). Msg("[ConsensusMainLoop] Failed to get randomness") + 1207 } + 1208 } + .... + 1248 newBlock.AddVrf(append(vrf[:], proof...)) + 1249 + 1250: consensus.getLogger().Info(). + 1251 Uint64("MsgBlockNum", newBlock.NumberU64()). + 1252 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). + .... + 1271 + 1272 if err != nil { + 1273: consensus.getLogger().Warn(). + 1274 Err(err). + 1275 Str("MsgBlockNum", headerObj.Number().String()). + .... + 1279 + 1280 if !bytes.Equal(hash[:], headerObj.Vrf()[:32]) { + 1281: consensus.getLogger().Warn(). + 1282 Str("MsgBlockNum", headerObj.Number().String()). + 1283 Msg("[OnAnnounce] VRF proof is not valid") + .... + 1288 headerObj.Epoch(), + 1289 ) + 1290: consensus.getLogger().Info(). + 1291 Str("MsgBlockNum", headerObj.Number().String()). + 1292 Int("Number of VRF", len(vrfBlockNumbers)). + .... + 1307 } + 1308 + 1309: consensus.getLogger().Info(). + 1310 Uint64("MsgBlockNum", newBlock.NumberU64()). + 1311 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). + .... + 1320 vdf.Execute() + 1321 duration := time.Now().Sub(start) + 1322: consensus.getLogger().Info(). + 1323 Dur("duration", duration). + 1324 Msg("[ConsensusMainLoop] VDF computation finished") + .... + 1337 vrfBlockNumbers, err := consensus.ChainReader.ReadEpochVrfBlockNums(headerObj.Epoch()) + 1338 if err != nil { + 1339: consensus.getLogger().Error().Err(err). + 1340 Str("MsgBlockNum", headerObj.Number().String()). + 1341 Msg("[OnAnnounce] failed to read VRF block numbers for VDF computation") + .... + 1360 copy(vdfOutput[:], headerObj.Vdf()) + 1361 if vdfObject.Verify(vdfOutput) { + 1362: consensus.getLogger().Info(). + 1363 Str("MsgBlockNum", headerObj.Number().String()). + 1364 Int("Num of VRF", consensus.VdfSeedSize()). + .... + 1366 + 1367 } else { + 1368: consensus.getLogger().Warn(). + 1369 Str("MsgBlockNum", headerObj.Number().String()). + 1370 Uint64("Epoch", headerObj.Epoch().Uint64()). + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/internal/utils/logging.go: + 37 } + 38 + 39: // GetLogger is a shorthand for WithCaller(GetLogInstance()). + 40: func GetLogger() log.Logger { + 41 return WithCallerSkip(GetLogInstance(), 1) + 42 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/internal/utils/logging_test.go: + 70 } + 71 + 72: func TestGetLogger(t *testing.T) { + 73 oldHandler := GetLogInstance().GetHandler() + 74 defer GetLogInstance().SetHandler(oldHandler) + .. + 80 "port", "", // added by the singleton instance + 81 "ip", "", // added by the singleton instance + 82: "funcName", thisPkg + ".TestGetLogger", + 83 "funcFile", thisFile, + 84 "funcLine", 88, // keep this in sync with Debug() call below + .. + 86 }) + 87 GetLogInstance().SetHandler(handler) + 88: GetLogger().Debug("omg") + 89 } + 90 + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node.go: + 246 err = ctxerror.New("cannot get shard chain", "shardID", shardID). + 247 WithCause(err) + 248: ctxerror.Log15(utils.GetLogger().Crit, err) + 249 } + 250 return bc + ... + 256 if err != nil { + 257 err = ctxerror.New("cannot get beaconchain").WithCause(err) + 258: ctxerror.Log15(utils.GetLogger().Crit, err) + 259 } + 260 return bc + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_handler.go: + 162 case proto_node.ShardState: + 163 if err := node.epochShardStateMessageHandler(msgPayload); err != nil { + 164: ctxerror.Log15(utils.GetLogger().Warn, err) + 165 } + 166 } + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_newblock.go: + 85 + 86 if err := node.Worker.CommitTransactions(selectedTxs, selectedStakingTxs, coinbase); err != nil { + 87: ctxerror.Log15(utils.GetLogger().Error, + 88 ctxerror.New("cannot commit transactions"). + 89 WithCause(err)) + .. + 95 if len(receiptsList) != 0 { + 96 if err := node.Worker.CommitReceipts(receiptsList); err != nil { + 97: ctxerror.Log15(utils.GetLogger().Error, + 98 ctxerror.New("cannot commit receipts"). + 99 WithCause(err)) + ... + 116 sig, mask, err := node.Consensus.LastCommitSig() + 117 if err != nil { + 118: ctxerror.Log15(utils.GetLogger().Error, + 119 ctxerror.New("Cannot get commit signatures from last block"). + 120 WithCause(err)) + +/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_resharding.go: + 72 // Don't treat this as a blocker until we fix the nondeterminism. + 73 //return err + 74: ctxerror.Log15(utils.GetLogger().Warn, err) + 75 } + 76 } else { + .. + 194 + 195 if myShardID == math.MaxUint32 { + 196: getLogger().Info("Somehow I got kicked out. Exiting") + 197 os.Exit(8) // 8 represents it's a loop and the program restart itself + 198 } + ... + 206 err := key.Deserialize(nodeID.BlsPublicKey[:]) + 207 if err != nil { + 208: getLogger().Error("Failed to deserialize BLS public key in shard state", + 209 "idx", idx, + 210 "error", err) + ... + 216 + 217 if node.Blockchain().ShardID() == myShardID { + 218: getLogger().Info("staying in the same shard") + 219 } else { + 220: getLogger().Info("moving to another shard") + 221 if err := node.shardChains.Close(); err != nil { + 222: getLogger().Error("cannot close shard chains", "error", err) + 223 } + 224 restartProcess(getRestartArguments(myShardID)) + +165 matches across 21 files diff --git a/node/node.go b/node/node.go index 782089b8e..262b1a5fe 100644 --- a/node/node.go +++ b/node/node.go @@ -243,9 +243,10 @@ func (node *Node) Blockchain() *core.BlockChain { shardID := node.NodeConfig.ShardID bc, err := node.shardChains.ShardChain(shardID) if err != nil { - err = ctxerror.New("cannot get shard chain", "shardID", shardID). - WithCause(err) - ctxerror.Log15(utils.GetLogger().Crit, err) + utils.Logger().Error(). + Uint32("shardID", shardID). + Err(err). + Msg("cannot get shard chain") } return bc } @@ -254,8 +255,7 @@ func (node *Node) Blockchain() *core.BlockChain { func (node *Node) Beaconchain() *core.BlockChain { bc, err := node.shardChains.ShardChain(0) if err != nil { - err = ctxerror.New("cannot get beaconchain").WithCause(err) - ctxerror.Log15(utils.GetLogger().Crit, err) + utils.Logger().Error().Err(err).Msg("cannot get beaconchain") } return bc } @@ -556,6 +556,7 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc // CalculateInitShardState initialize shard state from latest epoch and update committee pub keys for consensus and drand func (node *Node) CalculateInitShardState() (err error) { if node.Consensus == nil { + utils.Logger().Error().Msg("[CalculateInitShardState] consenus is nil; Cannot figure out shardID") return ctxerror.New("[CalculateInitShardState] consenus is nil; Cannot figure out shardID") } shardID := node.Consensus.ShardID @@ -571,6 +572,10 @@ func (node *Node) CalculateInitShardState() (err error) { Msg("[CalculateInitShardState] Try To Get PublicKeys from database") pubKeys := core.CalculatePublicKeys(epoch, shardID) if len(pubKeys) == 0 { + utils.Logger().Error(). + Uint32("shardID", shardID). + Uint64("blockNum", blockNum). + Msg("[CalculateInitShardState] PublicKeys is Empty, Cannot update public keys") return ctxerror.New( "[CalculateInitShardState] PublicKeys is Empty, Cannot update public keys", "shardID", shardID, diff --git a/node/node_handler.go b/node/node_handler.go index c67f533f0..8eed706cb 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -161,7 +161,7 @@ func (node *Node) HandleMessage(content []byte, sender libp2p_peer.ID) { node.pingMessageHandler(msgPayload, sender) case proto_node.ShardState: if err := node.epochShardStateMessageHandler(msgPayload); err != nil { - ctxerror.Log15(utils.GetLogger().Warn, err) + utils.Logger().Warn().Err(err) } } default: @@ -257,15 +257,28 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error { err := node.Blockchain().Validator().ValidateHeader(newBlock, true) if err != nil { + utils.Logger().Error(). + Str("blockHash", newBlock.Hash().Hex()). + Err(err). + Msg("cannot ValidateHeader for the new block") return ctxerror.New("cannot ValidateHeader for the new block", "blockHash", newBlock.Hash()).WithCause(err) } if newBlock.ShardID() != node.Blockchain().ShardID() { + utils.Logger().Error(). + Uint32("my shard ID", node.Blockchain().ShardID()). + Uint32("new block's shard ID", newBlock.ShardID()). + Msg("wrong shard ID") return ctxerror.New("wrong shard ID", "my shard ID", node.Blockchain().ShardID(), "new block's shard ID", newBlock.ShardID()) } err = node.Blockchain().ValidateNewBlock(newBlock) if err != nil { + utils.Logger().Error(). + Str("blockHash", newBlock.Hash().Hex()). + Int("numTx", len(newBlock.Transactions())). + Err(err). + Msg("cannot ValidateNewBlock") return ctxerror.New("cannot ValidateNewBlock", "blockHash", newBlock.Hash(), "numTx", len(newBlock.Transactions()), @@ -285,6 +298,11 @@ func (node *Node) VerifyNewBlock(newBlock *types.Block) error { // TODO: move into ValidateNewBlock err = node.verifyIncomingReceipts(newBlock) if err != nil { + utils.Logger().Error(). + Str("blockHash", newBlock.Hash().Hex()). + Int("numIncomingReceipts", len(newBlock.IncomingReceipts())). + Err(err). + Msg("[VerifyNewBlock] Cannot ValidateNewBlock") return ctxerror.New("[VerifyNewBlock] Cannot ValidateNewBlock", "blockHash", newBlock.Hash(), "numIncomingReceipts", len(newBlock.IncomingReceipts())).WithCause(err) } @@ -335,9 +353,6 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit utils.Logger().Info(). Uint64("BlockNum", newBlock.NumberU64()). Msg("BINGO !!! Reached Consensus") - // Print to normal log too - utils.GetLogInstance().Info("BINGO !!! Reached Consensus", "BlockNum", newBlock.NumberU64()) - // 15% of the validator also need to do broadcasting rand.Seed(time.Now().UTC().UnixNano()) rnd := rand.Intn(100) diff --git a/node/node_newblock.go b/node/node_newblock.go index 711cb7225..dceae14e3 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -8,7 +8,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" ) @@ -84,9 +83,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { selectedTxs, selectedStakingTxs := node.getTransactionsForNewBlock(coinbase) if err := node.Worker.CommitTransactions(selectedTxs, selectedStakingTxs, coinbase); err != nil { - ctxerror.Log15(utils.GetLogger().Error, - ctxerror.New("cannot commit transactions"). - WithCause(err)) + utils.Logger().Error().Err(err).Msg("cannot commit transactions") return nil, err } @@ -94,9 +91,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { receiptsList := node.proposeReceiptsProof() if len(receiptsList) != 0 { if err := node.Worker.CommitReceipts(receiptsList); err != nil { - ctxerror.Log15(utils.GetLogger().Error, - ctxerror.New("cannot commit receipts"). - WithCause(err)) + utils.Logger().Error().Err(err).Msg("cannot commit receipts") } } @@ -115,9 +110,7 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { // Prepare last commit signatures sig, mask, err := node.Consensus.LastCommitSig() if err != nil { - ctxerror.Log15(utils.GetLogger().Error, - ctxerror.New("Cannot get commit signatures from last block"). - WithCause(err)) + utils.Logger().Error().Err(err).Msg("Cannot get commit signatures from last block") return nil, err } diff --git a/node/node_resharding.go b/node/node_resharding.go index fa9bc1e63..d5de36bed 100644 --- a/node/node_resharding.go +++ b/node/node_resharding.go @@ -60,6 +60,7 @@ func (node *Node) validateNewShardState(block *types.Block) error { // DRand may or or may not get in the way. Test this out. expected, err := core.CalculateNewShardState(node.Blockchain(), nextEpoch) if err != nil { + utils.Logger().Error().Err(err).Msg("cannot calculate expected shard state") return ctxerror.New("cannot calculate expected shard state"). WithCause(err) } @@ -70,8 +71,7 @@ func (node *Node) validateNewShardState(block *types.Block) error { // TODO ek/chao – calculated shard state is different even with the // same input, i.e. it is nondeterministic. // Don't treat this as a blocker until we fix the nondeterminism. - //return err - ctxerror.Log15(utils.GetLogger().Warn, err) + utils.Logger().Warn().Err(err).Msg("shard state proposal is different from expected") } } else { // Regular validators fetch the local-shard copy on the beacon chain @@ -90,6 +90,7 @@ func (node *Node) validateNewShardState(block *types.Block) error { // Proposal to discontinue shard if expected != nil { // TODO ek – invoke view change + utils.Logger().Error().Msg("leader proposed to disband against beacon decision") return errors.New( "leader proposed to disband against beacon decision") } @@ -99,6 +100,10 @@ func (node *Node) validateNewShardState(block *types.Block) error { // Sanity check: Shard ID should match if proposed.ShardID != block.ShardID() { // TODO ek – invoke view change + utils.Logger().Error(). + Uint32("proposedShard", proposed.ShardID). + Uint32("blockShard", block.ShardID()). + Msg("proposal has incorrect shard ID") return ctxerror.New("proposal has incorrect shard ID", "proposedShard", proposed.ShardID, "blockShard", block.ShardID()) @@ -106,6 +111,8 @@ func (node *Node) validateNewShardState(block *types.Block) error { // Did beaconchain say we are no more? if expected == nil { // TODO ek – invoke view change + + utils.Logger().Error().Msg("leader proposed to continue against beacon decision") return errors.New( "leader proposed to continue against beacon decision") } @@ -113,10 +120,14 @@ func (node *Node) validateNewShardState(block *types.Block) error { if shard.CompareCommittee(expected, &proposed) != 0 { // TODO ek – log differences // TODO ek – invoke view change + utils.Logger().Error().Msg("proposal differs from one in beacon chain") return errors.New("proposal differs from one in beacon chain") } default: // TODO ek – invoke view change + utils.Logger().Error(). + Int("numShards", len(proposed)). + Msg("regular resharding proposal has incorrect number of shards") return ctxerror.New( "regular resharding proposal has incorrect number of shards", "numShards", len(proposed)) @@ -144,6 +155,7 @@ func (node *Node) broadcastEpochShardState(newBlock *types.Block) error { func (node *Node) epochShardStateMessageHandler(msgPayload []byte) error { epochShardState, err := proto_node.DeserializeEpochShardStateFromMessage(msgPayload) if err != nil { + utils.Logger().Error().Err(err).Msg("Can't get shard state message") return ctxerror.New("Can't get shard state message").WithCause(err) } if node.Consensus == nil { @@ -168,6 +180,9 @@ func (node *Node) epochShardStateMessageHandler(msgPayload []byte) error { err = node.Beaconchain().WriteShardState( receivedEpoch, epochShardState.ShardState) if err != nil { + utils.Logger().Error(). + Uint64("epoch", receivedEpoch.Uint64()). + Err(err).Msg("cannot store shard state") return ctxerror.New("cannot store shard state", "epoch", receivedEpoch). WithCause(err) } diff --git a/node/node_syncing.go b/node/node_syncing.go index a1fdf67c4..b6d7ddf67 100644 --- a/node/node_syncing.go +++ b/node/node_syncing.go @@ -430,7 +430,10 @@ func (node *Node) CalculateResponse(request *downloader_pb.DownloaderRequest, in return response, nil } else if len(node.peerRegistrationRecord) >= maxBroadcastNodes { response.Type = downloader_pb.DownloaderResponse_FAIL - utils.GetLogInstance().Debug("[SYNC] maximum registration limit exceeds", "ip", ip, "port", port) + utils.Logger().Debug(). + Str("ip", ip). + Str("port", port). + Msg("[SYNC] maximum registration limit exceeds") return response, nil } else { response.Type = downloader_pb.DownloaderResponse_FAIL From 25671cfc7627f9a78f908b7bc00251bbe4d31913 Mon Sep 17 00:00:00 2001 From: flicker-harmony Date: Mon, 4 Nov 2019 16:46:28 +0300 Subject: [PATCH 03/10] Revert max log size --- cmd/harmony/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 1427ed616..2e3202bd2 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -75,7 +75,7 @@ var ( ip = flag.String("ip", "127.0.0.1", "ip of the node") port = flag.String("port", "9000", "port of the node.") logFolder = flag.String("log_folder", "latest", "the folder collecting the logs of this execution") - logMaxSize = flag.Int("log_max_size", 100000, "the max size in megabytes of the log file before it gets rotated") + logMaxSize = flag.Int("log_max_size", 100, "the max size in megabytes of the log file before it gets rotated") freshDB = flag.Bool("fresh_db", false, "true means the existing disk based db will be removed") profile = flag.Bool("profile", false, "Turn on profiling (CPU, Memory).") metricsReportURL = flag.String("metrics_report_url", "", "If set, reports metrics to this URL.") From e86f25a5b922ccabb91d44c72fb4ada709eb45a5 Mon Sep 17 00:00:00 2001 From: Rongjian Lan Date: Thu, 7 Nov 2019 16:04:21 -0800 Subject: [PATCH 04/10] Do not propose new block if I am not the leader --- node/node_newblock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/node_newblock.go b/node/node_newblock.go index faeff3426..b47445201 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -42,7 +42,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan struct{}, stopChan ch Msg("Consensus new block proposal: STOPPED!") return case <-readySignal: - for { + for node.Consensus != nil && node.Consensus.IsLeader() { time.Sleep(PeriodicBlock) if time.Now().Before(deadline) { continue From f3f5eb5cc379fe67a7dbb3b637c3dee6528f3c82 Mon Sep 17 00:00:00 2001 From: flicker-harmony <52401354+flicker-harmony@users.noreply.github.com> Date: Sat, 9 Nov 2019 09:44:09 +0300 Subject: [PATCH 05/10] Delete Find Results --- node/Find Results | 448 ---------------------------------------------- 1 file changed, 448 deletions(-) delete mode 100644 node/Find Results diff --git a/node/Find Results b/node/Find Results deleted file mode 100644 index 31fc312e7..000000000 --- a/node/Find Results +++ /dev/null @@ -1,448 +0,0 @@ -Searching 2456 files for "utils.GetLogInstance(" - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/accounts/abi/bind/util.go: - 34 defer queryTicker.Stop() - 35 - 36: logger := utils.GetLogInstance().New("hash", tx.Hash()) - 37 for { - 38 receipt, err := b.TransactionReceipt(ctx, tx.Hash()) - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/api/service/explorer/service.go: - 689 defer func() { - 690 if err := json.NewEncoder(w).Encode(data.Address); err != nil { - 691: ctxerror.Warn(utils.WithCallerSkip(utils.GetLogInstance(), 1), err, - 692 "cannot JSON-encode Address") - 693 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/cmd/bootnode/main.go: - 68 - 69 if *logConn { - 70: host.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogInstance())) - 71 } - 72 - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/cmd/harmony/main.go: - 155 - 156 if *onlyLogTps { - 157: matchFilterHandler := log.MatchFilterHandler("msg", "TPS Report", utils.GetLogInstance().GetHandler()) - 158: utils.GetLogInstance().SetHandler(matchFilterHandler) - 159 } - 160 - ... - 281 myHost, err = p2pimpl.NewHost(&selfPeer, nodeConfig.P2pPriKey) - 282 if *logConn && nodeConfig.GetNetworkType() != nodeconfig.Mainnet { - 283: myHost.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogInstance())) - 284 } - 285 if err != nil { - ... - 479 if nodeConfig.ShardID != 0 && currentNode.NodeConfig.Role() != nodeconfig.ExplorerNode { - 480 utils.Logger().Info().Uint32("shardID", currentNode.Blockchain().ShardID()).Uint32("shardID", nodeConfig.ShardID).Msg("SupportBeaconSyncing") - 481: //utils.GetLogInstance().Info("SupportBeaconSyncing", "shardID", currentNode.Blockchain().ShardID(), "shardID", nodeConfig.ShardID) - 482 go currentNode.SupportBeaconSyncing() - 483 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/consensus/consensus_v2.go: - 834 Msg("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!") - 835 // Print to normal log too - 836: utils.GetLogInstance().Info("HOORAY!!!!!!! CONSENSUS REACHED!!!!!!!", "BlockNum", block.NumberU64()) - 837 - 838 // Send signal to Node so the new block can be added and new round of consensus can be triggered - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/contracts/contract_caller.go: - 36 msg, err := tx.AsMessage(types.MakeSigner(cc.config, currBlock.Header().Epoch())) - 37 if err != nil { - 38: utils.GetLogInstance().Error("[ABI] Failed to convert transaction to message", "error", err) - 39 return []byte{}, err - 40 } - .. - 44 stateDB, err := cc.blockchain.State() - 45 if err != nil { - 46: utils.GetLogInstance().Error("[ABI] Failed to retrieve state db", "error", err) - 47 return []byte{}, err - 48 } - .. - 52 returnValue, _, failed, err := core.NewStateTransition(vmenv, msg, gaspool).TransitionDb() - 53 if err != nil || failed { - 54: utils.GetLogInstance().Error("[ABI] Failed executing the transaction", "error", err) - 55 return []byte{}, err - 56 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/internal/utils/testing.go: - 18 // - 19 // func TestMyFunc(t *testing.T) { - 20: // l := utils.GetLogInstance() - 21 // lrd := NewTestLogRedirector(l, t) - 22 // defer lrd.Close() - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_handler.go: - 337 Msg("BINGO !!! Reached Consensus") - 338 // Print to normal log too - 339: utils.GetLogInstance().Info("BINGO !!! Reached Consensus", "BlockNum", newBlock.NumberU64()) - 340 - 341 // 15% of the validator also need to do broadcasting - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_syncing.go: - 431 } else if len(node.peerRegistrationRecord) >= maxBroadcastNodes { - 432 response.Type = downloader_pb.DownloaderResponse_FAIL - 433: utils.GetLogInstance().Debug("[SYNC] maximum registration limit exceeds", "ip", ip, "port", port) - 434 return response, nil - 435 } else { - -14 matches across 9 files - - -Searching 2433 files for "GetLogger" - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/accounts/keystore/account_cache.go: - 237 creates, deletes, updates, err := ac.fileC.scan(ac.keydir) - 238 if err != nil { - 239: utils.GetLogger().Debug("Failed to reload keystore contents", "err", err) - 240 return err - 241 } - ... - 253 fd, err := os.Open(path) - 254 if err != nil { - 255: utils.GetLogger().Trace("Failed to open keystore file", "path", path, "err", err) - 256 return nil - 257 } - ... - 264 switch { - 265 case err != nil: - 266: utils.GetLogger().Debug("Failed to decode keystore key", "path", path, "err", err) - 267 case (addr == common.Address{}): - 268: utils.GetLogger().Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address") - 269 default: - 270 return &accounts.Account{ - ... - 299 default: - 300 } - 301: utils.GetLogger().Trace("Handled keystore changes", "time", end.Sub(start)) - 302 return nil - 303 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/accounts/keystore/file_cache.go: - 61 // Skip any non-key files from the folder - 62 if nonKeyFile(fi) { - 63: utils.GetLogger().Trace("Ignoring file on account scan", "path", path) - 64 continue - 65 } - .. - 86 - 87 // Report on the scanning stats and return - 88: utils.GetLogger().Debug("FS scan times", "list", t1.Sub(t0), "set", t2.Sub(t1), "diff", t3.Sub(t2)) - 89 return creates, deletes, updates, nil - 90 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/api/service/explorer/service.go: - 689 if err := json.NewEncoder(w).Encode(data.Address); err != nil { - 690 utils.Logger().Warn().Err(err).Msg("cannot JSON-encode Address") - 691: //ctxerror.Warn(utils.WithCallerSkip(utils.GetLogger(), 1), err, "cannot JSON-encode Address") - 692 } - 693 }() - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/api/service/explorer/storage.go: - 118 } - 119 if err := batch.Write(); err != nil { - 120: ctxerror.Warn(utils.GetLogger(), err, "cannot write batch") - 121 } - 122 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/0: - - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/bootnode: - - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/harmony: - - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/staking-standalone: - - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/txgen: - - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/wallet: - - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/bin/wallet_stress_test: - - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/cmd/client/txgen/main.go: - 223 stateMutex.Lock() - 224 if err := txGen.Worker.UpdateCurrent(block.Coinbase()); err != nil { - 225: ctxerror.Warn(utils.GetLogger(), err, - 226 "(*Worker).UpdateCurrent failed") - 227 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/cmd/harmony/main.go: - 281 myHost, err = p2pimpl.NewHost(&selfPeer, nodeConfig.P2pPriKey) - 282 if *logConn && nodeConfig.GetNetworkType() != nodeconfig.Mainnet { - 283: myHost.GetP2PHost().Network().Notify(utils.NewConnLogger(utils.GetLogger())) - 284 } - 285 if err != nil { - ... - 383 // This needs to be executed after consensus and drand are setup - 384 if err := currentNode.CalculateInitShardState(); err != nil { - 385: ctxerror.Crit(utils.GetLogger(), err, "CalculateInitShardState failed", - 386 "shardID", *shardID) - 387 } - ... - 507 // RPC for SDK not supported for mainnet. - 508 if err := currentNode.StartRPC(*port); err != nil { - 509: ctxerror.Warn(utils.GetLogger(), err, "StartRPC failed") - 510 } - 511 - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/consensus/consensus_service.go: - 67 // sender address - 68 request.SenderPubkey = consensus.PubKey.Serialize() - 69: consensus.getLogger().Debug(). - 70 Str("senderKey", consensus.PubKey.SerializeToHexStr()). - 71 Msg("[populateMessageFields]") - .. - 205 // ResetState resets the state of the consensus - 206 func (consensus *Consensus) ResetState() { - 207: consensus.getLogger().Debug(). - 208 Str("Phase", consensus.phase.String()). - 209 Msg("[ResetState] Resetting consensus state") - ... - 382 numOfTxs := len(block.Transactions()) - 383 tps := float64(numOfTxs) / timeElapsed.Seconds() - 384: consensus.getLogger().Info(). - 385 Int("numOfTXs", numOfTxs). - 386 Time("startTime", startTime). - ... - 413 } - 414 - 415: // getLogger returns logger for consensus contexts added - 416: func (consensus *Consensus) getLogger() *zerolog.Logger { - 417 logger := utils.Logger().With(). - 418 Uint64("myEpoch", consensus.epoch). - ... - 478 consensus.numPrevPubKeys = len(curPubKeys) - 479 - 480: consensus.getLogger().Info().Msg("[UpdateConsensusInformation] Updating.....") - 481 - 482 if core.IsEpochLastBlockByHeader(header) { - 483 // increase epoch by one if it's the last block - 484 consensus.SetEpochNum(epoch.Uint64() + 1) - 485: consensus.getLogger().Info().Uint64("headerNum", header.Number().Uint64()). - 486 Msg("[UpdateConsensusInformation] Epoch updated for next epoch") - 487 nextEpoch := new(big.Int).Add(epoch, common.Big1) - ... - 493 - 494 if len(pubKeys) == 0 { - 495: consensus.getLogger().Warn(). - 496 Msg("[UpdateConsensusInformation] PublicKeys is Nil") - 497 hasError = true - ... - 499 - 500 // update public keys committee - 501: consensus.getLogger().Info(). - 502 Int("numPubKeys", len(pubKeys)). - 503 Msg("[UpdateConsensusInformation] Successfully updated public keys") - ... - 508 leaderPubKey, err := consensus.getLeaderPubKeyFromCoinbase(header) - 509 if err != nil || leaderPubKey == nil { - 510: consensus.getLogger().Debug().Err(err). - 511 Msg("[SYNC] Unable to get leaderPubKey from coinbase") - 512 consensus.ignoreViewIDCheck = true - 513 hasError = true - 514 } else { - 515: consensus.getLogger().Debug(). - 516 Str("leaderPubKey", leaderPubKey.SerializeToHexStr()). - 517 Msg("[SYNC] Most Recent LeaderPubKey Updated Based on BlockChain") - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/consensus/consensus_v2.go: - 34 err := protobuf.Unmarshal(payload, msg) - 35 if err != nil { - 36: consensus.getLogger().Error().Err(err).Msg("Failed to unmarshal message payload.") - 37 return - 38 } - .. - 1138 vrfBlockNumbers, err := consensus.ChainReader.ReadEpochVrfBlockNums(newBlock.Header().Epoch()) - 1139 if err != nil { - 1140: consensus.getLogger().Info(). - 1141 Uint64("MsgBlockNum", newBlock.NumberU64()). - 1142 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). - .... - 1148 for _, v := range vrfBlockNumbers { - 1149 if v == newBlock.NumberU64() { - 1150: consensus.getLogger().Info(). - 1151 Uint64("MsgBlockNum", newBlock.NumberU64()). - 1152 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). - .... - 1182 vdfObject := vdf_go.New(core.ShardingSchedule.VdfDifficulty(), seed) - 1183 if !vdfObject.Verify(vdfOutput) { - 1184: consensus.getLogger().Warn(). - 1185 Uint64("MsgBlockNum", newBlock.NumberU64()). - 1186 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). - .... - 1190 _, err := consensus.ChainReader.ReadEpochVdfBlockNum(newBlock.Header().Epoch()) - 1191 if err == nil { - 1192: consensus.getLogger().Info(). - 1193 Uint64("MsgBlockNum", newBlock.NumberU64()). - 1194 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). - 1195 Msg("[ConsensusMainLoop] VDF has already been generated previously") - 1196 } else { - 1197: consensus.getLogger().Info(). - 1198 Uint64("MsgBlockNum", newBlock.NumberU64()). - 1199 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). - .... - 1204 } - 1205 } else { - 1206: //consensus.getLogger().Error().Err(err). Msg("[ConsensusMainLoop] Failed to get randomness") - 1207 } - 1208 } - .... - 1248 newBlock.AddVrf(append(vrf[:], proof...)) - 1249 - 1250: consensus.getLogger().Info(). - 1251 Uint64("MsgBlockNum", newBlock.NumberU64()). - 1252 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). - .... - 1271 - 1272 if err != nil { - 1273: consensus.getLogger().Warn(). - 1274 Err(err). - 1275 Str("MsgBlockNum", headerObj.Number().String()). - .... - 1279 - 1280 if !bytes.Equal(hash[:], headerObj.Vrf()[:32]) { - 1281: consensus.getLogger().Warn(). - 1282 Str("MsgBlockNum", headerObj.Number().String()). - 1283 Msg("[OnAnnounce] VRF proof is not valid") - .... - 1288 headerObj.Epoch(), - 1289 ) - 1290: consensus.getLogger().Info(). - 1291 Str("MsgBlockNum", headerObj.Number().String()). - 1292 Int("Number of VRF", len(vrfBlockNumbers)). - .... - 1307 } - 1308 - 1309: consensus.getLogger().Info(). - 1310 Uint64("MsgBlockNum", newBlock.NumberU64()). - 1311 Uint64("Epoch", newBlock.Header().Epoch().Uint64()). - .... - 1320 vdf.Execute() - 1321 duration := time.Now().Sub(start) - 1322: consensus.getLogger().Info(). - 1323 Dur("duration", duration). - 1324 Msg("[ConsensusMainLoop] VDF computation finished") - .... - 1337 vrfBlockNumbers, err := consensus.ChainReader.ReadEpochVrfBlockNums(headerObj.Epoch()) - 1338 if err != nil { - 1339: consensus.getLogger().Error().Err(err). - 1340 Str("MsgBlockNum", headerObj.Number().String()). - 1341 Msg("[OnAnnounce] failed to read VRF block numbers for VDF computation") - .... - 1360 copy(vdfOutput[:], headerObj.Vdf()) - 1361 if vdfObject.Verify(vdfOutput) { - 1362: consensus.getLogger().Info(). - 1363 Str("MsgBlockNum", headerObj.Number().String()). - 1364 Int("Num of VRF", consensus.VdfSeedSize()). - .... - 1366 - 1367 } else { - 1368: consensus.getLogger().Warn(). - 1369 Str("MsgBlockNum", headerObj.Number().String()). - 1370 Uint64("Epoch", headerObj.Epoch().Uint64()). - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/internal/utils/logging.go: - 37 } - 38 - 39: // GetLogger is a shorthand for WithCaller(GetLogInstance()). - 40: func GetLogger() log.Logger { - 41 return WithCallerSkip(GetLogInstance(), 1) - 42 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/internal/utils/logging_test.go: - 70 } - 71 - 72: func TestGetLogger(t *testing.T) { - 73 oldHandler := GetLogInstance().GetHandler() - 74 defer GetLogInstance().SetHandler(oldHandler) - .. - 80 "port", "", // added by the singleton instance - 81 "ip", "", // added by the singleton instance - 82: "funcName", thisPkg + ".TestGetLogger", - 83 "funcFile", thisFile, - 84 "funcLine", 88, // keep this in sync with Debug() call below - .. - 86 }) - 87 GetLogInstance().SetHandler(handler) - 88: GetLogger().Debug("omg") - 89 } - 90 - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node.go: - 246 err = ctxerror.New("cannot get shard chain", "shardID", shardID). - 247 WithCause(err) - 248: ctxerror.Log15(utils.GetLogger().Crit, err) - 249 } - 250 return bc - ... - 256 if err != nil { - 257 err = ctxerror.New("cannot get beaconchain").WithCause(err) - 258: ctxerror.Log15(utils.GetLogger().Crit, err) - 259 } - 260 return bc - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_handler.go: - 162 case proto_node.ShardState: - 163 if err := node.epochShardStateMessageHandler(msgPayload); err != nil { - 164: ctxerror.Log15(utils.GetLogger().Warn, err) - 165 } - 166 } - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_newblock.go: - 85 - 86 if err := node.Worker.CommitTransactions(selectedTxs, selectedStakingTxs, coinbase); err != nil { - 87: ctxerror.Log15(utils.GetLogger().Error, - 88 ctxerror.New("cannot commit transactions"). - 89 WithCause(err)) - .. - 95 if len(receiptsList) != 0 { - 96 if err := node.Worker.CommitReceipts(receiptsList); err != nil { - 97: ctxerror.Log15(utils.GetLogger().Error, - 98 ctxerror.New("cannot commit receipts"). - 99 WithCause(err)) - ... - 116 sig, mask, err := node.Consensus.LastCommitSig() - 117 if err != nil { - 118: ctxerror.Log15(utils.GetLogger().Error, - 119 ctxerror.New("Cannot get commit signatures from last block"). - 120 WithCause(err)) - -/Users/vladlazarus/go/src/github.com/harmony-one/harmony/node/node_resharding.go: - 72 // Don't treat this as a blocker until we fix the nondeterminism. - 73 //return err - 74: ctxerror.Log15(utils.GetLogger().Warn, err) - 75 } - 76 } else { - .. - 194 - 195 if myShardID == math.MaxUint32 { - 196: getLogger().Info("Somehow I got kicked out. Exiting") - 197 os.Exit(8) // 8 represents it's a loop and the program restart itself - 198 } - ... - 206 err := key.Deserialize(nodeID.BlsPublicKey[:]) - 207 if err != nil { - 208: getLogger().Error("Failed to deserialize BLS public key in shard state", - 209 "idx", idx, - 210 "error", err) - ... - 216 - 217 if node.Blockchain().ShardID() == myShardID { - 218: getLogger().Info("staying in the same shard") - 219 } else { - 220: getLogger().Info("moving to another shard") - 221 if err := node.shardChains.Close(); err != nil { - 222: getLogger().Error("cannot close shard chains", "error", err) - 223 } - 224 restartProcess(getRestartArguments(myShardID)) - -165 matches across 21 files From 4e628224e66724d68f36639527b1de02c52e9fe2 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sat, 9 Nov 2019 15:05:19 -0800 Subject: [PATCH 06/10] [rpc] Fix mistake how nodemetadata gets ShardID, fix older networkType mistake (#1817) --- hmy/api_backend.go | 2 +- internal/configs/node/config.go | 1 + internal/hmyapi/harmony.go | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hmy/api_backend.go b/hmy/api_backend.go index 4e583bc73..f22857630 100644 --- a/hmy/api_backend.go +++ b/hmy/api_backend.go @@ -236,7 +236,7 @@ func (b *APIBackend) RPCGasCap() *big.Int { return b.hmy.RPCGasCap // TODO(ricl): should be hmy.config.RPCGasCap } -// GetShardID returns the gas cap of rpc +// GetShardID returns shardID of this node func (b *APIBackend) GetShardID() uint32 { return b.hmy.shardID } diff --git a/internal/configs/node/config.go b/internal/configs/node/config.go index 1744011fc..d633a1790 100644 --- a/internal/configs/node/config.go +++ b/internal/configs/node/config.go @@ -224,6 +224,7 @@ func (conf *ConfigType) Role() Role { // SetNetworkType set the networkType func SetNetworkType(networkType NetworkType) { + defaultConfig.networkType = networkType for i := range shardConfigs { shardConfigs[i].networkType = networkType } diff --git a/internal/hmyapi/harmony.go b/internal/hmyapi/harmony.go index 5dc544a23..51f26cbb3 100644 --- a/internal/hmyapi/harmony.go +++ b/internal/hmyapi/harmony.go @@ -53,7 +53,7 @@ type NodeMetadata struct { ShardID uint32 `json:"shard-id"` } -// GetNodeMetadata produces a NodeMetadata record. Note the data is from the answering RPC +// GetNodeMetadata produces a NodeMetadata record, data is from the answering RPC node func (s *PublicHarmonyAPI) GetNodeMetadata() NodeMetadata { cfg := nodeconfig.GetDefaultConfig() return NodeMetadata{ @@ -62,6 +62,6 @@ func (s *PublicHarmonyAPI) GetNodeMetadata() NodeMetadata { string(cfg.GetNetworkType()), s.b.ChainConfig().ChainID.String(), s.b.IsLeader(), - cfg.GetShardID(), + s.b.GetShardID(), } } From e6a4fbea4f60201df528597e3f04b534a1d98b45 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sun, 10 Nov 2019 20:57:29 -0800 Subject: [PATCH 07/10] [committee] Factor out committee membership, provide entry for alternative committee membership (#1818) * [committee] Move core.ShardingSchedule to shard.Schedule * [consensus] Remove redundant PublicKeys field of Consensus as Decider maintains that * [committee] Use committee package to pick PublicKeys * [committee] Use committee inplace of CalculateShardState * [committee] Remove core/resharding.go, complete usage of committee as implementation replacement * [committee] Address PR comments --- api/service/resharding/service.go | 97 -------- cmd/client/txgen/main.go | 17 +- cmd/harmony/main.go | 47 ++-- consensus/consensus.go | 3 - consensus/consensus_leader_msg_test.go | 6 +- consensus/consensus_service.go | 39 ++-- consensus/consensus_service_test.go | 8 +- consensus/consensus_test.go | 4 +- consensus/consensus_v2.go | 8 +- consensus/consensus_validator_msg_test.go | 6 +- consensus/fbft_log_test.go | 4 +- core/blockchain.go | 39 ++-- core/core_test.go | 3 +- core/genesis.go | 5 + core/resharding.go | 259 --------------------- core/resharding.md | 12 - core/resharding_test.go | 149 ------------ core/state_processor.go | 3 +- core/values/blockchain.go | 10 - drand/drand_leader.go | 6 +- internal/chain/engine.go | 7 +- internal/configs/sharding/localnet.go | 13 +- internal/hmyapi/blockchain.go | 5 +- internal/params/config.go | 2 +- node/node.go | 41 ++-- node/node_cross_shard.go | 10 +- node/node_explorer.go | 3 +- node/node_genesis.go | 7 +- node/node_handler.go | 70 +----- node/node_handler_test.go | 6 +- node/node_newblock.go | 21 +- node/node_resharding.go | 15 +- node/node_test.go | 8 +- node/worker/worker.go | 12 +- shard/committee/assignment.go | 261 ++++++++++++++++++++++ shard/shard_state.go | 76 +++++-- shard/shard_state_test.go | 36 +-- shard/values.go | 19 ++ 38 files changed, 534 insertions(+), 803 deletions(-) delete mode 100644 api/service/resharding/service.go delete mode 100644 core/resharding.go delete mode 100644 core/resharding.md delete mode 100644 core/resharding_test.go delete mode 100644 core/values/blockchain.go create mode 100644 shard/committee/assignment.go create mode 100644 shard/values.go diff --git a/api/service/resharding/service.go b/api/service/resharding/service.go deleted file mode 100644 index 18558ab39..000000000 --- a/api/service/resharding/service.go +++ /dev/null @@ -1,97 +0,0 @@ -package resharding - -import ( - "time" - - "github.com/ethereum/go-ethereum/rpc" - msg_pb "github.com/harmony-one/harmony/api/proto/message" - "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/internal/utils" -) - -// Constants for resharding service. -const ( - ReshardingCheckTime = time.Second -) - -// Service is the role conversion service. -type Service struct { - stopChan chan struct{} - stoppedChan chan struct{} - messageChan chan *msg_pb.Message - beaconChain *core.BlockChain -} - -// New returns role conversion service. -func New(beaconChain *core.BlockChain) *Service { - return &Service{beaconChain: beaconChain} -} - -// StartService starts role conversion service. -func (s *Service) StartService() { - s.stopChan = make(chan struct{}) - s.stoppedChan = make(chan struct{}) - - s.Init() - s.Run(s.stopChan, s.stoppedChan) -} - -// Init initializes role conversion service. -func (s *Service) Init() { -} - -// Run runs role conversion. -func (s *Service) Run(stopChan chan struct{}, stoppedChan chan struct{}) { - go func() { - defer close(stoppedChan) - for { - select { - default: - utils.Logger().Info().Msg("Running role conversion") - // TODO: Write some logic here. - s.DoService() - case <-stopChan: - return - } - } - }() -} - -// DoService does role conversion. -func (s *Service) DoService() { - tick := time.NewTicker(ReshardingCheckTime) - // Get current shard state hash. - currentShardStateHash := s.beaconChain.CurrentBlock().Header().ShardStateHash() - for { - select { - case <-tick.C: - LatestShardStateHash := s.beaconChain.CurrentBlock().Header().ShardStateHash() - if currentShardStateHash != LatestShardStateHash { - // TODO(minhdoan): Add resharding logic later after modifying the resharding func as it current doesn't calculate the role (leader/validator) - } - } - } -} - -// StopService stops role conversion service. -func (s *Service) StopService() { - utils.Logger().Info().Msg("Stopping role conversion service") - s.stopChan <- struct{}{} - <-s.stoppedChan - utils.Logger().Info().Msg("Role conversion stopped") -} - -// NotifyService notify service -func (s *Service) NotifyService(params map[string]interface{}) { - return -} - -// SetMessageChan sets up message channel to service. -func (s *Service) SetMessageChan(messageChan chan *msg_pb.Message) { - s.messageChan = messageChan -} - -// APIs for the services. -func (s *Service) APIs() []rpc.API { - return nil -} diff --git a/cmd/client/txgen/main.go b/cmd/client/txgen/main.go index 5363d193d..fe8b9b4c6 100644 --- a/cmd/client/txgen/main.go +++ b/cmd/client/txgen/main.go @@ -10,29 +10,28 @@ import ( "sync" "time" - "github.com/harmony-one/harmony/consensus" - "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core" - "github.com/harmony-one/harmony/internal/ctxerror" - "github.com/harmony-one/harmony/internal/shardchain" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" bls2 "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/internal/params" - "github.com/harmony-one/harmony/api/client" proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/common/denominations" + "github.com/harmony-one/harmony/consensus" + "github.com/harmony-one/harmony/consensus/quorum" + "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" + "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/genesis" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/internal/shardchain" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/node" "github.com/harmony-one/harmony/p2p" p2p_host "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) var ( @@ -105,7 +104,7 @@ func setUpTXGen() *node.Node { txGen := node.New(myhost, consensusObj, chainDBFactory, false) //Changed it : no longer archival node. txGen.Client = client.NewClient(txGen.GetHost(), uint32(shardID)) consensusObj.ChainReader = txGen.Blockchain() - genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) + genesisShardingConfig := shard.Schedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) startIdx := 0 endIdx := startIdx + genesisShardingConfig.NumNodesPerShard() pubs := []*bls2.PublicKey{} diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index 5f890c3f6..efe440d8e 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -16,7 +16,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/api/service/syncing" "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/quorum" @@ -34,6 +33,7 @@ import ( "github.com/harmony-one/harmony/node" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) // Version string variables @@ -71,7 +71,6 @@ func printVersion() { os.Exit(0) } -// Flags var ( ip = flag.String("ip", "127.0.0.1", "ip of the node") port = flag.String("port", "9000", "port of the node.") @@ -102,7 +101,6 @@ var ( syncFreq = flag.Int("sync_freq", 60, "unit in seconds") // beaconSyncFreq indicates beaconchain sync frequency beaconSyncFreq = flag.Int("beacon_sync_freq", 60, "unit in seconds") - // blockPeriod indicates the how long the leader waits to propose a new block. blockPeriod = flag.Int("block_period", 8, "how long in second the leader waits to propose a new block.") leaderOverride = flag.Bool("leader_override", false, "true means override the default leader role and acts as validator") @@ -113,34 +111,25 @@ var ( blsKeyFile = flag.String("blskey_file", "", "The encrypted file of bls serialized private key by passphrase.") blsPass = flag.String("blspass", "", "The file containing passphrase to decrypt the encrypted bls file.") blsPassphrase string - // Sharding configuration parameters for devnet devnetNumShards = flag.Uint("dn_num_shards", 2, "number of shards for -network_type=devnet (default: 2)") devnetShardSize = flag.Int("dn_shard_size", 10, "number of nodes per shard for -network_type=devnet (default 10)") devnetHarmonySize = flag.Int("dn_hmy_size", -1, "number of Harmony-operated nodes per shard for -network_type=devnet; negative (default) means equal to -dn_shard_size") - // logConn logs incoming/outgoing connections - logConn = flag.Bool("log_conn", false, "log incoming/outgoing connections") - - keystoreDir = flag.String("keystore", hmykey.DefaultKeyStoreDir, "The default keystore directory") - + logConn = flag.Bool("log_conn", false, "log incoming/outgoing connections") + keystoreDir = flag.String("keystore", hmykey.DefaultKeyStoreDir, "The default keystore directory") initialAccount = &genesis.DeployAccount{} - // logging verbosity verbosity = flag.Int("verbosity", 5, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail (default: 5)") - // dbDir is the database directory. dbDir = flag.String("db_dir", "", "blockchain database directory") - // Disable view change. disableViewChange = flag.Bool("disable_view_change", false, "Do not propose view change (testing only)") - // metrics flag to collct meetrics or not, pushgateway ip and port for metrics metricsFlag = flag.Bool("metrics", false, "Collect and upload node metrics") pushgatewayIP = flag.String("pushgateway_ip", "grafana.harmony.one", "Metrics view ip") pushgatewayPort = flag.String("pushgateway_port", "9091", "Metrics view port") - - publicRPC = flag.Bool("public_rpc", false, "Enable Public RPC Access (default: false)") + publicRPC = flag.Bool("public_rpc", false, "Enable Public RPC Access (default: false)") ) func initSetup() { @@ -203,13 +192,13 @@ func passphraseForBls() { } func setupInitialAccount() (isLeader bool) { - genesisShardingConfig := core.ShardingSchedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) + genesisShardingConfig := shard.Schedule.InstanceForEpoch(big.NewInt(core.GenesisEpoch)) pubKey := setupConsensusKey(nodeconfig.GetDefaultConfig()) reshardingEpoch := genesisShardingConfig.ReshardingEpoch() if reshardingEpoch != nil && len(reshardingEpoch) > 0 { for _, epoch := range reshardingEpoch { - config := core.ShardingSchedule.InstanceForEpoch(epoch) + config := shard.Schedule.InstanceForEpoch(epoch) isLeader, initialAccount = config.FindAccount(pubKey.SerializeToHexStr()) if initialAccount != nil { break @@ -323,7 +312,7 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { switch { case *networkType == nodeconfig.Localnet: - epochConfig := core.ShardingSchedule.InstanceForEpoch(ethCommon.Big0) + epochConfig := shard.Schedule.InstanceForEpoch(ethCommon.Big0) selfPort, err := strconv.ParseUint(*port, 10, 16) if err != nil { utils.Logger().Fatal(). @@ -359,9 +348,9 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(nodeconfig.ShardID(*shardID))) case "validator": currentNode.NodeConfig.SetRole(nodeconfig.Validator) - if nodeConfig.ShardID == 0 { - currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(0)) - currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(0)) + if nodeConfig.ShardID == shard.BeaconChainShardID { + currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(shard.BeaconChainShardID)) + currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(shard.BeaconChainShardID)) } else { currentNode.NodeConfig.SetShardGroupID(nodeconfig.NewGroupIDByShardID(nodeconfig.ShardID(nodeConfig.ShardID))) currentNode.NodeConfig.SetClientGroupID(nodeconfig.NewClientGroupIDByShardID(nodeconfig.ShardID(nodeConfig.ShardID))) @@ -381,8 +370,8 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { // currentNode.DRand = dRand // This needs to be executed after consensus and drand are setup - if err := currentNode.CalculateInitShardState(); err != nil { - ctxerror.Crit(utils.GetLogger(), err, "CalculateInitShardState failed", + if err := currentNode.InitConsensusWithValidators(); err != nil { + ctxerror.Crit(utils.GetLogger(), err, "InitConsensusWithMembers failed", "shardID", *shardID) } @@ -429,13 +418,13 @@ func main() { switch *networkType { case nodeconfig.Mainnet: - core.ShardingSchedule = shardingconfig.MainnetSchedule + shard.Schedule = shardingconfig.MainnetSchedule case nodeconfig.Testnet: - core.ShardingSchedule = shardingconfig.TestnetSchedule + shard.Schedule = shardingconfig.TestnetSchedule case nodeconfig.Pangaea: - core.ShardingSchedule = shardingconfig.PangaeaSchedule + shard.Schedule = shardingconfig.PangaeaSchedule case nodeconfig.Localnet: - core.ShardingSchedule = shardingconfig.LocalnetSchedule + shard.Schedule = shardingconfig.LocalnetSchedule case nodeconfig.Devnet: if *devnetHarmonySize < 0 { *devnetHarmonySize = *devnetShardSize @@ -448,7 +437,7 @@ func main() { err) os.Exit(1) } - core.ShardingSchedule = shardingconfig.NewFixedSchedule(devnetConfig) + shard.Schedule = shardingconfig.NewFixedSchedule(devnetConfig) } initSetup() @@ -476,7 +465,7 @@ func main() { currentNode.SetSyncFreq(*syncFreq) currentNode.SetBeaconSyncFreq(*beaconSyncFreq) - if nodeConfig.ShardID != 0 && currentNode.NodeConfig.Role() != nodeconfig.ExplorerNode { + if nodeConfig.ShardID != shard.BeaconChainShardID && currentNode.NodeConfig.Role() != nodeconfig.ExplorerNode { utils.GetLogInstance().Info("SupportBeaconSyncing", "shardID", currentNode.Blockchain().ShardID(), "shardID", nodeConfig.ShardID) go currentNode.SupportBeaconSyncing() } diff --git a/consensus/consensus.go b/consensus/consensus.go index 54e43cf32..fc59d5a2c 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -79,8 +79,6 @@ type Consensus struct { // If the number of validators is less than minPeers, the consensus won't start MinPeers int - CommitteePublicKeys map[string]bool - pubKeyLock sync.Mutex // private/public keys of current node @@ -216,7 +214,6 @@ func New( consensus.current = State{mode: Normal} // FBFT timeout consensus.consensusTimeout = createTimeout() - consensus.CommitteePublicKeys = make(map[string]bool) consensus.validators.Store(leader.ConsensusPubKey.SerializeToHexStr(), leader) if blsPriKey != nil { diff --git a/consensus/consensus_leader_msg_test.go b/consensus/consensus_leader_msg_test.go index 22f5e81ae..9abff5be1 100644 --- a/consensus/consensus_leader_msg_test.go +++ b/consensus/consensus_leader_msg_test.go @@ -7,12 +7,12 @@ import ( "github.com/harmony-one/harmony/api/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestConstructAnnounceMessage(test *testing.T) { @@ -24,7 +24,7 @@ func TestConstructAnnounceMessage(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot create consensus: %v", err) @@ -57,7 +57,7 @@ func TestConstructPreparedMessage(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_service.go b/consensus/consensus_service.go index b8d9ea667..2f5bba52a 100644 --- a/consensus/consensus_service.go +++ b/consensus/consensus_service.go @@ -14,7 +14,6 @@ import ( "github.com/harmony-one/harmony/block" consensus_engine "github.com/harmony-one/harmony/consensus/engine" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" bls_cosi "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/hash" @@ -23,6 +22,8 @@ import ( "github.com/harmony-one/harmony/internal/profiler" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" + "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" libp2p_peer "github.com/libp2p/go-libp2p-peer" "github.com/rs/zerolog" ) @@ -110,17 +111,14 @@ func (consensus *Consensus) DebugPrintPublicKeys() { utils.Logger().Debug().Strs("PublicKeys", keys).Int("count", len(keys)).Msgf("Debug Public Keys") } -// UpdatePublicKeys updates the PublicKeys variable, protected by a mutex +// UpdatePublicKeys updates the PublicKeys for quorum on current subcommittee, protected by a mutex func (consensus *Consensus) UpdatePublicKeys(pubKeys []*bls.PublicKey) int64 { consensus.pubKeyLock.Lock() consensus.Decider.UpdateParticipants(pubKeys) - consensus.CommitteePublicKeys = map[string]bool{} utils.Logger().Info().Msg("My Committee updated") - for i, pubKey := range consensus.Decider.DumpParticipants() { - utils.Logger().Info().Int("index", i).Str("BlsPubKey", pubKey).Msg("Member") - consensus.CommitteePublicKeys[pubKey] = true + for i := range pubKeys { + utils.Logger().Info().Int("index", i).Str("BLSPubKey", pubKeys[i].SerializeToHexStr()).Msg("Member") } - consensus.LeaderPubKey = pubKeys[0] utils.Logger().Info(). Str("info", consensus.LeaderPubKey.SerializeToHexStr()).Msg("My Leader") @@ -230,8 +228,7 @@ func (consensus *Consensus) ToggleConsensusCheck() { // IsValidatorInCommittee returns whether the given validator BLS address is part of my committee func (consensus *Consensus) IsValidatorInCommittee(pubKey *bls.PublicKey) bool { - _, ok := consensus.CommitteePublicKeys[pubKey.SerializeToHexStr()] - return ok + return consensus.Decider.IndexOf(pubKey) != -1 } // Verify the signature of the message are valid from the signer's public key. @@ -458,22 +455,21 @@ func (consensus *Consensus) getLeaderPubKeyFromCoinbase(header *block.Header) (* func (consensus *Consensus) UpdateConsensusInformation() Mode { pubKeys := []*bls.PublicKey{} hasError := false - header := consensus.ChainReader.CurrentHeader() - epoch := header.Epoch() - curPubKeys := core.CalculatePublicKeys(epoch, header.ShardID()) + _, curPubKeys := committee.WithStakingEnabled.ComputePublicKeys( + epoch, consensus.ChainReader, int(header.ShardID()), + ) consensus.numPrevPubKeys = len(curPubKeys) - consensus.getLogger().Info().Msg("[UpdateConsensusInformation] Updating.....") - - if core.IsEpochLastBlockByHeader(header) { + if shard.Schedule.IsLastBlock(header.Number().Uint64()) { // increase epoch by one if it's the last block consensus.SetEpochNum(epoch.Uint64() + 1) consensus.getLogger().Info().Uint64("headerNum", header.Number().Uint64()). Msg("[UpdateConsensusInformation] Epoch updated for next epoch") - nextEpoch := new(big.Int).Add(epoch, common.Big1) - pubKeys = core.CalculatePublicKeys(nextEpoch, header.ShardID()) + _, pubKeys = committee.WithStakingEnabled.ComputePublicKeys( + new(big.Int).Add(epoch, common.Big1), consensus.ChainReader, int(header.ShardID()), + ) } else { consensus.SetEpochNum(epoch.Uint64()) pubKeys = curPubKeys @@ -493,7 +489,8 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { consensus.UpdatePublicKeys(pubKeys) // take care of possible leader change during the epoch - if !core.IsEpochLastBlockByHeader(header) && header.Number().Uint64() != 0 { + if !shard.Schedule.IsLastBlock(header.Number().Uint64()) && + header.Number().Uint64() != 0 { leaderPubKey, err := consensus.getLeaderPubKeyFromCoinbase(header) if err != nil || leaderPubKey == nil { consensus.getLogger().Debug().Err(err). @@ -508,9 +505,9 @@ func (consensus *Consensus) UpdateConsensusInformation() Mode { } } - for _, key := range pubKeys { + for i := range pubKeys { // in committee - if key.IsEqual(consensus.PubKey) { + if pubKeys[i].IsEqual(consensus.PubKey) { if hasError { return Syncing } @@ -544,7 +541,7 @@ func (consensus *Consensus) IsLeader() bool { // NeedsRandomNumberGeneration returns true if the current epoch needs random number generation func (consensus *Consensus) NeedsRandomNumberGeneration(epoch *big.Int) bool { - if consensus.ShardID == 0 && epoch.Uint64() >= core.ShardingSchedule.RandomnessStartingEpoch() { + if consensus.ShardID == 0 && epoch.Uint64() >= shard.Schedule.RandomnessStartingEpoch() { return true } diff --git a/consensus/consensus_service_test.go b/consensus/consensus_service_test.go index 36673a1f6..ae5042f5d 100644 --- a/consensus/consensus_service_test.go +++ b/consensus/consensus_service_test.go @@ -6,11 +6,11 @@ import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestPopulateMessageFields(t *testing.T) { @@ -23,7 +23,7 @@ func TestPopulateMessageFields(t *testing.T) { blsPriKey := bls.RandPrivateKey() decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, blsPriKey, decider, + host, shard.BeaconChainShardID, leader, blsPriKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -60,7 +60,7 @@ func TestSignAndMarshalConsensusMessage(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -88,7 +88,7 @@ func TestSetViewID(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index a67002d6c..87a1ee11f 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -4,11 +4,11 @@ import ( "testing" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestNew(test *testing.T) { @@ -20,7 +20,7 @@ func TestNew(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 8da4c5459..cc917476c 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -14,7 +14,6 @@ import ( msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" vrf_bls "github.com/harmony-one/harmony/crypto/vrf/bls" "github.com/harmony-one/harmony/internal/chain" @@ -22,6 +21,7 @@ import ( "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/shard" "github.com/harmony-one/vdf/src/vdf_go" ) @@ -1187,7 +1187,7 @@ func (consensus *Consensus) Start(blockChannel chan *types.Block, stopChan chan if err == nil { vdfInProgress = false // Verify the randomness - vdfObject := vdf_go.New(core.ShardingSchedule.VdfDifficulty(), seed) + vdfObject := vdf_go.New(shard.Schedule.VdfDifficulty(), seed) if !vdfObject.Verify(vdfOutput) { consensus.getLogger().Warn(). Uint64("MsgBlockNum", newBlock.NumberU64()). @@ -1323,7 +1323,7 @@ func (consensus *Consensus) GenerateVdfAndProof(newBlock *types.Block, vrfBlockN // TODO ek – limit concurrency go func() { - vdf := vdf_go.New(core.ShardingSchedule.VdfDifficulty(), seed) + vdf := vdf_go.New(shard.Schedule.VdfDifficulty(), seed) outputChannel := vdf.GetOutputChannel() start := time.Now() vdf.Execute() @@ -1364,7 +1364,7 @@ func (consensus *Consensus) ValidateVdfAndProof(headerObj *block.Header) bool { } } - vdfObject := vdf_go.New(core.ShardingSchedule.VdfDifficulty(), seed) + vdfObject := vdf_go.New(shard.Schedule.VdfDifficulty(), seed) vdfOutput := [516]byte{} copy(vdfOutput[:], headerObj.Vdf()) if vdfObject.Verify(vdfOutput) { diff --git a/consensus/consensus_validator_msg_test.go b/consensus/consensus_validator_msg_test.go index 0bb6658b1..4a8305777 100644 --- a/consensus/consensus_validator_msg_test.go +++ b/consensus/consensus_validator_msg_test.go @@ -7,11 +7,11 @@ import ( "github.com/harmony-one/harmony/api/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestConstructPrepareMessage(test *testing.T) { @@ -23,7 +23,7 @@ func TestConstructPrepareMessage(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) @@ -54,7 +54,7 @@ func TestConstructCommitMessage(test *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { test.Fatalf("Cannot craeate consensus: %v", err) diff --git a/consensus/fbft_log_test.go b/consensus/fbft_log_test.go index bcf390ea9..a3f545ab1 100644 --- a/consensus/fbft_log_test.go +++ b/consensus/fbft_log_test.go @@ -7,11 +7,11 @@ import ( "github.com/harmony-one/harmony/api/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func constructAnnounceMessage(t *testing.T) []byte { @@ -23,7 +23,7 @@ func constructAnnounceMessage(t *testing.T) []byte { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := New( - host, values.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, + host, shard.BeaconChainShardID, leader, bls.RandPrivateKey(), decider, ) if err != nil { t.Fatalf("Cannot create consensus: %v", err) diff --git a/core/blockchain.go b/core/blockchain.go index 7d4af6818..33fb29f92 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -45,6 +45,7 @@ import ( "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" lru "github.com/hashicorp/golang-lru" ) @@ -246,27 +247,16 @@ func IsEpochBlock(block *types.Block) bool { // genesis block is the first epoch block return true } - return ShardingSchedule.IsLastBlock(block.NumberU64() - 1) + return shard.Schedule.IsLastBlock(block.NumberU64() - 1) } // EpochFirstBlock returns the block number of the first block of an epoch. // TODO: instead of using fixed epoch schedules, determine the first block by epoch changes. func EpochFirstBlock(epoch *big.Int) *big.Int { - if epoch.Cmp(big.NewInt(0)) == 0 { - return big.NewInt(0) + if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 { + return big.NewInt(GenesisEpoch) } - return big.NewInt(int64(ShardingSchedule.EpochLastBlock(epoch.Uint64()-1) + 1)) -} - -// IsEpochLastBlock returns whether this block is the last block of an epoch. -func IsEpochLastBlock(block *types.Block) bool { - return ShardingSchedule.IsLastBlock(block.NumberU64()) -} - -// IsEpochLastBlockByHeader returns whether this block is the last block of an epoch -// given block header -func IsEpochLastBlockByHeader(header *block.Header) bool { - return ShardingSchedule.IsLastBlock(header.Number().Uint64()) + return big.NewInt(int64(shard.Schedule.EpochLastBlock(epoch.Uint64()-1) + 1)) } func (bc *BlockChain) getProcInterrupt() bool { @@ -1083,7 +1073,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. epoch := block.Header().Epoch() if bc.chainConfig.IsCrossTx(block.Epoch()) { - shardingConfig := ShardingSchedule.InstanceForEpoch(epoch) + shardingConfig := shard.Schedule.InstanceForEpoch(epoch) shardNum := int(shardingConfig.NumShards()) for i := 0; i < shardNum; i++ { if i == int(block.ShardID()) { @@ -1943,7 +1933,18 @@ func (bc *BlockChain) GetShardState(epoch *big.Int) (shard.State, error) { if err == nil { // TODO ek – distinguish ErrNotFound return shardState, err } - shardState, err = CalculateNewShardState(bc, epoch) + + if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 { + shardState, err = committee.WithStakingEnabled.ReadFromComputation( + big.NewInt(GenesisEpoch), *bc.Config(), nil, + ) + } else { + prevEpoch := new(big.Int).Sub(epoch, common.Big1) + shardState, err = committee.WithStakingEnabled.ReadFromChain( + prevEpoch, bc, + ) + } + if err != nil { return nil, err } @@ -2164,7 +2165,7 @@ func (bc *BlockChain) CXMerkleProof(shardID uint32, block *types.Block) (*types. } epoch := block.Header().Epoch() - shardingConfig := ShardingSchedule.InstanceForEpoch(epoch) + shardingConfig := shard.Schedule.InstanceForEpoch(epoch) shardNum := int(shardingConfig.NumShards()) for i := 0; i < shardNum; i++ { @@ -2384,7 +2385,7 @@ func (bc *BlockChain) CurrentValidatorAddresses() []common.Address { if err != nil { continue } - epoch := ShardingSchedule.CalcEpochNumber(val.CreationHeight.Uint64()) + epoch := shard.Schedule.CalcEpochNumber(val.CreationHeight.Uint64()) if epoch.Cmp(currentEpoch) >= 0 { // wait for next epoch continue diff --git a/core/core_test.go b/core/core_test.go index 36ddde25b..4ea9ee2c8 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -7,6 +7,7 @@ import ( blockfactory "github.com/harmony-one/harmony/block/factory" "github.com/harmony-one/harmony/core/types" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" + "github.com/harmony-one/harmony/shard" ) func TestIsEpochBlock(t *testing.T) { @@ -58,7 +59,7 @@ func TestIsEpochBlock(t *testing.T) { }, } for i, test := range tests { - ShardingSchedule = test.schedule + shard.Schedule = test.schedule r := IsEpochBlock(test.block) if r != test.expected { t.Errorf("index: %v, expected: %v, got: %v\n", i, test.expected, r) diff --git a/core/genesis.go b/core/genesis.go index 9e2d3ecc0..cd3d4aa21 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -46,6 +46,11 @@ import ( var errGenesisNoConfig = errors.New("genesis has no chain configuration") +const ( + // GenesisEpoch is the number of the genesis epoch. + GenesisEpoch = 0 +) + // Genesis specifies the header fields, state of a genesis block. It also defines hard // fork switch-over blocks through the chain configuration. type Genesis struct { diff --git a/core/resharding.go b/core/resharding.go deleted file mode 100644 index 2686549f7..000000000 --- a/core/resharding.go +++ /dev/null @@ -1,259 +0,0 @@ -package core - -import ( - "encoding/hex" - "errors" - "math/big" - "math/rand" - "sort" - - "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/bls/ffi/go/bls" - common2 "github.com/harmony-one/harmony/internal/common" - shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" - "github.com/harmony-one/harmony/internal/ctxerror" - "github.com/harmony-one/harmony/internal/utils" - "github.com/harmony-one/harmony/shard" -) - -const ( - // GenesisEpoch is the number of the genesis epoch. - GenesisEpoch = 0 - // CuckooRate is the percentage of nodes getting reshuffled in the second step of cuckoo resharding. - CuckooRate = 0.1 -) - -// ShardingState is data structure hold the sharding state -type ShardingState struct { - epoch uint64 // current epoch - rnd uint64 // random seed for resharding - numShards int // TODO ek – equal to len(shardState); remove this - shardState shard.State -} - -// sortedCommitteeBySize will sort shards by size -// Suppose there are N shards, the first N/2 larger shards are called active committees -// the rest N/2 smaller committees are called inactive committees -// actually they are all just normal shards -// TODO: sort the committee weighted by total staking instead of shard size -func (ss *ShardingState) sortCommitteeBySize() { - sort.Slice(ss.shardState, func(i, j int) bool { - return len(ss.shardState[i].NodeList) > len(ss.shardState[j].NodeList) - }) -} - -// assignNewNodes add new nodes into the N/2 active committees evenly -func (ss *ShardingState) assignNewNodes(newNodeList []shard.NodeID) { - ss.sortCommitteeBySize() - numActiveShards := ss.numShards / 2 - Shuffle(newNodeList) - for i, nid := range newNodeList { - id := 0 - if numActiveShards > 0 { - id = i % numActiveShards - } - if id < len(ss.shardState) { - ss.shardState[id].NodeList = append(ss.shardState[id].NodeList, nid) - } else { - utils.Logger().Error().Int("id", id).Int("shardState Count", len(ss.shardState)).Msg("assignNewNodes index out of range") - } - } -} - -// cuckooResharding uses cuckoo rule to reshard X% of active committee(shards) into inactive committee(shards) -func (ss *ShardingState) cuckooResharding(percent float64) { - numActiveShards := ss.numShards / 2 - kickedNodes := []shard.NodeID{} - for i := range ss.shardState { - if i >= numActiveShards { - break - } - numKicked := int(percent * float64(len(ss.shardState[i].NodeList))) - if numKicked == 0 { - numKicked++ // At least kick one node out - } - length := len(ss.shardState[i].NodeList) - if length-numKicked <= 0 { - continue // Never empty a shard - } - tmp := ss.shardState[i].NodeList[length-numKicked:] - kickedNodes = append(kickedNodes, tmp...) - ss.shardState[i].NodeList = ss.shardState[i].NodeList[:length-numKicked] - } - - Shuffle(kickedNodes) - numInactiveShards := ss.numShards - numActiveShards - for i, nid := range kickedNodes { - id := numActiveShards - if numInactiveShards > 0 { - id += i % numInactiveShards - } - ss.shardState[id].NodeList = append(ss.shardState[id].NodeList, nid) - } -} - -// Reshard will first add new nodes into shards, then use cuckoo rule to reshard to get new shard state -func (ss *ShardingState) Reshard(newNodeList []shard.NodeID, percent float64) { - rand.Seed(int64(ss.rnd)) - ss.sortCommitteeBySize() - - // Take out and preserve leaders - leaders := []shard.NodeID{} - for i := 0; i < ss.numShards; i++ { - if len(ss.shardState[i].NodeList) > 0 { - leaders = append(leaders, ss.shardState[i].NodeList[0]) - ss.shardState[i].NodeList = ss.shardState[i].NodeList[1:] - // Also shuffle the rest of the nodes - Shuffle(ss.shardState[i].NodeList) - } - } - - ss.assignNewNodes(newNodeList) - ss.cuckooResharding(percent) - - // Put leader back - if len(leaders) < ss.numShards { - utils.Logger().Error().Msg("Not enough leaders to assign to shards") - } - for i := 0; i < ss.numShards; i++ { - ss.shardState[i].NodeList = append([]shard.NodeID{leaders[i]}, ss.shardState[i].NodeList...) - } -} - -// Shuffle will shuffle the list with result uniquely determined by seed, assuming there is no repeat items in the list -func Shuffle(list []shard.NodeID) { - // Sort to make sure everyone will generate the same with the same rand seed. - sort.Slice(list, func(i, j int) bool { - return shard.CompareNodeIDByBLSKey(list[i], list[j]) == -1 - }) - rand.Shuffle(len(list), func(i, j int) { - list[i], list[j] = list[j], list[i] - }) -} - -// GetEpochFromBlockNumber calculates the epoch number the block belongs to -func GetEpochFromBlockNumber(blockNumber uint64) uint64 { - return ShardingSchedule.CalcEpochNumber(blockNumber).Uint64() -} - -// GetShardingStateFromBlockChain will retrieve random seed and shard map from beacon chain for given a epoch -func GetShardingStateFromBlockChain(bc *BlockChain, epoch *big.Int) (*ShardingState, error) { - if bc == nil { - return nil, errors.New("no blockchain is supplied to get shard state") - } - shardState, err := bc.ReadShardState(epoch) - if err != nil { - return nil, err - } - shardState = shardState.DeepCopy() - - // TODO(RJ,HB): use real randomness for resharding - //blockNumber := GetBlockNumberFromEpoch(epoch.Uint64()) - //rndSeedBytes := bc.GetVdfByNumber(blockNumber) - rndSeed := uint64(0) - - return &ShardingState{epoch: epoch.Uint64(), rnd: rndSeed, shardState: shardState, numShards: len(shardState)}, nil -} - -// CalculateNewShardState get sharding state from previous epoch and calculate sharding state for new epoch -func CalculateNewShardState(bc *BlockChain, epoch *big.Int) (shard.State, error) { - if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 { - return CalculateInitShardState(), nil - } - prevEpoch := new(big.Int).Sub(epoch, common.Big1) - ss, err := GetShardingStateFromBlockChain(bc, prevEpoch) - if err != nil { - return nil, ctxerror.New("cannot retrieve previous sharding state"). - WithCause(err) - } - utils.Logger().Info().Float64("percentage", CuckooRate).Msg("Cuckoo Rate") - return ss.shardState, nil -} - -// TODO ek – shardingSchedule should really be part of a general-purpose network -// configuration. We are OK for the time being, -// until the day we should let one node process join multiple networks. - -// ShardingSchedule is the sharding configuration schedule. -// Depends on the type of the network. Defaults to the mainnet schedule. -var ShardingSchedule shardingconfig.Schedule = shardingconfig.MainnetSchedule - -// CalculateInitShardState returns the initial shard state at genesis. -func CalculateInitShardState() shard.State { - return CalculateShardState(big.NewInt(GenesisEpoch)) -} - -// CalculateShardState returns the shard state based on epoch number -// This api for getting shard state is what should be used to get shard state regardless of -// current chain dependency (ex. getting shard state from block header received during cross-shard transaction) -func CalculateShardState(epoch *big.Int) shard.State { - utils.Logger().Info().Int64("epoch", epoch.Int64()).Msg("Get Shard State of Epoch.") - shardingConfig := ShardingSchedule.InstanceForEpoch(epoch) - shardNum := int(shardingConfig.NumShards()) - shardHarmonyNodes := shardingConfig.NumHarmonyOperatedNodesPerShard() - shardSize := shardingConfig.NumNodesPerShard() - hmyAccounts := shardingConfig.HmyAccounts() - fnAccounts := shardingConfig.FnAccounts() - - shardState := shard.State{} - for i := 0; i < shardNum; i++ { - com := shard.Committee{ShardID: uint32(i)} - for j := 0; j < shardHarmonyNodes; j++ { - index := i + j*shardNum // The initial account to use for genesis nodes - - pub := &bls.PublicKey{} - pub.DeserializeHexStr(hmyAccounts[index].BlsPublicKey) - pubKey := shard.BlsPublicKey{} - pubKey.FromLibBLSPublicKey(pub) - // TODO: directly read address for bls too - curNodeID := shard.NodeID{ - EcdsaAddress: common2.ParseAddr(hmyAccounts[index].Address), - BlsPublicKey: pubKey, - } - com.NodeList = append(com.NodeList, curNodeID) - } - - // add FN runner's key - for j := shardHarmonyNodes; j < shardSize; j++ { - index := i + (j-shardHarmonyNodes)*shardNum - - pub := &bls.PublicKey{} - pub.DeserializeHexStr(fnAccounts[index].BlsPublicKey) - - pubKey := shard.BlsPublicKey{} - pubKey.FromLibBLSPublicKey(pub) - // TODO: directly read address for bls too - curNodeID := shard.NodeID{ - EcdsaAddress: common2.ParseAddr(fnAccounts[index].Address), - BlsPublicKey: pubKey, - } - com.NodeList = append(com.NodeList, curNodeID) - } - shardState = append(shardState, com) - } - return shardState -} - -// CalculatePublicKeys returns the publickeys given epoch and shardID -func CalculatePublicKeys(epoch *big.Int, shardID uint32) []*bls.PublicKey { - shardState := CalculateShardState(epoch) - - // Update validator public keys - committee := shardState.FindCommitteeByID(shardID) - if committee == nil { - utils.Logger().Warn().Uint32("shardID", shardID).Uint64("epoch", epoch.Uint64()).Msg("Cannot find committee") - return nil - } - pubKeys := []*bls.PublicKey{} - for _, node := range committee.NodeList { - pubKey := &bls.PublicKey{} - pubKeyBytes := node.BlsPublicKey[:] - err := pubKey.Deserialize(pubKeyBytes) - if err != nil { - utils.Logger().Warn().Str("pubKeyBytes", hex.EncodeToString(pubKeyBytes)).Msg("Cannot Deserialize pubKey") - return nil - } - pubKeys = append(pubKeys, pubKey) - } - return pubKeys -} diff --git a/core/resharding.md b/core/resharding.md deleted file mode 100644 index 54dadb24d..000000000 --- a/core/resharding.md +++ /dev/null @@ -1,12 +0,0 @@ -## Resharding - -In current design, the epoch is defined to be fixed length, the epoch length is a constant parameter BlocksPerEpoch. In future, it will be dynamically adjustable according to security parameter. During the epoch transition, suppose there are N shards, we sort the shards according to the size of active nodes (that had staking for next epoch). The first N/2 larger shards will be called active committees, and the last N/2 smaller shards will be called inactive committees. Don't be confused by -the name, they are all normal shards with same function. - -All the information about sharding will be stored in BeaconChain. A sharding state is defined as a map which maps each NodeID to the ShardID the node belongs to. Every node will have a unique NodeID and be mapped to one ShardID. At the beginning of a new epoch, the BeaconChain leader will propose a new block containing the new sharding state, the new sharding state is uniquely determined by the randomness generated by distributed randomness protocol. During the consensus process, all the validators will perform the same calculation and verify the proposed sharding state is valid. After consensus is reached, each node will write the new sharding state into the block. This block is called epoch block. In current code, it's the first block of each epoch in BeaconChain. - -The main function of resharding is CalculcateNewShardState. It will take 3 inputs: newNodeList, oldShardState, randomSeed and output newShardState. -The newNodeList will be retrieved from BeaconChain staking transaction during the previous epoch. The randomSeed and oldShardState is stored in previous epoch block. It should be noticed that the randomSeed generation currently is mocked. After the distributed randomness protocol(drand) is ready, the drand service will generate the random seed for resharding. - -The resharding process is as follows: we first get newNodeList from staking transactions from previous epoch and assign the new nodes evenly into the N/2 active committees. Then, we kick out X% of nodes from each active committees and put these kicked out nodes into inactive committees evenly. The percentage X roughly equals to the percentage of new nodes into active committee in order to balance the committee size. - diff --git a/core/resharding_test.go b/core/resharding_test.go deleted file mode 100644 index dd59cc5ed..000000000 --- a/core/resharding_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package core - -import ( - "fmt" - "math/rand" - "strconv" - "testing" - - "github.com/ethereum/go-ethereum/common" - - "github.com/harmony-one/harmony/shard" - - "github.com/stretchr/testify/assert" -) - -var ( - blsPubKey1 = [48]byte{} - blsPubKey2 = [48]byte{} - blsPubKey3 = [48]byte{} - blsPubKey4 = [48]byte{} - blsPubKey5 = [48]byte{} - blsPubKey6 = [48]byte{} - blsPubKey7 = [48]byte{} - blsPubKey8 = [48]byte{} - blsPubKey9 = [48]byte{} - blsPubKey10 = [48]byte{} -) - -func init() { - copy(blsPubKey1[:], []byte("random key 1")) - copy(blsPubKey2[:], []byte("random key 2")) - copy(blsPubKey3[:], []byte("random key 3")) - copy(blsPubKey4[:], []byte("random key 4")) - copy(blsPubKey5[:], []byte("random key 5")) - copy(blsPubKey6[:], []byte("random key 6")) - copy(blsPubKey7[:], []byte("random key 7")) - copy(blsPubKey8[:], []byte("random key 8")) - copy(blsPubKey9[:], []byte("random key 9")) - copy(blsPubKey10[:], []byte("random key 10")) -} - -func fakeGetInitShardState(numberOfShards, numOfNodes int) shard.State { - rand.Seed(int64(42)) - shardState := shard.State{} - for i := 0; i < numberOfShards; i++ { - sid := uint32(i) - com := shard.Committee{ShardID: sid} - for j := 0; j < numOfNodes; j++ { - nid := strconv.Itoa(int(rand.Int63())) - blsPubKey := [48]byte{} - copy(blsPubKey1[:], []byte(nid)) - com.NodeList = append(com.NodeList, shard.NodeID{ - EcdsaAddress: common.BytesToAddress([]byte(nid)), - BlsPublicKey: blsPubKey, - }) - } - shardState = append(shardState, com) - } - return shardState -} - -func fakeNewNodeList(seed int64) []shard.NodeID { - rand.Seed(seed) - numNewNodes := rand.Intn(10) - nodeList := []shard.NodeID{} - for i := 0; i < numNewNodes; i++ { - nid := strconv.Itoa(int(rand.Int63())) - blsPubKey := [48]byte{} - copy(blsPubKey1[:], []byte(nid)) - nodeList = append(nodeList, shard.NodeID{ - EcdsaAddress: common.BytesToAddress([]byte(nid)), - BlsPublicKey: blsPubKey, - }) - } - return nodeList -} - -func TestFakeNewNodeList(t *testing.T) { - nodeList := fakeNewNodeList(42) - fmt.Println("newNodeList: ", nodeList) -} - -func TestShuffle(t *testing.T) { - nodeList := []shard.NodeID{ - {EcdsaAddress: common.Address{0x12}, BlsPublicKey: blsPubKey1}, - {EcdsaAddress: common.Address{0x22}, BlsPublicKey: blsPubKey2}, - {EcdsaAddress: common.Address{0x32}, BlsPublicKey: blsPubKey3}, - {EcdsaAddress: common.Address{0x42}, BlsPublicKey: blsPubKey4}, - {EcdsaAddress: common.Address{0x52}, BlsPublicKey: blsPubKey5}, - {EcdsaAddress: common.Address{0x62}, BlsPublicKey: blsPubKey6}, - {EcdsaAddress: common.Address{0x72}, BlsPublicKey: blsPubKey7}, - {EcdsaAddress: common.Address{0x82}, BlsPublicKey: blsPubKey8}, - {EcdsaAddress: common.Address{0x92}, BlsPublicKey: blsPubKey9}, - {EcdsaAddress: common.Address{0x02}, BlsPublicKey: blsPubKey10}, - } - - cpList := []shard.NodeID{} - cpList = append(cpList, nodeList...) - Shuffle(nodeList) - cnt := 0 - for i := 0; i < 10; i++ { - if cpList[i] == nodeList[i] { - cnt++ - } - } - if cnt == 10 { - t.Error("Shuffle list is the same as original list") - } - return -} - -func TestSortCommitteeBySize(t *testing.T) { - shardState := fakeGetInitShardState(6, 10) - ss := &ShardingState{epoch: 1, rnd: 42, shardState: shardState, numShards: len(shardState)} - ss.sortCommitteeBySize() - for i := 0; i < ss.numShards-1; i++ { - assert.Equal(t, true, len(ss.shardState[i].NodeList) >= len(ss.shardState[i+1].NodeList)) - } -} - -func TestUpdateShardState(t *testing.T) { - shardState := fakeGetInitShardState(6, 10) - ss := &ShardingState{epoch: 1, rnd: 42, shardState: shardState, numShards: len(shardState)} - newNodeList := []shard.NodeID{ - {EcdsaAddress: common.Address{0x12}, BlsPublicKey: blsPubKey1}, - {EcdsaAddress: common.Address{0x22}, BlsPublicKey: blsPubKey2}, - {EcdsaAddress: common.Address{0x32}, BlsPublicKey: blsPubKey3}, - {EcdsaAddress: common.Address{0x42}, BlsPublicKey: blsPubKey4}, - {EcdsaAddress: common.Address{0x52}, BlsPublicKey: blsPubKey5}, - {EcdsaAddress: common.Address{0x62}, BlsPublicKey: blsPubKey6}, - } - - ss.Reshard(newNodeList, 0.2) - assert.Equal(t, 6, ss.numShards) -} - -func TestAssignNewNodes(t *testing.T) { - shardState := fakeGetInitShardState(2, 2) - ss := &ShardingState{epoch: 1, rnd: 42, shardState: shardState, numShards: len(shardState)} - newNodes := []shard.NodeID{ - {EcdsaAddress: common.Address{0x12}, BlsPublicKey: blsPubKey1}, - {EcdsaAddress: common.Address{0x22}, BlsPublicKey: blsPubKey2}, - {EcdsaAddress: common.Address{0x32}, BlsPublicKey: blsPubKey3}, - } - - ss.assignNewNodes(newNodes) - assert.Equal(t, 2, ss.numShards) - assert.Equal(t, 5, len(ss.shardState[0].NodeList)) -} diff --git a/core/state_processor.go b/core/state_processor.go index f72c2e809..fee37267e 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -30,6 +30,7 @@ import ( "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" staking "github.com/harmony-one/harmony/staking/types" ) @@ -122,7 +123,7 @@ func getTransactionType(config *params.ChainConfig, header *block.Header, tx *ty if header.ShardID() == tx.ShardID() && (!config.IsCrossTx(header.Epoch()) || tx.ShardID() == tx.ToShardID()) { return types.SameShardTx } - numShards := ShardingSchedule.InstanceForEpoch(header.Epoch()).NumShards() + numShards := shard.Schedule.InstanceForEpoch(header.Epoch()).NumShards() // Assuming here all the shards are consecutive from 0 to n-1, n is total number of shards if tx.ShardID() != tx.ToShardID() && header.ShardID() == tx.ShardID() && tx.ToShardID() < numShards { return types.SubtractionOnly diff --git a/core/values/blockchain.go b/core/values/blockchain.go deleted file mode 100644 index 1225e2277..000000000 --- a/core/values/blockchain.go +++ /dev/null @@ -1,10 +0,0 @@ -package values - -const ( - // BeaconChainShardID is the ShardID of the BeaconChain - BeaconChainShardID = 0 - // VotingPowerReduceBlockThreshold roughly corresponds to 3 hours - VotingPowerReduceBlockThreshold = 1350 - // VotingPowerFullReduce roughly corresponds to 12 hours - VotingPowerFullReduce = 4 * VotingPowerReduceBlockThreshold -) diff --git a/drand/drand_leader.go b/drand/drand_leader.go index f9d7346be..52b844c7a 100644 --- a/drand/drand_leader.go +++ b/drand/drand_leader.go @@ -4,17 +4,17 @@ import ( "bytes" "time" - "github.com/harmony-one/harmony/crypto/bls" - protobuf "github.com/golang/protobuf/proto" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/vdf" "github.com/harmony-one/harmony/crypto/vrf/p256" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/shard" ) const ( @@ -30,7 +30,7 @@ func (dRand *DRand) WaitForEpochBlock(blockChannel chan *types.Block, stopChan c default: // keep waiting for epoch block newBlock := <-blockChannel - if core.IsEpochLastBlock(newBlock) { + if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { dRand.init(newBlock) } // TODO: use real vrf diff --git a/internal/chain/engine.go b/internal/chain/engine.go index c9f0d9306..e927e6a65 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -8,12 +8,12 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus/engine" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" "github.com/pkg/errors" "golang.org/x/crypto/sha3" @@ -116,6 +116,7 @@ func (e *engineImpl) VerifySeal(chain engine.ChainReader, header *block.Header) return nil } publicKeys, err := ReadPublicKeysFromLastBlock(chain, header) + if err != nil { return ctxerror.New("[VerifySeal] Cannot retrieve publickeys from last block").WithCause(err) } @@ -166,7 +167,7 @@ func (e *engineImpl) Finalize( func QuorumForBlock(chain engine.ChainReader, h *block.Header, reCalculate bool) (quorum int, err error) { var ss shard.State if reCalculate { - ss = core.CalculateShardState(h.Epoch()) + ss, _ = committee.WithStakingEnabled.ReadFromComputation(h.Epoch(), *chain.Config(), nil) } else { ss, err = chain.ReadShardState(h.Epoch()) if err != nil { @@ -225,7 +226,7 @@ func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate b var shardState shard.State var err error if reCalculate { - shardState = core.CalculateShardState(header.Epoch()) + shardState, _ = committee.WithStakingEnabled.ReadFromComputation(header.Epoch(), *chain.Config(), nil) } else { shardState, err = chain.ReadShardState(header.Epoch()) if err != nil { diff --git a/internal/configs/sharding/localnet.go b/internal/configs/sharding/localnet.go index 2a0f0b302..a36a3b714 100644 --- a/internal/configs/sharding/localnet.go +++ b/internal/configs/sharding/localnet.go @@ -152,8 +152,11 @@ func (ls localnetSchedule) GetShardingStructure(numShard, shardID int) []map[str return res } -var localnetReshardingEpoch = []*big.Int{big.NewInt(0), big.NewInt(localnetV1Epoch), big.NewInt(localnetV2Epoch)} - -var localnetV0 = MustNewInstance(2, 7, 5, genesis.LocalHarmonyAccounts, genesis.LocalFnAccounts, localnetReshardingEpoch) -var localnetV1 = MustNewInstance(2, 8, 5, genesis.LocalHarmonyAccountsV1, genesis.LocalFnAccountsV1, localnetReshardingEpoch) -var localnetV2 = MustNewInstance(2, 9, 6, genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, localnetReshardingEpoch) +var ( + localnetReshardingEpoch = []*big.Int{ + big.NewInt(0), big.NewInt(localnetV1Epoch), big.NewInt(localnetV2Epoch), + } + localnetV0 = MustNewInstance(2, 7, 5, genesis.LocalHarmonyAccounts, genesis.LocalFnAccounts, localnetReshardingEpoch) + localnetV1 = MustNewInstance(2, 8, 5, genesis.LocalHarmonyAccountsV1, genesis.LocalFnAccountsV1, localnetReshardingEpoch) + localnetV2 = MustNewInstance(2, 9, 6, genesis.LocalHarmonyAccountsV2, genesis.LocalFnAccountsV2, localnetReshardingEpoch) +) diff --git a/internal/hmyapi/blockchain.go b/internal/hmyapi/blockchain.go index 525f239ee..ab7ade411 100644 --- a/internal/hmyapi/blockchain.go +++ b/internal/hmyapi/blockchain.go @@ -20,6 +20,7 @@ import ( internal_bls "github.com/harmony-one/harmony/crypto/bls" internal_common "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" ) const ( @@ -359,10 +360,10 @@ func (s *PublicBlockChainAPI) GetDelegatorsInformation(ctx context.Context, addr func (s *PublicBlockChainAPI) GetShardingStructure(ctx context.Context) ([]map[string]interface{}, error) { // Get header and number of shards. epoch := s.GetEpoch(ctx) - numShard := core.ShardingSchedule.InstanceForEpoch(big.NewInt(int64(epoch))).NumShards() + numShard := shard.Schedule.InstanceForEpoch(big.NewInt(int64(epoch))).NumShards() // Return shareding structure for each case. - return core.ShardingSchedule.GetShardingStructure(int(numShard), int(s.b.GetShardID())), nil + return shard.Schedule.GetShardingStructure(int(numShard), int(s.b.GetShardID())), nil } // GetShardID returns shard ID of the requested node. diff --git a/internal/params/config.go b/internal/params/config.go index 9bb5e49e0..21af1fda3 100644 --- a/internal/params/config.go +++ b/internal/params/config.go @@ -36,7 +36,7 @@ var ( ChainID: TestnetChainID, CrossTxEpoch: big.NewInt(0), CrossLinkEpoch: big.NewInt(0), - StakingEpoch: big.NewInt(0), + StakingEpoch: EpochTBD, EIP155Epoch: big.NewInt(0), S3Epoch: big.NewInt(0), } diff --git a/node/node.go b/node/node.go index e068aa700..8e97b23e0 100644 --- a/node/node.go +++ b/node/node.go @@ -20,7 +20,6 @@ import ( "github.com/harmony-one/harmony/contracts" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/drand" "github.com/harmony-one/harmony/internal/chain" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" @@ -33,6 +32,7 @@ import ( "github.com/harmony-one/harmony/p2p" p2p_host "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" ) @@ -277,7 +277,7 @@ func (node *Node) tryBroadcast(tx *types.Transaction) { // Add new transactions to the pending transaction list. func (node *Node) addPendingTransactions(newTxs types.Transactions) { - txPoolLimit := core.ShardingSchedule.MaxTxPoolSizeLimit() + txPoolLimit := shard.Schedule.MaxTxPoolSizeLimit() node.pendingTxMutex.Lock() for _, tx := range newTxs { if _, ok := node.pendingTransactions[tx.Hash()]; !ok { @@ -293,7 +293,7 @@ func (node *Node) addPendingTransactions(newTxs types.Transactions) { // Add new staking transactions to the pending staking transaction list. func (node *Node) addPendingStakingTransactions(newStakingTxs staking.StakingTransactions) { - txPoolLimit := core.ShardingSchedule.MaxTxPoolSizeLimit() + txPoolLimit := shard.Schedule.MaxTxPoolSizeLimit() node.pendingStakingTxMutex.Lock() for _, tx := range newStakingTxs { if _, ok := node.pendingStakingTransactions[tx.Hash()]; !ok { @@ -350,7 +350,7 @@ func (node *Node) AddPendingReceipts(receipts *types.CXReceiptsProof) { // Take out a subset of valid transactions from the pending transaction list // Note the pending transaction list will then contain the rest of the txs func (node *Node) getTransactionsForNewBlock(coinbase common.Address) (types.Transactions, staking.StakingTransactions) { - txsThrottleConfig := core.ShardingSchedule.TxsThrottleConfig() + txsThrottleConfig := shard.Schedule.TxsThrottleConfig() // the next block number to be added in consensus protocol, which is always one more than current chain header block newBlockNum := node.Blockchain().CurrentBlock().NumberU64() + 1 @@ -483,7 +483,8 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc node.chainConfig = chainConfig collection := shardchain.NewCollection( - chainDBFactory, &genesisInitializer{&node}, chain.Engine, &chainConfig) + chainDBFactory, &genesisInitializer{&node}, chain.Engine, &chainConfig, + ) if isArchival { collection.DisableCache() } @@ -505,7 +506,7 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc node.CxPool = core.NewCxPool(core.CxPoolSize) node.Worker = worker.New(node.Blockchain().Config(), blockchain, chain.Engine) - if node.Blockchain().ShardID() != values.BeaconChainShardID { + if node.Blockchain().ShardID() != shard.BeaconChainShardID { node.BeaconWorker = worker.New(node.Beaconchain().Config(), beaconChain, chain.Engine) } @@ -545,44 +546,42 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc // Setup initial state of syncing. node.peerRegistrationRecord = make(map[string]*syncConfig) - node.startConsensus = make(chan struct{}) - go node.bootstrapConsensus() - return &node } -// CalculateInitShardState initialize shard state from latest epoch and update committee pub keys for consensus and drand -func (node *Node) CalculateInitShardState() (err error) { +// InitConsensusWithValidators initialize shard state from latest epoch and update committee pub +// keys for consensus and drand +func (node *Node) InitConsensusWithValidators() (err error) { if node.Consensus == nil { - return ctxerror.New("[CalculateInitShardState] consenus is nil; Cannot figure out shardID") + return ctxerror.New("[InitConsensusWithValidators] consenus is nil; Cannot figure out shardID") } shardID := node.Consensus.ShardID - - // Get genesis epoch shard state from chain blockNum := node.Blockchain().CurrentBlock().NumberU64() node.Consensus.SetMode(consensus.Listening) - epoch := core.ShardingSchedule.CalcEpochNumber(blockNum) + epoch := shard.Schedule.CalcEpochNumber(blockNum) utils.Logger().Info(). Uint64("blockNum", blockNum). Uint32("shardID", shardID). Uint64("epoch", epoch.Uint64()). - Msg("[CalculateInitShardState] Try To Get PublicKeys from database") - pubKeys := core.CalculatePublicKeys(epoch, shardID) + Msg("[InitConsensusWithValidators] Try To Get PublicKeys") + _, pubKeys := committee.WithStakingEnabled.ComputePublicKeys( + epoch, node.Consensus.ChainReader, int(shardID), + ) if len(pubKeys) == 0 { return ctxerror.New( - "[CalculateInitShardState] PublicKeys is Empty, Cannot update public keys", + "[InitConsensusWithValidators] PublicKeys is Empty, Cannot update public keys", "shardID", shardID, "blockNum", blockNum) } - for _, key := range pubKeys { - if key.IsEqual(node.Consensus.PubKey) { + for i := range pubKeys { + if pubKeys[i].IsEqual(node.Consensus.PubKey) { utils.Logger().Info(). Uint64("blockNum", blockNum). Int("numPubKeys", len(pubKeys)). - Msg("[CalculateInitShardState] Successfully updated public keys") + Msg("[InitConsensusWithValidators] Successfully updated public keys") node.Consensus.UpdatePublicKeys(pubKeys) node.Consensus.SetMode(consensus.Normal) return nil diff --git a/node/node_cross_shard.go b/node/node_cross_shard.go index e1040db43..ee306c389 100644 --- a/node/node_cross_shard.go +++ b/node/node_cross_shard.go @@ -4,13 +4,9 @@ import ( "encoding/binary" "errors" - "github.com/harmony-one/harmony/p2p/host" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" - "github.com/harmony-one/bls/ffi/go/bls" - proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/core" @@ -19,6 +15,8 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/p2p/host" + "github.com/harmony-one/harmony/shard" ) // BroadcastCXReceipts broadcasts cross shard receipts to correspoding @@ -38,7 +36,7 @@ func (node *Node) BroadcastCXReceipts(newBlock *types.Block, lastCommits []byte) //#### END Read payload data from committed msg epoch := newBlock.Header().Epoch() - shardingConfig := core.ShardingSchedule.InstanceForEpoch(epoch) + shardingConfig := shard.Schedule.InstanceForEpoch(epoch) shardNum := int(shardingConfig.NumShards()) myShardID := node.Consensus.ShardID utils.Logger().Info().Int("shardNum", shardNum).Uint32("myShardID", myShardID).Uint64("blockNum", newBlock.NumberU64()).Msg("[BroadcastCXReceipts]") @@ -345,7 +343,7 @@ func (node *Node) ProposeCrossLinkDataForBeaconchain() (types.CrossLinks, error) Uint64("blockNum", node.Blockchain().CurrentBlock().NumberU64()+1). Msg("Proposing cross links ...") curBlock := node.Blockchain().CurrentBlock() - numShards := core.ShardingSchedule.InstanceForEpoch(curBlock.Header().Epoch()).NumShards() + numShards := shard.Schedule.InstanceForEpoch(curBlock.Header().Epoch()).NumShards() shardCrossLinks := make([]types.CrossLinks, numShards) diff --git a/node/node_explorer.go b/node/node_explorer.go index 64ba627d8..70fa2fe6e 100644 --- a/node/node_explorer.go +++ b/node/node_explorer.go @@ -14,6 +14,7 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/shard" ) var once sync.Once @@ -107,7 +108,7 @@ func (node *Node) ExplorerMessageHandler(payload []byte) { func (node *Node) AddNewBlockForExplorer(block *types.Block) { utils.Logger().Debug().Uint64("blockHeight", block.NumberU64()).Msg("[Explorer] Adding new block for explorer node") if err := node.AddNewBlock(block); err == nil { - if core.IsEpochLastBlock(block) { + if shard.Schedule.IsLastBlock(block.Number().Uint64()) { node.Consensus.UpdateConsensusInformation() } // Clean up the blocks to avoid OOM. diff --git a/node/node_genesis.go b/node/node_genesis.go index 47f36760e..bc9740743 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -21,6 +21,7 @@ import ( "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" ) const ( @@ -41,8 +42,10 @@ type genesisInitializer struct { // InitChainDB sets up a new genesis block in the database for the given shard. func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) error { - shardState := core.CalculateInitShardState() - if shardID != 0 { + shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + big.NewInt(core.GenesisEpoch), gi.node.chainConfig, nil, + ) + if shardID != shard.BeaconChainShardID { // store only the local shard for shard chains c := shardState.FindCommitteeByID(shardID) if c == nil { diff --git a/node/node_handler.go b/node/node_handler.go index 5551333bf..947cbfd00 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -5,11 +5,9 @@ import ( "context" "math/big" "math/rand" - "sync" "sync/atomic" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" @@ -19,7 +17,6 @@ import ( proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" proto_node "github.com/harmony-one/harmony/api/proto/node" "github.com/harmony-one/harmony/block" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" @@ -325,7 +322,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit if node.NodeConfig.ShardID == 0 { node.BroadcastNewBlock(newBlock) } - if node.NodeConfig.ShardID != 0 && newBlock.Epoch().Cmp(node.Blockchain().Config().CrossLinkEpoch) >= 0 { + if node.NodeConfig.ShardID != shard.BeaconChainShardID && newBlock.Epoch().Cmp(node.Blockchain().Config().CrossLinkEpoch) >= 0 { node.BroadcastCrossLinkHeader(newBlock) } node.BroadcastCXReceipts(newBlock, commitSigAndBitmap) @@ -348,7 +345,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit node.BroadcastMissingCXReceipts() // Update consensus keys at last so the change of leader status doesn't mess up normal flow - if core.IsEpochLastBlock(newBlock) { + if shard.Schedule.IsLastBlock(newBlock.Number().Uint64()) { node.Consensus.UpdateConsensusInformation() } @@ -380,33 +377,7 @@ func (node *Node) PostConsensusProcessing(newBlock *types.Block, commitSigAndBit // node.ConfirmedBlockChannel <- newBlock // }() //} - - // TODO: enable staking - // TODO: update staking information once per epoch. - //node.UpdateStakingList(node.QueryStakeInfo()) - //node.printStakingList() } - - // TODO: enable shard state update - //newBlockHeader := newBlock.Header() - //if newBlockHeader.ShardStateHash != (common.Hash{}) { - // if node.Consensus.ShardID == 0 { - // // TODO ek – this is a temp hack until beacon chain sync is fixed - // // End-of-epoch block on beacon chain; block's EpochState is the - // // master resharding table. Broadcast it to the network. - // if err := node.broadcastEpochShardState(newBlock); err != nil { - // e := ctxerror.New("cannot broadcast shard state").WithCause(err) - // ctxerror.Log15(utils.Logger().Error, e) - // } - // } - // shardState, err := newBlockHeader.CalculateShardState() - // if err != nil { - // e := ctxerror.New("cannot get shard state from header").WithCause(err) - // ctxerror.Log15(utils.Logger().Error, e) - // } else { - // node.transitionIntoNextEpoch(shardState) - // } - //} } } @@ -448,43 +419,6 @@ func (node *Node) AddNewBlock(newBlock *types.Block) error { return err } -type genesisNode struct { - ShardID uint32 - MemberIndex int - NodeID shard.NodeID -} - -var ( - genesisCatalogOnce sync.Once - genesisNodeByStakingAddress = make(map[common.Address]*genesisNode) - genesisNodeByConsensusKey = make(map[shard.BlsPublicKey]*genesisNode) -) - -func initGenesisCatalog() { - genesisShardState := core.CalculateInitShardState() - for _, committee := range genesisShardState { - for i, nodeID := range committee.NodeList { - genesisNode := &genesisNode{ - ShardID: committee.ShardID, - MemberIndex: i, - NodeID: nodeID, - } - genesisNodeByStakingAddress[nodeID.EcdsaAddress] = genesisNode - genesisNodeByConsensusKey[nodeID.BlsPublicKey] = genesisNode - } - } -} - -func getGenesisNodeByStakingAddress(address common.Address) *genesisNode { - genesisCatalogOnce.Do(initGenesisCatalog) - return genesisNodeByStakingAddress[address] -} - -func getGenesisNodeByConsensusKey(key shard.BlsPublicKey) *genesisNode { - genesisCatalogOnce.Do(initGenesisCatalog) - return genesisNodeByConsensusKey[key] -} - func (node *Node) pingMessageHandler(msgPayload []byte, sender libp2p_peer.ID) int { ping, err := proto_discovery.GetPingMessage(msgPayload) if err != nil { diff --git a/node/node_handler_test.go b/node/node_handler_test.go index 49a012270..e1609b9e0 100644 --- a/node/node_handler_test.go +++ b/node/node_handler_test.go @@ -6,12 +6,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/crypto/bls" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" ) func TestAddNewBlock(t *testing.T) { @@ -25,7 +25,7 @@ func TestAddNewBlock(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -58,7 +58,7 @@ func TestVerifyNewBlock(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/node/node_newblock.go b/node/node_newblock.go index b47445201..f6df9c1bd 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -6,11 +6,11 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" ) // Constants of proposing a new block @@ -124,12 +124,13 @@ func (node *Node) proposeNewBlock() (*types.Block, error) { } func (node *Node) proposeShardStateWithoutBeaconSync(block *types.Block) shard.State { - if block == nil || !core.IsEpochLastBlock(block) { + if block == nil || !shard.Schedule.IsLastBlock(block.Number().Uint64()) { return nil } - - nextEpoch := new(big.Int).Add(block.Header().Epoch(), common.Big1) - return core.CalculateShardState(nextEpoch) + shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + new(big.Int).Add(block.Header().Epoch(), common.Big1), node.chainConfig, nil, + ) + return shardState } func (node *Node) proposeShardState(block *types.Block) error { @@ -144,13 +145,15 @@ func (node *Node) proposeShardState(block *types.Block) error { func (node *Node) proposeBeaconShardState(block *types.Block) error { // TODO ek - replace this with variable epoch logic. - if !core.IsEpochLastBlock(block) { + if !shard.Schedule.IsLastBlock(block.Number().Uint64()) { // We haven't reached the end of this epoch; don't propose yet. return nil } - nextEpoch := new(big.Int).Add(block.Header().Epoch(), common.Big1) - // TODO: add logic for EPoS - shardState, err := core.CalculateNewShardState(node.Blockchain(), nextEpoch) + // TODO Use ReadFromComputation + prevEpoch := new(big.Int).Sub(block.Header().Epoch(), common.Big1) + shardState, err := committee.WithStakingEnabled.ReadFromChain( + prevEpoch, node.Blockchain(), + ) if err != nil { return err } diff --git a/node/node_resharding.go b/node/node_resharding.go index fa9bc1e63..0c55369d9 100644 --- a/node/node_resharding.go +++ b/node/node_resharding.go @@ -15,13 +15,13 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/bls/ffi/go/bls" proto_node "github.com/harmony-one/harmony/api/proto/node" - "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" nodeconfig "github.com/harmony-one/harmony/internal/configs/node" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p/host" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" ) // validateNewShardState validate whether the new shard state root matches @@ -30,8 +30,8 @@ func (node *Node) validateNewShardState(block *types.Block) error { header := block.Header() if header.ShardStateHash() == (common.Hash{}) { // No new shard state was proposed - if block.ShardID() == 0 { - if core.IsEpochLastBlock(block) { + if block.ShardID() == shard.BeaconChainShardID { + if shard.Schedule.IsLastBlock(block.Number().Uint64()) { // TODO ek - invoke view change return errors.New("beacon leader did not propose resharding") } @@ -51,14 +51,17 @@ func (node *Node) validateNewShardState(block *types.Block) error { return err } proposed := *shardState - if block.ShardID() == 0 { + if block.ShardID() == shard.BeaconChainShardID { // Beacon validators independently recalculate the master state and // compare it against the proposed copy. - nextEpoch := new(big.Int).Add(block.Header().Epoch(), common.Big1) // TODO ek – this may be called from regular shards, // for vetting beacon chain blocks received during block syncing. // DRand may or or may not get in the way. Test this out. - expected, err := core.CalculateNewShardState(node.Blockchain(), nextEpoch) + expected, err := committee.WithStakingEnabled.ReadFromChain( + new(big.Int).Sub(block.Header().Epoch(), common.Big1), + node.Beaconchain(), + ) + if err != nil { return ctxerror.New("cannot calculate expected shard state"). WithCause(err) diff --git a/node/node_test.go b/node/node_test.go index a84cc1122..77cd65d08 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -11,7 +11,6 @@ import ( proto_discovery "github.com/harmony-one/harmony/api/proto/discovery" "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/consensus/quorum" - "github.com/harmony-one/harmony/core/values" bls2 "github.com/harmony-one/harmony/crypto/bls" "github.com/harmony-one/harmony/crypto/pki" "github.com/harmony-one/harmony/drand" @@ -19,6 +18,7 @@ import ( "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/p2p/p2pimpl" + "github.com/harmony-one/harmony/shard" "github.com/stretchr/testify/assert" ) @@ -35,7 +35,7 @@ func TestNewNode(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -202,7 +202,7 @@ func TestAddPeers(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) @@ -252,7 +252,7 @@ func TestAddBeaconPeer(t *testing.T) { } decider := quorum.NewDecider(quorum.SuperMajorityVote) consensus, err := consensus.New( - host, values.BeaconChainShardID, leader, blsKey, decider, + host, shard.BeaconChainShardID, leader, blsKey, decider, ) if err != nil { t.Fatalf("Cannot craeate consensus: %v", err) diff --git a/node/worker/worker.go b/node/worker/worker.go index d998d0e27..8fc3a2054 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -13,13 +13,13 @@ import ( "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" - "github.com/harmony-one/harmony/core/values" "github.com/harmony-one/harmony/core/vm" shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/params" "github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/shard" + "github.com/harmony-one/harmony/shard/committee" staking "github.com/harmony-one/harmony/staking/types" ) @@ -162,7 +162,7 @@ func (w *Worker) SelectStakingTransactionsForNewBlock( coinbase common.Address) (staking.StakingTransactions, staking.StakingTransactions, staking.StakingTransactions) { // only beaconchain process staking transaction - if w.chain.ShardID() != values.BeaconChainShardID { + if w.chain.ShardID() != shard.BeaconChainShardID { utils.Logger().Warn().Msgf("Invalid shardID: %v", w.chain.ShardID()) return nil, nil, nil } @@ -365,11 +365,13 @@ func (w *Worker) IncomingReceipts() []*types.CXReceiptsProof { // ProposeShardStateWithoutBeaconSync proposes the next shard state for next epoch. func (w *Worker) ProposeShardStateWithoutBeaconSync() shard.State { - if !core.ShardingSchedule.IsLastBlock(w.current.header.Number().Uint64()) { + if !shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) { return nil } - nextEpoch := new(big.Int).Add(w.current.header.Epoch(), common.Big1) - return core.CalculateShardState(nextEpoch) + shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + new(big.Int).Add(w.current.header.Epoch(), common.Big1), *w.config, nil, + ) + return shardState } // FinalizeNewBlock generate a new block for the next consensus round. diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go new file mode 100644 index 000000000..69afb9259 --- /dev/null +++ b/shard/committee/assignment.go @@ -0,0 +1,261 @@ +package committee + +import ( + "math/big" + "sort" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/block" + common2 "github.com/harmony-one/harmony/internal/common" + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" + "github.com/harmony-one/harmony/internal/ctxerror" + "github.com/harmony-one/harmony/internal/params" + "github.com/harmony-one/harmony/internal/utils" + "github.com/harmony-one/harmony/numeric" + "github.com/harmony-one/harmony/shard" + staking "github.com/harmony-one/harmony/staking/types" +) + +// StateID means reading off whole network when using calls that accept +// a shardID parameter +const StateID = -1 + +// MembershipList .. +type MembershipList interface { + ReadFromComputation( + epoch *big.Int, config params.ChainConfig, reader StakingCandidatesReader, + ) (shard.State, error) + ReadFromChain(epoch *big.Int, reader ChainReader) (shard.State, error) +} + +// PublicKeys per epoch +type PublicKeys interface { + // If call shardID with StateID then only superCommittee is non-nil, + // otherwise get back the shardSpecific slice as well. + ComputePublicKeys( + epoch *big.Int, reader ChainReader, shardID int, + ) (superCommittee, shardSpecific []*bls.PublicKey) + + ReadPublicKeysFromDB( + hash common.Hash, reader ChainReader, + ) ([]*bls.PublicKey, error) +} + +// Reader .. +type Reader interface { + PublicKeys + MembershipList +} + +// StakingCandidatesReader .. +type StakingCandidatesReader interface { + ValidatorInformation(addr common.Address) (*staking.Validator, error) + ValidatorStakingWithDelegation(addr common.Address) numeric.Dec + ValidatorCandidates() []common.Address +} + +// ChainReader is a subset of Engine.ChainReader, just enough to do assignment +type ChainReader interface { + // ReadShardState retrieves sharding state given the epoch number. + // This api reads the shard state cached or saved on the chaindb. + // Thus, only should be used to read the shard state of the current chain. + ReadShardState(epoch *big.Int) (shard.State, error) + // GetHeader retrieves a block header from the database by hash and number. + GetHeaderByHash(common.Hash) *block.Header + // Config retrieves the blockchain's chain configuration. + Config() *params.ChainConfig +} + +type partialStakingEnabled struct{} + +var ( + // WithStakingEnabled .. + WithStakingEnabled Reader = partialStakingEnabled{} +) + +func preStakingEnabledCommittee(s shardingconfig.Instance) shard.State { + shardNum := int(s.NumShards()) + shardHarmonyNodes := s.NumHarmonyOperatedNodesPerShard() + shardSize := s.NumNodesPerShard() + hmyAccounts := s.HmyAccounts() + fnAccounts := s.FnAccounts() + shardState := shard.State{} + for i := 0; i < shardNum; i++ { + com := shard.Committee{ShardID: uint32(i)} + for j := 0; j < shardHarmonyNodes; j++ { + index := i + j*shardNum // The initial account to use for genesis nodes + pub := &bls.PublicKey{} + pub.DeserializeHexStr(hmyAccounts[index].BlsPublicKey) + pubKey := shard.BlsPublicKey{} + pubKey.FromLibBLSPublicKey(pub) + // TODO: directly read address for bls too + curNodeID := shard.NodeID{ + common2.ParseAddr(hmyAccounts[index].Address), + pubKey, + nil, + } + com.NodeList = append(com.NodeList, curNodeID) + } + // add FN runner's key + for j := shardHarmonyNodes; j < shardSize; j++ { + index := i + (j-shardHarmonyNodes)*shardNum + pub := &bls.PublicKey{} + pub.DeserializeHexStr(fnAccounts[index].BlsPublicKey) + pubKey := shard.BlsPublicKey{} + pubKey.FromLibBLSPublicKey(pub) + // TODO: directly read address for bls too + curNodeID := shard.NodeID{ + common2.ParseAddr(fnAccounts[index].Address), + pubKey, + nil, + } + com.NodeList = append(com.NodeList, curNodeID) + } + shardState = append(shardState, com) + } + return shardState +} + +func with400Stakers( + s shardingconfig.Instance, stakerReader StakingCandidatesReader, +) (shard.State, error) { + // TODO Nervous about this because overtime the list will become quite large + candidates := stakerReader.ValidatorCandidates() + stakers := make([]*staking.Validator, len(candidates)) + for i := range candidates { + // TODO Should be using .ValidatorStakingWithDelegation, not implemented yet + validator, err := stakerReader.ValidatorInformation(candidates[i]) + if err != nil { + return nil, err + } + stakers[i] = validator + } + + sort.SliceStable( + stakers, + func(i, j int) bool { return stakers[i].Stake.Cmp(stakers[j].Stake) >= 0 }, + ) + const sCount = 401 + top := stakers[:sCount] + shardCount := int(s.NumShards()) + superComm := make(shard.State, shardCount) + fillCount := make([]int, shardCount) + // TODO Finish this logic, not correct, need to operate EPoS on slot level, + // not validator level + + for i := 0; i < shardCount; i++ { + superComm[i] = shard.Committee{} + superComm[i].NodeList = make(shard.NodeIDList, s.NumNodesPerShard()) + } + + scratchPad := &bls.PublicKey{} + + for i := range top { + spot := int(top[i].Address.Big().Int64()) % shardCount + fillCount[spot]++ + // scratchPad.DeserializeHexStr() + pubKey := shard.BlsPublicKey{} + pubKey.FromLibBLSPublicKey(scratchPad) + superComm[spot].NodeList = append( + superComm[spot].NodeList, + shard.NodeID{ + top[i].Address, + pubKey, + &shard.StakedMember{big.NewInt(0)}, + }, + ) + } + + utils.Logger().Info().Ints("distribution of Stakers in Shards", fillCount) + return superComm, nil +} + +func (def partialStakingEnabled) ReadPublicKeysFromDB( + h common.Hash, reader ChainReader, +) ([]*bls.PublicKey, error) { + header := reader.GetHeaderByHash(h) + shardID := header.ShardID() + superCommittee, err := reader.ReadShardState(header.Epoch()) + if err != nil { + return nil, err + } + subCommittee := superCommittee.FindCommitteeByID(shardID) + if subCommittee == nil { + return nil, ctxerror.New("cannot find shard in the shard state", + "blockNumber", header.Number(), + "shardID", header.ShardID(), + ) + } + committerKeys := []*bls.PublicKey{} + + for i := range subCommittee.NodeList { + committerKey := new(bls.PublicKey) + err := subCommittee.NodeList[i].BlsPublicKey.ToLibBLSPublicKey(committerKey) + if err != nil { + return nil, ctxerror.New("cannot convert BLS public key", + "blsPublicKey", subCommittee.NodeList[i].BlsPublicKey).WithCause(err) + } + committerKeys = append(committerKeys, committerKey) + } + return committerKeys, nil + + return nil, nil +} + +// ReadPublicKeysFromChain produces publicKeys of entire supercommittee per epoch, optionally providing a +// shard specific subcommittee +func (def partialStakingEnabled) ComputePublicKeys( + epoch *big.Int, reader ChainReader, shardID int, +) ([]*bls.PublicKey, []*bls.PublicKey) { + config := reader.Config() + instance := shard.Schedule.InstanceForEpoch(epoch) + if !config.IsStaking(epoch) { + superComm := preStakingEnabledCommittee(instance) + spot := 0 + allIdentities := make([]*bls.PublicKey, int(instance.NumShards())*instance.NumNodesPerShard()) + for i := range superComm { + for j := range superComm[i].NodeList { + identity := &bls.PublicKey{} + superComm[i].NodeList[j].BlsPublicKey.ToLibBLSPublicKey(identity) + allIdentities[spot] = identity + spot++ + } + } + + if shardID == StateID { + return allIdentities, nil + } + + subCommittee := superComm.FindCommitteeByID(uint32(shardID)) + subCommitteeIdentities := make([]*bls.PublicKey, len(subCommittee.NodeList)) + spot = 0 + for i := range subCommittee.NodeList { + identity := &bls.PublicKey{} + subCommittee.NodeList[i].BlsPublicKey.ToLibBLSPublicKey(identity) + subCommitteeIdentities[spot] = identity + spot++ + } + + return allIdentities, subCommitteeIdentities + } + // TODO Implement for the staked case + return nil, nil +} + +func (def partialStakingEnabled) ReadFromChain( + epoch *big.Int, reader ChainReader, +) (newSuperComm shard.State, err error) { + return reader.ReadShardState(epoch) +} + +// ReadFromComputation is single entry point for reading the State of the network +func (def partialStakingEnabled) ReadFromComputation( + epoch *big.Int, config params.ChainConfig, stakerReader StakingCandidatesReader, +) (newSuperComm shard.State, err error) { + instance := shard.Schedule.InstanceForEpoch(epoch) + if !config.IsStaking(epoch) { + return preStakingEnabledCommittee(instance), nil + } + return with400Stakers(instance, stakerReader) +} diff --git a/shard/shard_state.go b/shard/shard_state.go index d05cb1add..c868c702e 100644 --- a/shard/shard_state.go +++ b/shard/shard_state.go @@ -3,6 +3,8 @@ package shard import ( "bytes" "encoding/hex" + "encoding/json" + "math/big" "sort" "github.com/ethereum/go-ethereum/common" @@ -16,15 +18,71 @@ var ( emptyBlsPubKey = BlsPublicKey{} ) +// PublicKeySizeInBytes .. +const PublicKeySizeInBytes = 48 + // EpochShardState is the shard state of an epoch type EpochShardState struct { Epoch uint64 ShardState State } +// StakedMember is a committee member with stake +type StakedMember struct { + // nil means not active, 0 means our node, >= 0 means staked node + WithDelegationApplied *big.Int `json:"with-delegation-applied,omitempty"` +} + // State is the collection of all committees type State []Committee +// BlsPublicKey defines the bls public key +type BlsPublicKey [PublicKeySizeInBytes]byte + +// NodeID represents node id (BLS address) +type NodeID struct { + EcdsaAddress common.Address `json:"ecdsa_address"` + BlsPublicKey BlsPublicKey `json:"bls_pubkey"` + Validator *StakedMember `json:"staked-validator,omitempty" rlp:"nil"` +} + +// NodeIDList is a list of NodeIDList. +type NodeIDList []NodeID + +// Committee contains the active nodes in one shard +type Committee struct { + ShardID uint32 `json:"shard_id"` + NodeList NodeIDList `json:"node_list"` +} + +// JSON produces a non-pretty printed JSON string of the SuperCommittee +func (ss State) JSON() string { + type V struct { + ECDSAAddress common.Address `json:"ecdsa_address"` + BLSPublicKey string `json:"bls-public-key"` + } + + type T struct { + ShardID uint32 `json:"shard_id"` + Total int `json:"count"` + NodeList []V `json:"entries"` + } + t := []T{} + for i := range ss { + sub := ss[i] + subList := []V{} + for j := range sub.NodeList { + subList = append(subList, V{ + sub.NodeList[j].EcdsaAddress, + sub.NodeList[j].BlsPublicKey.Hex(), + }) + } + t = append(t, T{sub.ShardID, len(sub.NodeList), subList}) + } + buf, _ := json.Marshal(t) + return string(buf) +} + // FindCommitteeByID returns the committee configuration for the given shard, // or nil if the given shard is not found. func (ss State) FindCommitteeByID(shardID uint32) *Committee { @@ -65,9 +123,6 @@ func CompareShardState(s1, s2 State) int { return 0 } -// BlsPublicKey defines the bls public key -type BlsPublicKey [48]byte - // IsEmpty returns whether the bls public key is empty 0 bytes func (pk BlsPublicKey) IsEmpty() bool { return bytes.Compare(pk[:], emptyBlsPubKey[:]) == 0 @@ -100,12 +155,6 @@ func CompareBlsPublicKey(k1, k2 BlsPublicKey) int { return bytes.Compare(k1[:], k2[:]) } -// NodeID represents node id (BLS address) -type NodeID struct { - EcdsaAddress common.Address `json:"ecdsa_address"` - BlsPublicKey BlsPublicKey `json:"bls_pubkey"` -} - // CompareNodeID compares two node IDs. func CompareNodeID(id1, id2 *NodeID) int { if c := bytes.Compare(id1.EcdsaAddress[:], id2.EcdsaAddress[:]); c != 0 { @@ -117,9 +166,6 @@ func CompareNodeID(id1, id2 *NodeID) int { return 0 } -// NodeIDList is a list of NodeIDList. -type NodeIDList []NodeID - // DeepCopy returns a deep copy of the receiver. func (l NodeIDList) DeepCopy() NodeIDList { return append(l[:0:0], l...) @@ -145,12 +191,6 @@ func CompareNodeIDList(l1, l2 NodeIDList) int { return 0 } -// Committee contains the active nodes in one shard -type Committee struct { - ShardID uint32 `json:"shard_id"` - NodeList NodeIDList `json:"node_list"` -} - // DeepCopy returns a deep copy of the receiver. func (c Committee) DeepCopy() Committee { r := Committee{} diff --git a/shard/shard_state_test.go b/shard/shard_state_test.go index 707b1a335..12ce1784d 100644 --- a/shard/shard_state_test.go +++ b/shard/shard_state_test.go @@ -31,14 +31,14 @@ func init() { func TestGetHashFromNodeList(t *testing.T) { l1 := []NodeID{ - {common.Address{0x11}, blsPubKey1}, - {common.Address{0x22}, blsPubKey2}, - {common.Address{0x33}, blsPubKey3}, + {common.Address{0x11}, blsPubKey1, nil}, + {common.Address{0x22}, blsPubKey2, nil}, + {common.Address{0x33}, blsPubKey3, nil}, } l2 := []NodeID{ - {common.Address{0x22}, blsPubKey2}, - {common.Address{0x11}, blsPubKey1}, - {common.Address{0x33}, blsPubKey3}, + {common.Address{0x22}, blsPubKey2, nil}, + {common.Address{0x11}, blsPubKey1, nil}, + {common.Address{0x33}, blsPubKey3, nil}, } h1 := GetHashFromNodeList(l1) h2 := GetHashFromNodeList(l2) @@ -52,17 +52,17 @@ func TestHash(t *testing.T) { com1 := Committee{ ShardID: 22, NodeList: []NodeID{ - {common.Address{0x12}, blsPubKey11}, - {common.Address{0x23}, blsPubKey22}, - {common.Address{0x11}, blsPubKey1}, + {common.Address{0x12}, blsPubKey11, nil}, + {common.Address{0x23}, blsPubKey22, nil}, + {common.Address{0x11}, blsPubKey1, nil}, }, } com2 := Committee{ ShardID: 2, NodeList: []NodeID{ - {common.Address{0x44}, blsPubKey4}, - {common.Address{0x55}, blsPubKey5}, - {common.Address{0x66}, blsPubKey6}, + {common.Address{0x44}, blsPubKey4, nil}, + {common.Address{0x55}, blsPubKey5, nil}, + {common.Address{0x66}, blsPubKey6, nil}, }, } shardState1 := State{com1, com2} @@ -71,17 +71,17 @@ func TestHash(t *testing.T) { com3 := Committee{ ShardID: 2, NodeList: []NodeID{ - {common.Address{0x44}, blsPubKey4}, - {common.Address{0x55}, blsPubKey5}, - {common.Address{0x66}, blsPubKey6}, + {common.Address{0x44}, blsPubKey4, nil}, + {common.Address{0x55}, blsPubKey5, nil}, + {common.Address{0x66}, blsPubKey6, nil}, }, } com4 := Committee{ ShardID: 22, NodeList: []NodeID{ - {common.Address{0x12}, blsPubKey11}, - {common.Address{0x23}, blsPubKey22}, - {common.Address{0x11}, blsPubKey1}, + {common.Address{0x12}, blsPubKey11, nil}, + {common.Address{0x23}, blsPubKey22, nil}, + {common.Address{0x11}, blsPubKey1, nil}, }, } diff --git a/shard/values.go b/shard/values.go new file mode 100644 index 000000000..f925bea90 --- /dev/null +++ b/shard/values.go @@ -0,0 +1,19 @@ +package shard + +import ( + shardingconfig "github.com/harmony-one/harmony/internal/configs/sharding" +) + +const ( + // BeaconChainShardID is the ShardID of the BeaconChain + BeaconChainShardID = 0 +) + +// TODO ek – Schedule should really be part of a general-purpose network +// configuration. We are OK for the time being, +// until the day we should let one node process join multiple networks. +var ( + // Schedule is the sharding configuration schedule. + // Depends on the type of the network. Defaults to the mainnet schedule. + Schedule shardingconfig.Schedule = shardingconfig.MainnetSchedule +) From 55c9386e4c5588f68376ecb5b232e492c530a9b4 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sun, 10 Nov 2019 21:18:15 -0800 Subject: [PATCH 08/10] [quorum] Staked quorum member (#1819) * [quorum] Factor out single vote & provide for staked quorum vote * [quorum] Pass ShardID to decider via cb * [quorum] Move ShardIDProvider higher, fix nil mistake * [quorum] ReadAllSignatures optimization * [quorum] Safer way to read over map, then to slice * [quorum] Address PR comments - naming changes --- cmd/harmony/main.go | 3 + consensus/consensus_v2.go | 10 +-- consensus/quorum/one-node-one-vote.go | 78 ++++++++++++++++++ consensus/quorum/one-node-staked-vote.go | 75 +++++++++++++++++ consensus/quorum/quorum.go | 100 +++++++++++++---------- consensus/view_change.go | 12 +-- core/blockchain.go | 4 +- internal/chain/engine.go | 4 +- node/node_explorer.go | 2 +- node/node_genesis.go | 4 +- node/node_newblock.go | 4 +- node/node_resharding.go | 2 +- node/worker/worker.go | 2 +- shard/committee/assignment.go | 14 ++-- 14 files changed, 242 insertions(+), 72 deletions(-) create mode 100644 consensus/quorum/one-node-one-vote.go create mode 100644 consensus/quorum/one-node-staked-vote.go diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index efe440d8e..2cb582bcf 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -288,6 +288,9 @@ func setupConsensusAndNode(nodeConfig *nodeconfig.ConfigType) *node.Node { currentConsensus, err := consensus.New( myHost, nodeConfig.ShardID, p2p.Peer{}, nodeConfig.ConsensusPriKey, decider, ) + currentConsensus.Decider.SetShardIDProvider(func() (uint32, error) { + return currentConsensus.ShardID, nil + }) currentConsensus.SelfAddress = common.ParseAddr(initialAccount.Address) if err != nil { diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index cc917476c..6d68c905e 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -385,7 +385,7 @@ func (consensus *Consensus) onPrepare(msg *msg_pb.Message) { } logger = logger.With(). - Int64("NumReceivedSoFar", consensus.Decider.SignatoriesCount(quorum.Prepare)). + Int64("NumReceivedSoFar", consensus.Decider.SignersCount(quorum.Prepare)). Int64("PublicKeys", consensus.Decider.ParticipantsCount()).Logger() logger.Info().Msg("[OnPrepare] Received New Prepare Signature") consensus.Decider.AddSignature(quorum.Prepare, validatorPubKey, &sign) @@ -497,7 +497,7 @@ func (consensus *Consensus) onPrepared(msg *msg_pb.Message) { utils.Logger().Error().Err(err).Msg("ReadSignatureBitmapPayload failed!!") return } - prepareCount := consensus.Decider.SignatoriesCount(quorum.Prepare) + prepareCount := consensus.Decider.SignersCount(quorum.Prepare) if count := utils.CountOneBits(mask.Bitmap); count < prepareCount { utils.Logger().Debug(). Int64("Need", prepareCount). @@ -729,7 +729,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { } logger = logger.With(). - Int64("numReceivedSoFar", consensus.Decider.SignatoriesCount(quorum.Commit)). + Int64("numReceivedSoFar", consensus.Decider.SignersCount(quorum.Commit)). Logger() logger.Info().Msg("[OnCommit] Received new commit message") consensus.Decider.AddSignature(quorum.Commit, validatorPubKey, &sign) @@ -761,7 +761,7 @@ func (consensus *Consensus) onCommit(msg *msg_pb.Message) { func (consensus *Consensus) finalizeCommits() { utils.Logger().Info(). - Int64("NumCommits", consensus.Decider.SignatoriesCount(quorum.Commit)). + Int64("NumCommits", consensus.Decider.SignersCount(quorum.Commit)). Msg("[Finalizing] Finalizing Block") beforeCatchupNum := consensus.blockNum @@ -885,7 +885,7 @@ func (consensus *Consensus) onCommitted(msg *msg_pb.Message) { switch consensus.Decider.Policy() { case quorum.SuperMajorityVote: - threshold := consensus.Decider.QuorumThreshold() + threshold := consensus.Decider.QuorumThreshold().Int64() if count := utils.CountOneBits(mask.Bitmap); int64(count) < threshold { utils.Logger().Warn(). Int64("need", threshold). diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go new file mode 100644 index 000000000..23a9867ef --- /dev/null +++ b/consensus/quorum/one-node-one-vote.go @@ -0,0 +1,78 @@ +package quorum + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/bls/ffi/go/bls" + common2 "github.com/harmony-one/harmony/internal/common" + "github.com/harmony-one/harmony/internal/utils" + // "github.com/harmony-one/harmony/staking/effective" +) + +type uniformVoteWeight struct { + SignatureReader + DependencyInjectionWriter +} + +// Policy .. +func (v *uniformVoteWeight) Policy() Policy { + return SuperMajorityVote +} + +// func (v *uniformVoteWeight) SetShardIDProvider(p func() (uint32, error)) { +// v.p = p +// } + +// IsQuorumAchieved .. +func (v *uniformVoteWeight) IsQuorumAchieved(p Phase) bool { + r := v.SignersCount(p) >= v.QuorumThreshold().Int64() + utils.Logger().Info().Str("phase", p.String()). + Int64("signers-count", v.SignersCount(p)). + Int64("threshold", v.QuorumThreshold().Int64()). + Int64("participants", v.ParticipantsCount()). + Msg("Quorum details") + return r +} + +// QuorumThreshold .. +func (v *uniformVoteWeight) QuorumThreshold() *big.Int { + return big.NewInt(v.ParticipantsCount()*2/3 + 1) +} + +// RewardThreshold .. +func (v *uniformVoteWeight) IsRewardThresholdAchieved() bool { + return v.SignersCount(Commit) >= (v.ParticipantsCount() * 9 / 10) +} + +// func (v *uniformVoteWeight) UpdateVotingPower(effective.StakeKeeper) { +// NO-OP do not add anything here +// } + +// ToggleActive for uniform vote is a no-op, always says that voter is active +func (v *uniformVoteWeight) ToggleActive(*bls.PublicKey) bool { + // NO-OP do not add anything here + return true +} + +// Award .. +func (v *uniformVoteWeight) Award( + // Here hook is the callback which gets the amount the earner is due in just reward + // up to the hook to do side-effects like write the statedb + Pie *big.Int, earners []common2.Address, hook func(earner common.Address, due *big.Int), +) *big.Int { + payout := big.NewInt(0) + last := big.NewInt(0) + count := big.NewInt(int64(len(earners))) + + for i, account := range earners { + cur := big.NewInt(0) + cur.Mul(Pie, big.NewInt(int64(i+1))).Div(cur, count) + diff := big.NewInt(0).Sub(cur, last) + hook(common.Address(account), diff) + payout = big.NewInt(0).Add(payout, diff) + last = cur + } + + return payout +} diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go new file mode 100644 index 000000000..6af84a7aa --- /dev/null +++ b/consensus/quorum/one-node-staked-vote.go @@ -0,0 +1,75 @@ +package quorum + +import ( + "math/big" + + "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/internal/common" + "github.com/harmony-one/harmony/numeric" + "github.com/harmony-one/harmony/shard" +) + +var ( + twoThirds = numeric.NewDec(2).QuoInt64(3).Int +) + +type stakedVoter struct { + isActive, isHarmonyNode bool + effective numeric.Dec +} + +type stakedVoteWeight struct { + SignatureReader + DependencyInjectionWriter + // EPOS based staking + validatorStakes map[[shard.PublicKeySizeInBytes]byte]stakedVoter + totalEffectiveStakedAmount *big.Int +} + +// Policy .. +func (v *stakedVoteWeight) Policy() Policy { + return SuperMajorityStake +} + +// We must maintain 2/3 quoroum, so whatever is 2/3 staked amount, +// we divide that out & you +// IsQuorumAchieved .. +func (v *stakedVoteWeight) IsQuorumAchieved(p Phase) bool { + // TODO Implement this logic + return true +} + +// QuorumThreshold .. +func (v *stakedVoteWeight) QuorumThreshold() *big.Int { + return new(big.Int).Mul(v.totalEffectiveStakedAmount, twoThirds) +} + +// RewardThreshold .. +func (v *stakedVoteWeight) IsRewardThresholdAchieved() bool { + // TODO Implement + return false +} + +// HACK +var ( + hSentinel = big.NewInt(0) + hEffectiveSentinel = numeric.ZeroDec() +) + +// Award .. +func (v *stakedVoteWeight) Award( + Pie *big.Int, earners []common.Address, hook func(earner common.Address, due *big.Int), +) *big.Int { + // TODO Implement + return nil +} + +// UpdateVotingPower called only at epoch change, prob need to move to CalculateShardState +// func (v *stakedVoteWeight) UpdateVotingPower(keeper effective.StakeKeeper) { +// TODO Implement +// } + +func (v *stakedVoteWeight) ToggleActive(*bls.PublicKey) bool { + // TODO Implement + return true +} diff --git a/consensus/quorum/quorum.go b/consensus/quorum/quorum.go index 6afcf4fc3..61d91cfc6 100644 --- a/consensus/quorum/quorum.go +++ b/consensus/quorum/quorum.go @@ -1,7 +1,12 @@ package quorum import ( + "fmt" + "math/big" + "github.com/harmony-one/bls/ffi/go/bls" + "github.com/harmony-one/harmony/shard" + // "github.com/harmony-one/harmony/staking/effective" ) // Phase is a phase that needs quorum to proceed @@ -16,6 +21,19 @@ const ( ViewChange ) +var phaseNames = map[Phase]string{ + Prepare: "Announce", + Commit: "Prepare", + ViewChange: "Commit", +} + +func (p Phase) String() string { + if name, ok := phaseNames[p]; ok { + return name + } + return fmt.Sprintf("Unknown Quorum Phase %+v", byte(p)) +} + // Policy is the rule we used to decide is quorum achieved type Policy byte @@ -41,7 +59,7 @@ type SignatoryTracker interface { ParticipantTracker AddSignature(p Phase, PubKey *bls.PublicKey, sig *bls.Sign) // Caller assumes concurrency protection - SignatoriesCount(Phase) int64 + SignersCount(Phase) int64 Reset([]Phase) } @@ -52,6 +70,23 @@ type SignatureReader interface { ReadSignature(p Phase, PubKey *bls.PublicKey) *bls.Sign } +// DependencyInjectionWriter .. +type DependencyInjectionWriter interface { + SetShardIDProvider(func() (uint32, error)) +} + +// Decider .. +type Decider interface { + SignatureReader + DependencyInjectionWriter + ToggleActive(*bls.PublicKey) bool + // UpdateVotingPower(keeper effective.StakeKeeper) + Policy() Policy + IsQuorumAchieved(Phase) bool + QuorumThreshold() *big.Int + IsRewardThresholdAchieved() bool +} + // These maps represent the signatories (validators), keys are BLS public keys // and values are BLS private key signed signatures type cIdentities struct { @@ -64,6 +99,14 @@ type cIdentities struct { viewID map[string]*bls.Sign } +type depInject struct { + shardIDProvider func() (uint32, error) +} + +func (d *depInject) SetShardIDProvider(p func() (uint32, error)) { + d.shardIDProvider = p +} + func (s *cIdentities) IndexOf(pubKey *bls.PublicKey) int { idx := -1 for k, v := range s.publicKeys { @@ -104,7 +147,7 @@ func (s *cIdentities) ParticipantsCount() int64 { return int64(len(s.publicKeys)) } -func (s *cIdentities) SignatoriesCount(p Phase) int64 { +func (s *cIdentities) SignersCount(p Phase) int64 { switch p { case Prepare: return int64(len(s.prepare)) @@ -164,9 +207,7 @@ func (s *cIdentities) ReadSignature(p Phase, PubKey *bls.PublicKey) *bls.Sign { } func (s *cIdentities) ReadAllSignatures(p Phase) []*bls.Sign { - sigs := []*bls.Sign{} m := map[string]*bls.Sign{} - switch p { case Prepare: m = s.prepare @@ -175,9 +216,9 @@ func (s *cIdentities) ReadAllSignatures(p Phase) []*bls.Sign { case ViewChange: m = s.viewID } - - for _, sig := range m { - sigs = append(sigs, sig) + sigs := make([]*bls.Sign, 0, len(m)) + for _, value := range m { + sigs = append(sigs, value) } return sigs } @@ -189,47 +230,22 @@ func newMapBackedSignatureReader() SignatureReader { } } -// Decider .. -type Decider interface { - SignatureReader - Policy() Policy - IsQuorumAchieved(Phase) bool - QuorumThreshold() int64 - IsRewardThresholdAchieved() bool -} - -type uniformVoteWeight struct { - SignatureReader -} - // NewDecider .. func NewDecider(p Policy) Decider { + signatureStore := newMapBackedSignatureReader() + dependencies := &depInject{} switch p { case SuperMajorityVote: - return &uniformVoteWeight{newMapBackedSignatureReader()} - // case SuperMajorityStake: + return &uniformVoteWeight{signatureStore, dependencies} + case SuperMajorityStake: + return &stakedVoteWeight{ + signatureStore, + dependencies, + map[[shard.PublicKeySizeInBytes]byte]stakedVoter{}, + big.NewInt(0), + } default: // Should not be possible return nil } } - -// Policy .. -func (v *uniformVoteWeight) Policy() Policy { - return SuperMajorityVote -} - -// IsQuorumAchieved .. -func (v *uniformVoteWeight) IsQuorumAchieved(p Phase) bool { - return v.SignatoriesCount(p) >= v.QuorumThreshold() -} - -// QuorumThreshold .. -func (v *uniformVoteWeight) QuorumThreshold() int64 { - return v.ParticipantsCount()*2/3 + 1 -} - -// RewardThreshold .. -func (v *uniformVoteWeight) IsRewardThresholdAchieved() bool { - return v.SignatoriesCount(Commit) >= (v.ParticipantsCount() * 9 / 10) -} diff --git a/consensus/view_change.go b/consensus/view_change.go index db2aa8e91..b785f3417 100644 --- a/consensus/view_change.go +++ b/consensus/view_change.go @@ -157,8 +157,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { if consensus.Decider.IsQuorumAchieved(quorum.ViewChange) { utils.Logger().Debug(). - Int64("have", consensus.Decider.SignatoriesCount(quorum.ViewChange)). - Int64("need", consensus.Decider.QuorumThreshold()). + Int64("have", consensus.Decider.SignersCount(quorum.ViewChange)). + Int64("need", consensus.Decider.QuorumThreshold().Int64()). Str("validatorPubKey", recvMsg.SenderPubkey.SerializeToHexStr()). Msg("[onViewChange] Received Enough View Change Messages") return @@ -282,7 +282,7 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { return } // check has 2f+1 signature in m1 type message - need := consensus.Decider.QuorumThreshold() + need := consensus.Decider.QuorumThreshold().Int64() if count := utils.CountOneBits(mask.Bitmap); count < need { utils.Logger().Debug().Int64("need", need).Int64("have", count). Msg("[onViewChange] M1 Payload Not Have Enough Signature") @@ -345,8 +345,8 @@ func (consensus *Consensus) onViewChange(msg *msg_pb.Message) { // Set the bitmap indicating that this validator signed. consensus.viewIDBitmap.SetKey(recvMsg.SenderPubkey, true) utils.Logger().Debug(). - Int64("numSigs", consensus.Decider.SignatoriesCount(quorum.ViewChange)). - Int64("needed", consensus.Decider.QuorumThreshold()). + Int64("numSigs", consensus.Decider.SignersCount(quorum.ViewChange)). + Int64("needed", consensus.Decider.QuorumThreshold().Int64()). Msg("[onViewChange]") // received enough view change messages, change state to normal consensus @@ -446,7 +446,7 @@ func (consensus *Consensus) onNewView(msg *msg_pb.Message) { viewIDBytes := make([]byte, 8) binary.LittleEndian.PutUint64(viewIDBytes, recvMsg.ViewID) // check total number of sigs >= 2f+1 - need := consensus.Decider.QuorumThreshold() + need := consensus.Decider.QuorumThreshold().Int64() if count := utils.CountOneBits(m3Mask.Bitmap); count < need { utils.Logger().Debug().Int64("need", need).Int64("have", count). Msg("[onNewView] Not Have Enough M3 (ViewID) Signature") diff --git a/core/blockchain.go b/core/blockchain.go index 33fb29f92..1e5f3a9cb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1935,12 +1935,12 @@ func (bc *BlockChain) GetShardState(epoch *big.Int) (shard.State, error) { } if epoch.Cmp(big.NewInt(GenesisEpoch)) == 0 { - shardState, err = committee.WithStakingEnabled.ReadFromComputation( + shardState, err = committee.WithStakingEnabled.Compute( big.NewInt(GenesisEpoch), *bc.Config(), nil, ) } else { prevEpoch := new(big.Int).Sub(epoch, common.Big1) - shardState, err = committee.WithStakingEnabled.ReadFromChain( + shardState, err = committee.WithStakingEnabled.ReadFromDB( prevEpoch, bc, ) } diff --git a/internal/chain/engine.go b/internal/chain/engine.go index e927e6a65..922987dea 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -167,7 +167,7 @@ func (e *engineImpl) Finalize( func QuorumForBlock(chain engine.ChainReader, h *block.Header, reCalculate bool) (quorum int, err error) { var ss shard.State if reCalculate { - ss, _ = committee.WithStakingEnabled.ReadFromComputation(h.Epoch(), *chain.Config(), nil) + ss, _ = committee.WithStakingEnabled.Compute(h.Epoch(), *chain.Config(), nil) } else { ss, err = chain.ReadShardState(h.Epoch()) if err != nil { @@ -226,7 +226,7 @@ func GetPublicKeys(chain engine.ChainReader, header *block.Header, reCalculate b var shardState shard.State var err error if reCalculate { - shardState, _ = committee.WithStakingEnabled.ReadFromComputation(header.Epoch(), *chain.Config(), nil) + shardState, _ = committee.WithStakingEnabled.Compute(header.Epoch(), *chain.Config(), nil) } else { shardState, err = chain.ReadShardState(header.Epoch()) if err != nil { diff --git a/node/node_explorer.go b/node/node_explorer.go index 70fa2fe6e..3919c3890 100644 --- a/node/node_explorer.go +++ b/node/node_explorer.go @@ -50,7 +50,7 @@ func (node *Node) ExplorerMessageHandler(payload []byte) { } // check has 2f+1 signatures - need := node.Consensus.Decider.QuorumThreshold() + need := node.Consensus.Decider.QuorumThreshold().Int64() if count := utils.CountOneBits(mask.Bitmap); count < need { utils.Logger().Error().Int64("need", need).Int64("have", count). Msg("[Explorer] not have enough signature") diff --git a/node/node_genesis.go b/node/node_genesis.go index bc9740743..61400eb5b 100644 --- a/node/node_genesis.go +++ b/node/node_genesis.go @@ -8,10 +8,8 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - blockfactory "github.com/harmony-one/harmony/block/factory" "github.com/harmony-one/harmony/common/denominations" "github.com/harmony-one/harmony/core" @@ -42,7 +40,7 @@ type genesisInitializer struct { // InitChainDB sets up a new genesis block in the database for the given shard. func (gi *genesisInitializer) InitChainDB(db ethdb.Database, shardID uint32) error { - shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + shardState, _ := committee.WithStakingEnabled.Compute( big.NewInt(core.GenesisEpoch), gi.node.chainConfig, nil, ) if shardID != shard.BeaconChainShardID { diff --git a/node/node_newblock.go b/node/node_newblock.go index f6df9c1bd..515ec4212 100644 --- a/node/node_newblock.go +++ b/node/node_newblock.go @@ -127,7 +127,7 @@ func (node *Node) proposeShardStateWithoutBeaconSync(block *types.Block) shard.S if block == nil || !shard.Schedule.IsLastBlock(block.Number().Uint64()) { return nil } - shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + shardState, _ := committee.WithStakingEnabled.Compute( new(big.Int).Add(block.Header().Epoch(), common.Big1), node.chainConfig, nil, ) return shardState @@ -151,7 +151,7 @@ func (node *Node) proposeBeaconShardState(block *types.Block) error { } // TODO Use ReadFromComputation prevEpoch := new(big.Int).Sub(block.Header().Epoch(), common.Big1) - shardState, err := committee.WithStakingEnabled.ReadFromChain( + shardState, err := committee.WithStakingEnabled.ReadFromDB( prevEpoch, node.Blockchain(), ) if err != nil { diff --git a/node/node_resharding.go b/node/node_resharding.go index 0c55369d9..c15e5bf04 100644 --- a/node/node_resharding.go +++ b/node/node_resharding.go @@ -57,7 +57,7 @@ func (node *Node) validateNewShardState(block *types.Block) error { // TODO ek – this may be called from regular shards, // for vetting beacon chain blocks received during block syncing. // DRand may or or may not get in the way. Test this out. - expected, err := committee.WithStakingEnabled.ReadFromChain( + expected, err := committee.WithStakingEnabled.ReadFromDB( new(big.Int).Sub(block.Header().Epoch(), common.Big1), node.Beaconchain(), ) diff --git a/node/worker/worker.go b/node/worker/worker.go index 8fc3a2054..e3cd959c9 100644 --- a/node/worker/worker.go +++ b/node/worker/worker.go @@ -368,7 +368,7 @@ func (w *Worker) ProposeShardStateWithoutBeaconSync() shard.State { if !shard.Schedule.IsLastBlock(w.current.header.Number().Uint64()) { return nil } - shardState, _ := committee.WithStakingEnabled.ReadFromComputation( + shardState, _ := committee.WithStakingEnabled.Compute( new(big.Int).Add(w.current.header.Epoch(), common.Big1), *w.config, nil, ) return shardState diff --git a/shard/committee/assignment.go b/shard/committee/assignment.go index 69afb9259..11143dd27 100644 --- a/shard/committee/assignment.go +++ b/shard/committee/assignment.go @@ -21,12 +21,12 @@ import ( // a shardID parameter const StateID = -1 -// MembershipList .. -type MembershipList interface { - ReadFromComputation( +// ValidatorList .. +type ValidatorList interface { + Compute( epoch *big.Int, config params.ChainConfig, reader StakingCandidatesReader, ) (shard.State, error) - ReadFromChain(epoch *big.Int, reader ChainReader) (shard.State, error) + ReadFromDB(epoch *big.Int, reader ChainReader) (shard.State, error) } // PublicKeys per epoch @@ -45,7 +45,7 @@ type PublicKeys interface { // Reader .. type Reader interface { PublicKeys - MembershipList + ValidatorList } // StakingCandidatesReader .. @@ -243,14 +243,14 @@ func (def partialStakingEnabled) ComputePublicKeys( return nil, nil } -func (def partialStakingEnabled) ReadFromChain( +func (def partialStakingEnabled) ReadFromDB( epoch *big.Int, reader ChainReader, ) (newSuperComm shard.State, err error) { return reader.ReadShardState(epoch) } // ReadFromComputation is single entry point for reading the State of the network -func (def partialStakingEnabled) ReadFromComputation( +func (def partialStakingEnabled) Compute( epoch *big.Int, config params.ChainConfig, stakerReader StakingCandidatesReader, ) (newSuperComm shard.State, err error) { instance := shard.Schedule.InstanceForEpoch(epoch) From a8163205d13337833653941e010dbd8b033ed346 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sun, 10 Nov 2019 21:37:05 -0800 Subject: [PATCH 09/10] [reward] Abstract out block-reward, use rewarder, check result correct (#1820) * [reward] Factor out interface for block-reward * [reward] Use factored out block rewarder which is actually same object as quorum.Decider * [quorum] Fix type error because of silly internal/common.Address vs common.Address of ethereum * [testing] Somehow this port being in tandem with previously used one fixes a build timing issue --- consensus/consensus_leader_msg_test.go | 2 +- consensus/engine/consensus_engine.go | 7 ++++ consensus/quorum/one-node-one-vote.go | 7 +--- consensus/quorum/one-node-staked-vote.go | 2 +- consensus/reward/rewarder.go | 16 +++++++ internal/chain/engine.go | 19 +++++++-- internal/chain/reward.go | 53 ++++++++++++++++-------- node/node.go | 2 + 8 files changed, 80 insertions(+), 28 deletions(-) create mode 100644 consensus/reward/rewarder.go diff --git a/consensus/consensus_leader_msg_test.go b/consensus/consensus_leader_msg_test.go index 9abff5be1..46a496ecb 100644 --- a/consensus/consensus_leader_msg_test.go +++ b/consensus/consensus_leader_msg_test.go @@ -46,7 +46,7 @@ func TestConstructAnnounceMessage(test *testing.T) { func TestConstructPreparedMessage(test *testing.T) { leaderPriKey := bls.RandPrivateKey() leaderPubKey := leaderPriKey.GetPublicKey() - leader := p2p.Peer{IP: "127.0.0.1", Port: "6000", ConsensusPubKey: leaderPubKey} + leader := p2p.Peer{IP: "127.0.0.1", Port: "19999", ConsensusPubKey: leaderPubKey} validatorPriKey := bls.RandPrivateKey() validatorPubKey := leaderPriKey.GetPublicKey() diff --git a/consensus/engine/consensus_engine.go b/consensus/engine/consensus_engine.go index 03bbe93d9..88f6dd887 100644 --- a/consensus/engine/consensus_engine.go +++ b/consensus/engine/consensus_engine.go @@ -5,6 +5,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/harmony/block" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/params" @@ -73,6 +74,12 @@ type Engine interface { // rules of a particular engine. The changes are executed inline. Prepare(chain ChainReader, header *block.Header) error + // Rewarder handles the distribution of block rewards + Rewarder() reward.Distributor + + // SetRewarder assigns the Distributor used in block reward + SetRewarder(reward.Distributor) + // Finalize runs any post-transaction state modifications (e.g. block rewards) // and assembles the final block. // Note: The block header and state database might be updated to reflect any diff --git a/consensus/quorum/one-node-one-vote.go b/consensus/quorum/one-node-one-vote.go index 23a9867ef..8595a9c31 100644 --- a/consensus/quorum/one-node-one-vote.go +++ b/consensus/quorum/one-node-one-vote.go @@ -5,7 +5,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" - common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/utils" // "github.com/harmony-one/harmony/staking/effective" ) @@ -20,10 +19,6 @@ func (v *uniformVoteWeight) Policy() Policy { return SuperMajorityVote } -// func (v *uniformVoteWeight) SetShardIDProvider(p func() (uint32, error)) { -// v.p = p -// } - // IsQuorumAchieved .. func (v *uniformVoteWeight) IsQuorumAchieved(p Phase) bool { r := v.SignersCount(p) >= v.QuorumThreshold().Int64() @@ -59,7 +54,7 @@ func (v *uniformVoteWeight) ToggleActive(*bls.PublicKey) bool { func (v *uniformVoteWeight) Award( // Here hook is the callback which gets the amount the earner is due in just reward // up to the hook to do side-effects like write the statedb - Pie *big.Int, earners []common2.Address, hook func(earner common.Address, due *big.Int), + Pie *big.Int, earners []common.Address, hook func(earner common.Address, due *big.Int), ) *big.Int { payout := big.NewInt(0) last := big.NewInt(0) diff --git a/consensus/quorum/one-node-staked-vote.go b/consensus/quorum/one-node-staked-vote.go index 6af84a7aa..efd2b385b 100644 --- a/consensus/quorum/one-node-staked-vote.go +++ b/consensus/quorum/one-node-staked-vote.go @@ -3,8 +3,8 @@ package quorum import ( "math/big" + "github.com/ethereum/go-ethereum/common" "github.com/harmony-one/bls/ffi/go/bls" - "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/numeric" "github.com/harmony-one/harmony/shard" ) diff --git a/consensus/reward/rewarder.go b/consensus/reward/rewarder.go new file mode 100644 index 000000000..e13c91cb2 --- /dev/null +++ b/consensus/reward/rewarder.go @@ -0,0 +1,16 @@ +package reward + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +// Distributor .. +type Distributor interface { + Award( + Pie *big.Int, + earners []common.Address, + hook func(earner common.Address, due *big.Int), + ) (payout *big.Int) +} diff --git a/internal/chain/engine.go b/internal/chain/engine.go index 922987dea..20b96e432 100644 --- a/internal/chain/engine.go +++ b/internal/chain/engine.go @@ -8,6 +8,7 @@ import ( "github.com/harmony-one/bls/ffi/go/bls" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" "github.com/harmony-one/harmony/core/types" "github.com/harmony-one/harmony/internal/ctxerror" @@ -19,10 +20,22 @@ import ( "golang.org/x/crypto/sha3" ) -type engineImpl struct{} +type engineImpl struct { + d reward.Distributor +} // Engine is an algorithm-agnostic consensus engine. -var Engine = &engineImpl{} +var Engine = &engineImpl{nil} + +// Rewarder handles the distribution of block rewards +func (e *engineImpl) Rewarder() reward.Distributor { + return e.d +} + +// SetRewarder .. +func (e *engineImpl) SetRewarder(d reward.Distributor) { + e.d = d +} // SealHash returns the hash of a block prior to it being sealed. func (e *engineImpl) SealHash(header *block.Header) (hash common.Hash) { @@ -156,7 +169,7 @@ func (e *engineImpl) Finalize( incxs []*types.CXReceiptsProof, stks []*staking.StakingTransaction) (*types.Block, error) { // Accumulate any block and uncle rewards and commit the final state root // Header seems complete, assemble into a block and return - if err := AccumulateRewards(chain, state, header); err != nil { + if err := AccumulateRewards(chain, state, header, e.Rewarder()); err != nil { return nil, ctxerror.New("cannot pay block reward").WithCause(err) } header.SetRoot(state.IntermediateRoot(chain.Config().IsS3(header.Epoch()))) diff --git a/internal/chain/reward.go b/internal/chain/reward.go index 5ee12a212..c4f585161 100644 --- a/internal/chain/reward.go +++ b/internal/chain/reward.go @@ -8,21 +8,26 @@ import ( "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/common/denominations" "github.com/harmony-one/harmony/consensus/engine" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/core/state" bls2 "github.com/harmony-one/harmony/crypto/bls" common2 "github.com/harmony-one/harmony/internal/common" "github.com/harmony-one/harmony/internal/ctxerror" "github.com/harmony-one/harmony/internal/utils" + "github.com/pkg/errors" ) -// BlockReward is the block reward, to be split evenly among block signers. -var BlockReward = new(big.Int).Mul(big.NewInt(24), big.NewInt(denominations.One)) +var ( + // BlockReward is the block reward, to be split evenly among block signers. + BlockReward = new(big.Int).Mul(big.NewInt(24), big.NewInt(denominations.One)) + errPayoutNotEqualBlockReward = errors.New("total payout not equal to blockreward") +) // AccumulateRewards credits the coinbase of the given block with the mining // reward. The total reward consists of the static block reward and rewards for // included uncles. The coinbase of each uncle block is also rewarded. func AccumulateRewards( - bc engine.ChainReader, state *state.DB, header *block.Header, + bc engine.ChainReader, state *state.DB, header *block.Header, rewarder reward.Distributor, ) error { blockNum := header.Number().Uint64() if blockNum == 0 { @@ -72,9 +77,9 @@ func AccumulateRewards( if err := mask.SetMask(header.LastCommitBitmap()); err != nil { return ctxerror.New("cannot set group sig mask bits").WithCause(err) } - totalAmount := big.NewInt(0) - var accounts []common.Address - signers := []string{} + + accounts := []common.Address{} + for idx, member := range parentCommittee.NodeList { if signed, err := mask.IndexEnabled(idx); err != nil { return ctxerror.New("cannot check for committer bit", @@ -85,19 +90,33 @@ func AccumulateRewards( } } - numAccounts := big.NewInt(int64(len(accounts))) - last := new(big.Int) - for i, account := range accounts { - cur := new(big.Int) - cur.Mul(BlockReward, big.NewInt(int64(i+1))).Div(cur, numAccounts) - diff := new(big.Int).Sub(cur, last) - signers = append(signers, common2.MustAddressToBech32(account)) - state.AddBalance(account, diff) - totalAmount = new(big.Int).Add(totalAmount, diff) - last = cur + type t struct { + common.Address + *big.Int } + signers := []string{} + payable := []t{} + + totalAmount := rewarder.Award( + BlockReward, accounts, func(receipient common.Address, amount *big.Int) { + signers = append(signers, common2.MustAddressToBech32(receipient)) + payable = append(payable, t{receipient, amount}) + }) + + if totalAmount.Cmp(BlockReward) != 0 { + utils.Logger().Error(). + Int64("block-reward", BlockReward.Int64()). + Int64("total-amount-paid-out", totalAmount.Int64()). + Msg("Total paid out was not equal to block-reward") + return errors.Wrapf(errPayoutNotEqualBlockReward, "payout "+totalAmount.String()) + } + + for i := range payable { + state.AddBalance(payable[i].Address, payable[i].Int) + } + header.Logger(utils.Logger()).Debug(). - Str("NumAccounts", numAccounts.String()). + Int("NumAccounts", len(accounts)). Str("TotalAmount", totalAmount.String()). Strs("Signers", signers). Msg("[Block Reward] Successfully paid out block reward") diff --git a/node/node.go b/node/node.go index 8e97b23e0..5221ae2c0 100644 --- a/node/node.go +++ b/node/node.go @@ -17,6 +17,7 @@ import ( "github.com/harmony-one/harmony/api/service/syncing/downloader" "github.com/harmony-one/harmony/block" "github.com/harmony-one/harmony/consensus" + "github.com/harmony-one/harmony/consensus/reward" "github.com/harmony-one/harmony/contracts" "github.com/harmony-one/harmony/core" "github.com/harmony-one/harmony/core/types" @@ -514,6 +515,7 @@ func New(host p2p.Host, consensusObj *consensus.Consensus, chainDBFactory shardc node.pendingTransactions = make(map[common.Hash]*types.Transaction) node.pendingStakingTransactions = make(map[common.Hash]*staking.StakingTransaction) node.Consensus.VerifiedNewBlock = make(chan *types.Block) + chain.Engine.SetRewarder(node.Consensus.Decider.(reward.Distributor)) // the sequence number is the next block number to be added in consensus protocol, which is always one more than current chain header block node.Consensus.SetBlockNum(blockchain.CurrentBlock().NumberU64() + 1) From 381a7bd02da4ade46c698d3f757023b969a937d6 Mon Sep 17 00:00:00 2001 From: Edgar Aroutiounian Date: Sun, 10 Nov 2019 22:01:51 -0800 Subject: [PATCH 10/10] [project] Fix two leaking tickers (#1821) --- consensus/consensus_v2.go | 1 + node/node_handler.go | 1 + 2 files changed, 2 insertions(+) diff --git a/consensus/consensus_v2.go b/consensus/consensus_v2.go index 6d68c905e..591fb0dfe 100644 --- a/consensus/consensus_v2.go +++ b/consensus/consensus_v2.go @@ -1085,6 +1085,7 @@ func (consensus *Consensus) Start(blockChannel chan *types.Block, stopChan chan utils.Logger().Info().Time("time", time.Now()).Msg("[ConsensusMainLoop] Consensus started") defer close(stoppedChan) ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() consensus.consensusTimeout[timeoutBootstrap].Start() utils.Logger().Debug(). Uint64("viewID", consensus.viewID). diff --git a/node/node_handler.go b/node/node_handler.go index 947cbfd00..7d54d52ac 100644 --- a/node/node_handler.go +++ b/node/node_handler.go @@ -479,6 +479,7 @@ func (node *Node) pingMessageHandler(msgPayload []byte, sender libp2p_peer.ID) i // bootstrapConsensus is the a goroutine to check number of peers and start the consensus func (node *Node) bootstrapConsensus() { tick := time.NewTicker(5 * time.Second) + defer tick.Stop() lastPeerNum := node.numPeers for { select {