[node] graceful shutdown validators (#3483)

* [node] graceful shutdown validator and leader

when leader/validator is in commit phase, wait until it finishes the consensus
before shutdown the node.

This is useful for node upgrade to avoid unexpected blockchain state.

Signed-off-by: Leo Chen <leo@harmony.one>

* [node] stop block proposal service on leader

Signed-off-by: Leo Chen <leo@harmony.one>

* [node] warn on block proposal stopped

Signed-off-by: Leo Chen <leo@harmony.one>

Co-authored-by: Rongjian Lan <rongjian.lan@gmail.com>
pull/3487/head
Leo Chen 4 years ago committed by GitHub
parent 21b162cc67
commit b38118fcbb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 23
      cmd/harmony/main.go
  2. 2
      node/node_newblock.go

@ -19,6 +19,7 @@ import (
ethCommon "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/harmony-one/bls/ffi/go/bls"
"github.com/harmony-one/harmony/api/service"
"github.com/harmony-one/harmony/api/service/prometheus"
"github.com/harmony-one/harmony/api/service/syncing"
"github.com/harmony-one/harmony/common/fdlimit"
@ -296,16 +297,30 @@ func setupNodeAndRun(hc harmonyConfig) {
// Prepare for graceful shutdown from os signals
osSignal := make(chan os.Signal)
signal.Notify(osSignal, os.Interrupt, syscall.SIGTERM)
go func() {
go func(node *node.Node) {
for sig := range osSignal {
if sig == syscall.SIGTERM || sig == os.Interrupt {
utils.Logger().Warn().Str("signal", sig.String()).Msg("Gracefully shutting down...")
const msg = "Got %s signal. Gracefully shutting down...\n"
utils.Logger().Printf(msg, sig)
fmt.Printf(msg, sig)
fmt.Fprintf(os.Stderr, msg, sig)
// stop block proposal service for leader
if node.Consensus.IsLeader() {
node.ServiceManager().StopService(service.BlockProposal)
}
if node.Consensus.Mode() == consensus.Normal {
phase := node.Consensus.GetConsensusPhase()
utils.Logger().Warn().Str("phase", phase).Msg("[shutdown] commit phase has to wait")
maxWait := time.Now().Add(2 * node.Consensus.BlockPeriod) // wait up to 2 * blockperiod in commit phase
for time.Now().Before(maxWait) &&
node.Consensus.GetConsensusPhase() == "Commit" {
utils.Logger().Warn().Msg("[shutdown] wait for consensus finished")
time.Sleep(time.Millisecond * 100)
}
}
currentNode.ShutDown()
}
}
}()
}(currentNode)
// Parse RPC config
nodeConfig.RPCServer = nodeconfig.RPCServerConfig{

@ -40,7 +40,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp
// keep waiting for Consensus ready
select {
case <-stopChan:
utils.Logger().Debug().
utils.Logger().Warn().
Msg("Consensus new block proposal: STOPPED!")
return
case proposalType := <-readySignal:

Loading…
Cancel
Save