[node] graceful shutdown validators (#3483)

* [node] graceful shutdown validator and leader

when leader/validator is in commit phase, wait until it finishes the consensus
before shutdown the node.

This is useful for node upgrade to avoid unexpected blockchain state.

Signed-off-by: Leo Chen <leo@harmony.one>

* [node] stop block proposal service on leader

Signed-off-by: Leo Chen <leo@harmony.one>

* [node] warn on block proposal stopped

Signed-off-by: Leo Chen <leo@harmony.one>

Co-authored-by: Rongjian Lan <rongjian.lan@gmail.com>

cmd/harmony/main.go

@@ -19,6 +19,7 @@ import (
 	ethCommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/harmony-one/bls/ffi/go/bls"
+	"github.com/harmony-one/harmony/api/service"
 	"github.com/harmony-one/harmony/api/service/prometheus"
 	"github.com/harmony-one/harmony/api/service/syncing"
 	"github.com/harmony-one/harmony/common/fdlimit"
@@ -296,16 +297,30 @@ func setupNodeAndRun(hc harmonyConfig) {
 	// Prepare for graceful shutdown from os signals
 	osSignal := make(chan os.Signal)
 	signal.Notify(osSignal, os.Interrupt, syscall.SIGTERM)
-	go func() {
+	go func(node *node.Node) {
 		for sig := range osSignal {
 			if sig == syscall.SIGTERM || sig == os.Interrupt {
+				utils.Logger().Warn().Str("signal", sig.String()).Msg("Gracefully shutting down...")
 				const msg = "Got %s signal. Gracefully shutting down...\n"
-				utils.Logger().Printf(msg, sig)
-				fmt.Printf(msg, sig)
+				fmt.Fprintf(os.Stderr, msg, sig)
+				// stop block proposal service for leader
+				if node.Consensus.IsLeader() {
+					node.ServiceManager().StopService(service.BlockProposal)
+				}
+				if node.Consensus.Mode() == consensus.Normal {
+					phase := node.Consensus.GetConsensusPhase()
+					utils.Logger().Warn().Str("phase", phase).Msg("[shutdown] commit phase has to wait")
+					maxWait := time.Now().Add(2 * node.Consensus.BlockPeriod) // wait up to 2 * blockperiod in commit phase
+					for time.Now().Before(maxWait) &&
+						node.Consensus.GetConsensusPhase() == "Commit" {
+						utils.Logger().Warn().Msg("[shutdown] wait for consensus finished")
+						time.Sleep(time.Millisecond * 100)
+					}
+				}
 				currentNode.ShutDown()
 			}
 		}
-	}()
+	}(currentNode)
 
 	// Parse RPC config
 	nodeConfig.RPCServer = nodeconfig.RPCServerConfig{

node/node_newblock.go

@@ -40,7 +40,7 @@ func (node *Node) WaitForConsensusReadyV2(readySignal chan consensus.ProposalTyp
 			// keep waiting for Consensus ready
 			select {
 			case <-stopChan:
-				utils.Logger().Debug().
+				utils.Logger().Warn().
 					Msg("Consensus new block proposal: STOPPED!")
 				return
 			case proposalType := <-readySignal: