[sync] add a tolerance number for node-sync checking

Signed-off-by: Leo Chen <leo@harmony.one>

api/service/explorer/service.go

@@ -29,6 +29,7 @@ const (
 	explorerPortDifference = 4000
 	defaultPageSize        = "1000"
 	maxAddresses           = 100000
+	nodeSyncTolerance      = 5
 )
 
 // HTTPError is an HTTP error.
@@ -270,12 +271,19 @@ func (s *Service) GetTotalSupply(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// GetNodeSync returns status code 500 if node is not in sync
+// GetNodeSync returns status code Teapot 418 if node is not in sync
 func (s *Service) GetNodeSync(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
-	sync, _ := s.backend.SyncStatus(s.blockchain.ShardID())
+	sync, remote, diff := s.backend.SyncStatus(s.blockchain.ShardID())
 	if !sync {
-		w.WriteHeader(http.StatusTeapot)
+		utils.Logger().Debug().Uint64("remote", remote).Uint64("diff", diff).Msg("GetNodeSyncStatus")
+		if remote == 0 || diff > nodeSyncTolerance {
+			w.WriteHeader(http.StatusTeapot)
+		} else {
+			// return sync'ed if the diff is less than nodeSyncTolerance
+			// this tolerance is only applicable to /node-sync API call
+			sync = true
+		}
 	}
 	if err := json.NewEncoder(w).Encode(sync); err != nil {
 		utils.Logger().Warn().Msg("cannot JSON-encode total supply")

api/service/legacysync/syncing.go

@@ -1054,16 +1054,17 @@ func (ss *StateSync) GetMaxPeerHeight() uint64 {
 	return ss.getMaxPeerHeight(false)
 }
 
-// SyncStatus returns inSync and remote height
-func (ss *StateSync) SyncStatus(bc *core.BlockChain) (bool, uint64) {
-	outOfSync, remoteNum := ss.IsOutOfSync(bc, false)
-	return !outOfSync, remoteNum
+// SyncStatus returns inSync, remote height, and difference between remote height and local height
+func (ss *StateSync) SyncStatus(bc *core.BlockChain) (bool, uint64, uint64) {
+	outOfSync, remoteHeight, heightDiff := ss.IsOutOfSync(bc, false)
+	return !outOfSync, remoteHeight, heightDiff
 }
 
 // IsOutOfSync checks whether the node is out of sync from other peers
-func (ss *StateSync) IsOutOfSync(bc *core.BlockChain, doubleCheck bool) (bool, uint64) {
+// It returns the sync status, remote height, and the difference betwen local blockchain and remote max height
+func (ss *StateSync) IsOutOfSync(bc *core.BlockChain, doubleCheck bool) (bool, uint64, uint64) {
 	if ss.syncConfig == nil {
-		return true, 0 // If syncConfig is not instantiated, return not in sync
+		return true, 0, 0 // If syncConfig is not instantiated, return not in sync
 	}
 	otherHeight1 := ss.getMaxPeerHeight(false)
 	lastHeight := bc.CurrentBlock().NumberU64()
@@ -1074,7 +1075,7 @@ func (ss *StateSync) IsOutOfSync(bc *core.BlockChain, doubleCheck bool) (bool, u
 			Uint64("OtherHeight", otherHeight1).
 			Uint64("lastHeight", lastHeight).
 			Msg("[SYNC] Checking sync status")
-		return wasOutOfSync, otherHeight1
+		return wasOutOfSync, otherHeight1, otherHeight1 - lastHeight
 	}
 	time.Sleep(1 * time.Second)
 	// double check the sync status after 1 second to confirm (avoid false alarm)
@@ -1090,7 +1091,7 @@ func (ss *StateSync) IsOutOfSync(bc *core.BlockChain, doubleCheck bool) (bool, u
 		Uint64("currentHeight", currentHeight).
 		Msg("[SYNC] Checking sync status")
 	// Only confirm out of sync when the node has lower height and didn't move in heights for 2 consecutive checks
-	return wasOutOfSync && isOutOfSync && lastHeight == currentHeight, otherHeight2
+	return wasOutOfSync && isOutOfSync && lastHeight == currentHeight, otherHeight2, otherHeight2 - currentHeight
 }
 
 // SyncLoop will keep syncing with peers until catches up

hmy/downloader/downloader.go

@@ -117,12 +117,12 @@ func (d *Downloader) NumPeers() int {
 }
 
 // IsSyncing return the current sync status
-func (d *Downloader) SyncStatus() (bool, uint64) {
+func (d *Downloader) SyncStatus() (bool, uint64, uint64) {
 	syncing, target := d.status.get()
 	if !syncing {
 		target = d.bc.CurrentBlock().NumberU64()
 	}
-	return syncing, target
+	return syncing, target, 0
 }
 
 // SubscribeDownloadStarted subscribe download started

hmy/downloader/downloaders.go

@@ -83,10 +83,10 @@ func (ds *Downloaders) NumPeers() map[uint32]int {
 
 // SyncStatus returns whether the given shard is doing syncing task and the target block
 // number.
-func (ds *Downloaders) SyncStatus(shardID uint32) (bool, uint64) {
+func (ds *Downloaders) SyncStatus(shardID uint32) (bool, uint64, uint64) {
 	d, ok := ds.ds[shardID]
 	if !ok {
-		return false, 0
+		return false, 0, 0
 	}
 	return d.SyncStatus()
 }

hmy/hmy.go

@@ -88,7 +88,7 @@ type NodeAPI interface {
 	GetStakingTransactionsCount(address, txType string) (uint64, error)
 	IsCurrentlyLeader() bool
 	IsOutOfSync(shardID uint32) bool
-	SyncStatus(shardID uint32) (bool, uint64)
+	SyncStatus(shardID uint32) (bool, uint64, uint64)
 	SyncPeers() map[string]int
 	ReportStakingErrorSink() types.TransactionErrorReports
 	ReportPlainErrorSink() types.TransactionErrorReports

node/node_syncing.go

@@ -302,7 +302,7 @@ func (node *Node) doSync(bc *core.BlockChain, worker *worker.Worker, willJoinCon
 		utils.Logger().Debug().Int("len", node.stateSync.GetActivePeerNumber()).Msg("[SYNC] Get Active Peers")
 	}
 	// TODO: treat fake maximum height
-	if outOfSync, _ := node.stateSync.IsOutOfSync(bc, true); outOfSync {
+	if outOfSync, _, _ := node.stateSync.IsOutOfSync(bc, true); outOfSync {
 		node.IsInSync.UnSet()
 		if willJoinConsensus {
 			node.Consensus.BlocksNotSynchronized()
@@ -673,8 +673,9 @@ func (node *Node) getCommitSigFromDB(block *types.Block) ([]byte, error) {
 }
 
 // SyncStatus return the syncing status, including whether node is syncing
-// and the target block number.
-func (node *Node) SyncStatus(shardID uint32) (bool, uint64) {
+// and the target block number, and the difference between current block
+// and target block.
+func (node *Node) SyncStatus(shardID uint32) (bool, uint64, uint64) {
 	ds := node.getDownloaders()
 	if ds == nil || !ds.IsActive() {
 		// downloaders inactive. Ask DNS sync instead
@@ -683,23 +684,23 @@ func (node *Node) SyncStatus(shardID uint32) (bool, uint64) {
 	return ds.SyncStatus(shardID)
 }
 
-func (node *Node) legacySyncStatus(shardID uint32) (bool, uint64) {
+func (node *Node) legacySyncStatus(shardID uint32) (bool, uint64, uint64) {
 	switch shardID {
 	case node.NodeConfig.ShardID:
 		if node.stateSync == nil {
-			return false, 0
+			return false, 0, 0
 		}
 		return node.stateSync.SyncStatus(node.Blockchain())
 
 	case shard.BeaconChainShardID:
 		if node.beaconSync == nil {
-			return false, 0
+			return false, 0, 0
 		}
 		return node.beaconSync.SyncStatus(node.Beaconchain())
 
 	default:
 		// Shard node is not working on
-		return false, 0
+		return false, 0, 0
 	}
 }
 
@@ -710,7 +711,7 @@ func (node *Node) IsOutOfSync(shardID uint32) bool {
 		// downloaders inactive. Ask DNS sync instead
 		return node.legacyIsOutOfSync(shardID)
 	}
-	isSyncing, _ := ds.SyncStatus(shardID)
+	isSyncing, _, _ := ds.SyncStatus(shardID)
 	return !isSyncing
 }
 
@@ -720,14 +721,14 @@ func (node *Node) legacyIsOutOfSync(shardID uint32) bool {
 		if node.stateSync == nil {
 			return true
 		}
-		outOfSync, _ := node.stateSync.IsOutOfSync(node.Blockchain(), false)
+		outOfSync, _, _ := node.stateSync.IsOutOfSync(node.Blockchain(), false)
 		return outOfSync
 
 	case shard.BeaconChainShardID:
 		if node.beaconSync == nil {
 			return true
 		}
-		outOfSync, _ := node.beaconSync.IsOutOfSync(node.Beaconchain(), false)
+		outOfSync, _, _ := node.beaconSync.IsOutOfSync(node.Beaconchain(), false)
 		return outOfSync
 
 	default:

rosetta/services/network.go

@@ -82,7 +82,7 @@ func (s *NetworkAPI) NetworkStatus(
 	if rosettaError != nil {
 		return nil, rosettaError
 	}
-	isSyncing, targetHeight := s.hmy.NodeAPI.SyncStatus(s.hmy.BlockChain.ShardID())
+	isSyncing, targetHeight, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BlockChain.ShardID())
 	syncStatus := common.SyncingFinish
 	if targetHeight == 0 {
 		syncStatus = common.SyncingUnknown

rpc/blockchain.go

@@ -813,13 +813,13 @@ func (s *PublicBlockchainService) GetStakingNetworkInfo(
 
 // InSync returns if shard chain is syncing
 func (s *PublicBlockchainService) InSync(ctx context.Context) (bool, error) {
-	inSync, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BlockChain.ShardID())
+	inSync, _, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BlockChain.ShardID())
 	return inSync, nil
 }
 
 // BeaconInSync returns if beacon chain is syncing
 func (s *PublicBlockchainService) BeaconInSync(ctx context.Context) (bool, error) {
-	inSync, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BeaconChain.ShardID())
+	inSync, _, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BeaconChain.ShardID())
 	return inSync, nil
 }