[sync] add a tolerance number for node-sync checking

Signed-off-by: Leo Chen <leo@harmony.one>
pull/3794/head
Leo Chen 3 years ago
parent b88b86fc15
commit 07807c8f2d
  1. 14
      api/service/explorer/service.go
  2. 17
      api/service/legacysync/syncing.go
  3. 4
      hmy/downloader/downloader.go
  4. 4
      hmy/downloader/downloaders.go
  5. 2
      hmy/hmy.go
  6. 21
      node/node_syncing.go
  7. 2
      rosetta/services/network.go
  8. 4
      rpc/blockchain.go

@ -29,6 +29,7 @@ const (
explorerPortDifference = 4000
defaultPageSize = "1000"
maxAddresses = 100000
nodeSyncTolerance = 5
)
// HTTPError is an HTTP error.
@ -270,12 +271,19 @@ func (s *Service) GetTotalSupply(w http.ResponseWriter, r *http.Request) {
}
}
// GetNodeSync returns status code 500 if node is not in sync
// GetNodeSync returns status code Teapot 418 if node is not in sync
func (s *Service) GetNodeSync(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
sync, _ := s.backend.SyncStatus(s.blockchain.ShardID())
sync, remote, diff := s.backend.SyncStatus(s.blockchain.ShardID())
if !sync {
w.WriteHeader(http.StatusTeapot)
utils.Logger().Debug().Uint64("remote", remote).Uint64("diff", diff).Msg("GetNodeSyncStatus")
if remote == 0 || diff > nodeSyncTolerance {
w.WriteHeader(http.StatusTeapot)
} else {
// return sync'ed if the diff is less than nodeSyncTolerance
// this tolerance is only applicable to /node-sync API call
sync = true
}
}
if err := json.NewEncoder(w).Encode(sync); err != nil {
utils.Logger().Warn().Msg("cannot JSON-encode total supply")

@ -1054,16 +1054,17 @@ func (ss *StateSync) GetMaxPeerHeight() uint64 {
return ss.getMaxPeerHeight(false)
}
// SyncStatus returns inSync and remote height
func (ss *StateSync) SyncStatus(bc *core.BlockChain) (bool, uint64) {
outOfSync, remoteNum := ss.IsOutOfSync(bc, false)
return !outOfSync, remoteNum
// SyncStatus returns inSync, remote height, and difference between remote height and local height
func (ss *StateSync) SyncStatus(bc *core.BlockChain) (bool, uint64, uint64) {
outOfSync, remoteHeight, heightDiff := ss.IsOutOfSync(bc, false)
return !outOfSync, remoteHeight, heightDiff
}
// IsOutOfSync checks whether the node is out of sync from other peers
func (ss *StateSync) IsOutOfSync(bc *core.BlockChain, doubleCheck bool) (bool, uint64) {
// It returns the sync status, remote height, and the difference betwen local blockchain and remote max height
func (ss *StateSync) IsOutOfSync(bc *core.BlockChain, doubleCheck bool) (bool, uint64, uint64) {
if ss.syncConfig == nil {
return true, 0 // If syncConfig is not instantiated, return not in sync
return true, 0, 0 // If syncConfig is not instantiated, return not in sync
}
otherHeight1 := ss.getMaxPeerHeight(false)
lastHeight := bc.CurrentBlock().NumberU64()
@ -1074,7 +1075,7 @@ func (ss *StateSync) IsOutOfSync(bc *core.BlockChain, doubleCheck bool) (bool, u
Uint64("OtherHeight", otherHeight1).
Uint64("lastHeight", lastHeight).
Msg("[SYNC] Checking sync status")
return wasOutOfSync, otherHeight1
return wasOutOfSync, otherHeight1, otherHeight1 - lastHeight
}
time.Sleep(1 * time.Second)
// double check the sync status after 1 second to confirm (avoid false alarm)
@ -1090,7 +1091,7 @@ func (ss *StateSync) IsOutOfSync(bc *core.BlockChain, doubleCheck bool) (bool, u
Uint64("currentHeight", currentHeight).
Msg("[SYNC] Checking sync status")
// Only confirm out of sync when the node has lower height and didn't move in heights for 2 consecutive checks
return wasOutOfSync && isOutOfSync && lastHeight == currentHeight, otherHeight2
return wasOutOfSync && isOutOfSync && lastHeight == currentHeight, otherHeight2, otherHeight2 - currentHeight
}
// SyncLoop will keep syncing with peers until catches up

@ -117,12 +117,12 @@ func (d *Downloader) NumPeers() int {
}
// IsSyncing return the current sync status
func (d *Downloader) SyncStatus() (bool, uint64) {
func (d *Downloader) SyncStatus() (bool, uint64, uint64) {
syncing, target := d.status.get()
if !syncing {
target = d.bc.CurrentBlock().NumberU64()
}
return syncing, target
return syncing, target, 0
}
// SubscribeDownloadStarted subscribe download started

@ -83,10 +83,10 @@ func (ds *Downloaders) NumPeers() map[uint32]int {
// SyncStatus returns whether the given shard is doing syncing task and the target block
// number.
func (ds *Downloaders) SyncStatus(shardID uint32) (bool, uint64) {
func (ds *Downloaders) SyncStatus(shardID uint32) (bool, uint64, uint64) {
d, ok := ds.ds[shardID]
if !ok {
return false, 0
return false, 0, 0
}
return d.SyncStatus()
}

@ -88,7 +88,7 @@ type NodeAPI interface {
GetStakingTransactionsCount(address, txType string) (uint64, error)
IsCurrentlyLeader() bool
IsOutOfSync(shardID uint32) bool
SyncStatus(shardID uint32) (bool, uint64)
SyncStatus(shardID uint32) (bool, uint64, uint64)
SyncPeers() map[string]int
ReportStakingErrorSink() types.TransactionErrorReports
ReportPlainErrorSink() types.TransactionErrorReports

@ -302,7 +302,7 @@ func (node *Node) doSync(bc *core.BlockChain, worker *worker.Worker, willJoinCon
utils.Logger().Debug().Int("len", node.stateSync.GetActivePeerNumber()).Msg("[SYNC] Get Active Peers")
}
// TODO: treat fake maximum height
if outOfSync, _ := node.stateSync.IsOutOfSync(bc, true); outOfSync {
if outOfSync, _, _ := node.stateSync.IsOutOfSync(bc, true); outOfSync {
node.IsInSync.UnSet()
if willJoinConsensus {
node.Consensus.BlocksNotSynchronized()
@ -673,8 +673,9 @@ func (node *Node) getCommitSigFromDB(block *types.Block) ([]byte, error) {
}
// SyncStatus return the syncing status, including whether node is syncing
// and the target block number.
func (node *Node) SyncStatus(shardID uint32) (bool, uint64) {
// and the target block number, and the difference between current block
// and target block.
func (node *Node) SyncStatus(shardID uint32) (bool, uint64, uint64) {
ds := node.getDownloaders()
if ds == nil || !ds.IsActive() {
// downloaders inactive. Ask DNS sync instead
@ -683,23 +684,23 @@ func (node *Node) SyncStatus(shardID uint32) (bool, uint64) {
return ds.SyncStatus(shardID)
}
func (node *Node) legacySyncStatus(shardID uint32) (bool, uint64) {
func (node *Node) legacySyncStatus(shardID uint32) (bool, uint64, uint64) {
switch shardID {
case node.NodeConfig.ShardID:
if node.stateSync == nil {
return false, 0
return false, 0, 0
}
return node.stateSync.SyncStatus(node.Blockchain())
case shard.BeaconChainShardID:
if node.beaconSync == nil {
return false, 0
return false, 0, 0
}
return node.beaconSync.SyncStatus(node.Beaconchain())
default:
// Shard node is not working on
return false, 0
return false, 0, 0
}
}
@ -710,7 +711,7 @@ func (node *Node) IsOutOfSync(shardID uint32) bool {
// downloaders inactive. Ask DNS sync instead
return node.legacyIsOutOfSync(shardID)
}
isSyncing, _ := ds.SyncStatus(shardID)
isSyncing, _, _ := ds.SyncStatus(shardID)
return !isSyncing
}
@ -720,14 +721,14 @@ func (node *Node) legacyIsOutOfSync(shardID uint32) bool {
if node.stateSync == nil {
return true
}
outOfSync, _ := node.stateSync.IsOutOfSync(node.Blockchain(), false)
outOfSync, _, _ := node.stateSync.IsOutOfSync(node.Blockchain(), false)
return outOfSync
case shard.BeaconChainShardID:
if node.beaconSync == nil {
return true
}
outOfSync, _ := node.beaconSync.IsOutOfSync(node.Beaconchain(), false)
outOfSync, _, _ := node.beaconSync.IsOutOfSync(node.Beaconchain(), false)
return outOfSync
default:

@ -82,7 +82,7 @@ func (s *NetworkAPI) NetworkStatus(
if rosettaError != nil {
return nil, rosettaError
}
isSyncing, targetHeight := s.hmy.NodeAPI.SyncStatus(s.hmy.BlockChain.ShardID())
isSyncing, targetHeight, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BlockChain.ShardID())
syncStatus := common.SyncingFinish
if targetHeight == 0 {
syncStatus = common.SyncingUnknown

@ -813,13 +813,13 @@ func (s *PublicBlockchainService) GetStakingNetworkInfo(
// InSync returns if shard chain is syncing
func (s *PublicBlockchainService) InSync(ctx context.Context) (bool, error) {
inSync, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BlockChain.ShardID())
inSync, _, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BlockChain.ShardID())
return inSync, nil
}
// BeaconInSync returns if beacon chain is syncing
func (s *PublicBlockchainService) BeaconInSync(ctx context.Context) (bool, error) {
inSync, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BeaconChain.ShardID())
inSync, _, _ := s.hmy.NodeAPI.SyncStatus(s.hmy.BeaconChain.ShardID())
return inSync, nil
}

Loading…
Cancel
Save