[SYNC] Sync stuck fix2 (#3976)

* fix an data race cause by early unlock

* add a concurrency test for syncing cache

* skip the concurrent test for sync status

api/service/legacysync/syncing.go

@@ -1195,8 +1195,9 @@ func getSyncStatusExpiration(role nodeconfig.Role) time.Duration {
 func (status *syncStatus) Get(fallback func() SyncCheckResult) SyncCheckResult {
 	status.lock.RLock()
 	if !status.expired() {
+		result := status.lastResult
 		status.lock.RUnlock()
-		return status.lastResult
+		return result
 	}
 	status.lock.RUnlock()
 

api/service/legacysync/syncing_test.go

@@ -6,7 +6,10 @@ import (
 	"math/big"
 	"reflect"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 
@@ -250,3 +253,47 @@ func makeTestBlock(bn uint64, parentHash common.Hash) *types.Block {
 	block := types.NewBlockWithHeader(testHeader)
 	return block
 }
+
+func TestSyncStatus_Get_Concurrency(t *testing.T) {
+	t.Skip()
+
+	ss := newSyncStatus(nodeconfig.Validator)
+	ss.expiration = 2 * time.Second
+	var (
+		total   int32
+		updated int32
+		wg      sync.WaitGroup
+		stop    = make(chan struct{})
+	)
+
+	fb := func() SyncCheckResult {
+		time.Sleep(1 * time.Second)
+		atomic.AddInt32(&updated, 1)
+		return SyncCheckResult{IsInSync: true}
+	}
+	for i := 0; i != 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			t := time.NewTicker(20 * time.Millisecond)
+			defer t.Stop()
+			for {
+				select {
+				case <-stop:
+					return
+				case <-t.C:
+					atomic.AddInt32(&total, 1)
+					ss.Get(fb)
+				}
+			}
+		}()
+	}
+
+	time.Sleep(10 * time.Second)
+	close(stop)
+	wg.Wait()
+
+	fmt.Printf("updated %v times\n", updated)
+	fmt.Printf("total %v times\n", total)
+}