update the hardcoded settings in stream sync

api/service/stagedstreamsync/const.go

@@ -36,11 +36,14 @@ type (
 		ServerOnly bool
 
 		// parameters
-		Network              nodeconfig.NetworkType
+		Network               nodeconfig.NetworkType
-		Concurrency          int // Number of concurrent sync requests
+		Concurrency           int    // Number of concurrent sync requests
-		MinStreams           int // Minimum number of streams to do sync
+		MinStreams            int    // Minimum number of streams to do sync
-		InitStreams          int // Number of streams requirement for initial bootstrap
+		InitStreams           int    // Number of streams requirement for initial bootstrap
-		MaxAdvertiseWaitTime int // maximum time duration between protocol advertisements
+		MaxAdvertiseWaitTime  int    // maximum time duration between protocol advertisements
+		MaxBlocksPerSyncCycle uint64 // maximum number of blocks per each sync cycle, if set to zero, all blocks will be synced in one full cycle
+		MaxMemSyncCycleSize   uint64 // maximum number of blocks to use in a single transaction for staged sync
+
 		// stream manager config
 		SmSoftLowCap int
 		SmHardLowCap int

api/service/stagedstreamsync/stage_heads.go

@@ -52,7 +52,7 @@ func (heads *StageHeads) Exec(ctx context.Context, firstCycle bool, invalidBlock
 	}
 
 	maxHeight := s.state.status.targetBN
-	maxBlocksPerSyncCycle := uint64(1024) // TODO: should be in config -> s.state.MaxBlocksPerSyncCycle
+	maxBlocksPerSyncCycle := s.state.config.MaxBlocksPerSyncCycle
 	currentHeight := heads.configs.bc.CurrentBlock().NumberU64()
 	s.state.currentCycle.TargetHeight = maxHeight
 	targetHeight := uint64(0)

api/service/stagedstreamsync/syncing.go

@@ -227,7 +227,7 @@ func (s *StagedStreamSync) doSyncCycle(ctx context.Context, initSync bool) (int,
 
 	s.inserted = 0
 	startHead := s.bc.CurrentBlock().NumberU64()
-	canRunCycleInOneTransaction := false
+	canRunCycleInOneTransaction := s.config.MaxBlocksPerSyncCycle > 0 && s.config.MaxBlocksPerSyncCycle <= s.config.MaxMemSyncCycleSize
 
 	var tx kv.RwTx
 	if canRunCycleInOneTransaction {

cmd/harmony/main.go

@@ -932,17 +932,19 @@ func setupStagedSyncService(node *node.Node, host p2p.Host, hc harmonyconfig.Har
 	}
 
 	sConfig := stagedstreamsync.Config{
-		ServerOnly:           !hc.Sync.Downloader,
+		ServerOnly:            !hc.Sync.Downloader,
-		Network:              nodeconfig.NetworkType(hc.Network.NetworkType),
+		Network:               nodeconfig.NetworkType(hc.Network.NetworkType),
-		Concurrency:          hc.Sync.Concurrency,
+		Concurrency:           hc.Sync.Concurrency,
-		MinStreams:           hc.Sync.MinPeers,
+		MinStreams:            hc.Sync.MinPeers,
-		InitStreams:          hc.Sync.InitStreams,
+		InitStreams:           hc.Sync.InitStreams,
-		MaxAdvertiseWaitTime: hc.Sync.MaxAdvertiseWaitTime,
+		MaxAdvertiseWaitTime:  hc.Sync.MaxAdvertiseWaitTime,
-		SmSoftLowCap:         hc.Sync.DiscSoftLowCap,
+		MaxBlocksPerSyncCycle: hc.Sync.StagedSyncCfg.MaxBlocksPerSyncCycle,
-		SmHardLowCap:         hc.Sync.DiscHardLowCap,
+		MaxMemSyncCycleSize:   hc.Sync.StagedSyncCfg.MaxMemSyncCycleSize,
-		SmHiCap:              hc.Sync.DiscHighCap,
+		SmSoftLowCap:          hc.Sync.DiscSoftLowCap,
-		SmDiscBatch:          hc.Sync.DiscBatch,
+		SmHardLowCap:          hc.Sync.DiscHardLowCap,
-		LogProgress:          node.NodeConfig.LogProgress,
+		SmHiCap:               hc.Sync.DiscHighCap,
+		SmDiscBatch:           hc.Sync.DiscBatch,
+		LogProgress:           node.NodeConfig.LogProgress,
 	}
 
 	// If we are running side chain, we will need to do some extra works for beacon