add cache configurations

cmd/harmony/config.go

@@ -145,6 +145,35 @@ func getDefaultSyncConfig(nt nodeconfig.NetworkType) harmonyconfig.SyncConfig {
 	}
 }
 
+func getDefaultCacheConfig(nt nodeconfig.NetworkType) harmonyconfig.CacheConfig {
+	cacheConfig := harmonyconfig.CacheConfig{
+		Disabled:        defaultCacheConfig.Disabled,
+		TrieNodeLimit:   defaultCacheConfig.TrieNodeLimit,
+		TriesInMemory:   defaultCacheConfig.TriesInMemory,
+		TrieTimeLimit:   defaultCacheConfig.TrieTimeLimit,
+		SnapshotLimit:   defaultCacheConfig.SnapshotLimit,
+		SnapshotWait:    defaultCacheConfig.SnapshotWait,
+		Preimages:       defaultCacheConfig.Preimages,
+		SnapshotNoBuild: defaultCacheConfig.SnapshotNoBuild,
+	}
+
+	switch nt {
+	case nodeconfig.Mainnet:
+		cacheConfig.Disabled = true
+		cacheConfig.Preimages = true
+	case nodeconfig.Testnet:
+		cacheConfig.Disabled = false
+		cacheConfig.Preimages = true
+	case nodeconfig.Localnet:
+		cacheConfig.Disabled = false
+		cacheConfig.Preimages = false
+	default:
+		cacheConfig.Disabled = false
+		cacheConfig.Preimages = true
+	}
+	return cacheConfig
+}
+
 var configCmd = &cobra.Command{
 	Use:   "config",
 	Short: "dump or update config",

cmd/harmony/config_migrations.go

@@ -334,7 +334,7 @@ func init() {
 
 	migrations["2.5.11"] = func(confTree *toml.Tree) *toml.Tree {
 		if confTree.Get("General.TriesInMemory") == nil {
-			confTree.Set("General.TriesInMemory", defaultConfig.General.TriesInMemory)
+			confTree.Set("General.TriesInMemory", defaultConfig.Cache.TriesInMemory)
 		}
 		confTree.Set("Version", "2.5.12")
 		return confTree
@@ -405,6 +405,17 @@ func init() {
 		return confTree
 	}
 
+	migrations["2.6.0"] = func(confTree *toml.Tree) *toml.Tree {
+		confTree.Delete("General.TriesInMemory")
+
+		if confTree.Get("Cache") == nil {
+			confTree.Set("Cache", defaultConfig.Cache)
+		}
+		// upgrade minor version because of `Cache` section introduction
+		confTree.Set("Version", "2.6.1")
+		return confTree
+	}
+
 	// check that the latest version here is the same as in default.go
 	largestKey := getNextVersion(migrations)
 	if largestKey != tomlConfigVersion {

cmd/harmony/default.go

@@ -1,13 +1,15 @@
 package main
 
 import (
+	"time"
+
 	"github.com/harmony-one/harmony/core"
 	"github.com/harmony-one/harmony/hmy"
 	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 )
 
-const tomlConfigVersion = "2.6.0"
+const tomlConfigVersion = "2.6.1"
 
 const (
 	defNetworkType = nodeconfig.Mainnet
@@ -24,7 +26,6 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
 		IsOffline:        false,
 		DataDir:          "./",
 		TraceEnable:      false,
-		TriesInMemory:    128,
 	},
 	Network: getDefaultNetworkConfig(defNetworkType),
 	P2P: harmonyconfig.P2pConfig{
@@ -131,6 +132,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
 		LowUsageThreshold: hmy.DefaultGPOConfig.LowUsageThreshold,
 		BlockGasLimit:     hmy.DefaultGPOConfig.BlockGasLimit,
 	},
+	Cache: getDefaultCacheConfig(defNetworkType),
 }
 
 var defaultSysConfig = harmonyconfig.SysConfig{
@@ -176,7 +178,7 @@ var defaultPrometheusConfig = harmonyconfig.PrometheusConfig{
 }
 
 var defaultStagedSyncConfig = harmonyconfig.StagedSyncConfig{
-	TurboMode:              true,
+	TurboMode:              false,
 	DoubleCheckBlockHashes: false,
 	MaxBlocksPerSyncCycle:  512,   // sync new blocks in each cycle, if set to zero means all blocks in one full cycle
 	MaxBackgroundBlocks:    512,   // max blocks to be downloaded at background process in turbo mode
@@ -228,14 +230,14 @@ var (
 		Downloader:           true,
 		StagedSync:           true,
 		StagedSyncCfg:        defaultStagedSyncConfig,
-		Concurrency:          4,
+		Concurrency:          2,
-		MinPeers:             4,
+		MinPeers:             2,
-		InitStreams:          4,
+		InitStreams:          2,
-		MaxAdvertiseWaitTime: 5, //minutes
+		MaxAdvertiseWaitTime: 1, //minutes
-		DiscSoftLowCap:       4,
+		DiscSoftLowCap:       2,
-		DiscHardLowCap:       4,
+		DiscHardLowCap:       2,
 		DiscHighCap:          1024,
-		DiscBatch:            8,
+		DiscBatch:            3,
 	}
 
 	defaultPartnerSyncConfig = harmonyconfig.SyncConfig{
@@ -271,6 +273,17 @@ var (
 	}
 )
 
+var defaultCacheConfig = harmonyconfig.CacheConfig{
+	Disabled:        false,
+	TrieNodeLimit:   256,
+	TriesInMemory:   128,
+	TrieTimeLimit:   2 * time.Minute,
+	SnapshotLimit:   256,
+	SnapshotWait:    true,
+	Preimages:       true,
+	SnapshotNoBuild: false,
+}
+
 const (
 	defaultBroadcastInvalidTx = false
 )
@@ -285,6 +298,7 @@ func getDefaultHmyConfigCopy(nt nodeconfig.NetworkType) harmonyconfig.HarmonyCon
 	}
 	config.Sync = getDefaultSyncConfig(nt)
 	config.DNSSync = getDefaultDNSSyncConfig(nt)
+	config.Cache = getDefaultCacheConfig(nt)
 
 	return config
 }
@@ -324,6 +338,11 @@ func getDefaultPrometheusConfigCopy() harmonyconfig.PrometheusConfig {
 	return config
 }
 
+func getDefaultCacheConfigCopy() harmonyconfig.CacheConfig {
+	config := defaultCacheConfig
+	return config
+}
+
 const (
 	nodeTypeValidator = "validator"
 	nodeTypeExplorer  = "explorer"

cmd/harmony/flags.go

@@ -32,7 +32,6 @@ var (
 		legacyDataDirFlag,
 
 		taraceFlag,
-		triesInMemoryFlag,
 	}
 
 	dnsSyncFlags = []cli.Flag{
@@ -268,6 +267,16 @@ var (
 		gpoBlockGasLimitFlag,
 	}
 
+	cacheConfigFlags = []cli.Flag{
+		cacheDisabled,
+		cacheTrieNodeLimit,
+		cacheTriesInMemory,
+		cachePreimages,
+		cacheSnapshotLimit,
+		cacheSnapshotNoBuild,
+		cacheSnapshotWait,
+	}
+
 	metricsFlags = []cli.Flag{
 		metricsETHFlag,
 		metricsExpensiveETHFlag,
@@ -352,11 +361,6 @@ var (
 		Usage:    "indicates if full transaction tracing should be enabled",
 		DefValue: defaultConfig.General.TraceEnable,
 	}
-	triesInMemoryFlag = cli.IntFlag{
-		Name:     "blockchain.tries_in_memory",
-		Usage:    "number of blocks from header stored in disk before exiting",
-		DefValue: defaultConfig.General.TriesInMemory,
-	}
 )
 
 func getRootFlags() []cli.Flag {
@@ -436,14 +440,6 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig)
 	if cli.IsFlagChanged(cmd, isBackupFlag) {
 		config.General.IsBackup = cli.GetBoolFlagValue(cmd, isBackupFlag)
 	}
-
-	if cli.IsFlagChanged(cmd, triesInMemoryFlag) {
-		value := cli.GetIntFlagValue(cmd, triesInMemoryFlag)
-		if value <= 2 {
-			panic("Must provide number greater than 2 for General.TriesInMemory")
-		}
-		config.General.TriesInMemory = value
-	}
 }
 
 // network flags
@@ -2115,3 +2111,70 @@ func applyGPOFlags(cmd *cobra.Command, cfg *harmonyconfig.HarmonyConfig) {
 		cfg.GPO.BlockGasLimit = cli.GetIntFlagValue(cmd, gpoBlockGasLimitFlag)
 	}
 }
+
+// cache config flags
+var (
+	cacheDisabled = cli.BoolFlag{
+		Name:     "cache.disabled",
+		Usage:    "Whether to disable trie write caching (archive node)",
+		DefValue: defaultCacheConfig.Disabled,
+	}
+	cacheTrieNodeLimit = cli.IntFlag{
+		Name:     "cache.trie_node_limit",
+		Usage:    " Memory limit (MB) at which to flush the current in-memory trie to disk",
+		DefValue: defaultCacheConfig.TrieNodeLimit,
+	}
+	cacheTriesInMemory = cli.Uint64Flag{
+		Name:     "cache.tries_in_memory",
+		Usage:    "Block number from the head stored in disk before exiting",
+		DefValue: defaultCacheConfig.TriesInMemory,
+	}
+	cachePreimages = cli.BoolFlag{
+		Name:     "cache.preimages",
+		Usage:    "Whether to store preimage of trie key to the disk",
+		DefValue: defaultCacheConfig.Preimages,
+	}
+	cacheSnapshotLimit = cli.IntFlag{
+		Name:     "cache.snapshot_limit",
+		Usage:    "Memory allowance (MB) to use for caching snapshot entries in memory",
+		DefValue: defaultCacheConfig.SnapshotLimit,
+	}
+	cacheSnapshotNoBuild = cli.BoolFlag{
+		Name:     "cache.snapshot_no_build",
+		Usage:    "Whether the background generation is allowed",
+		DefValue: defaultCacheConfig.SnapshotNoBuild,
+	}
+	cacheSnapshotWait = cli.BoolFlag{
+		Name:     "cache.snapshot_wait",
+		Usage:    "Wait for snapshot construction on startup",
+		DefValue: defaultCacheConfig.SnapshotWait,
+	}
+)
+
+func applyCacheFlags(cmd *cobra.Command, cfg *harmonyconfig.HarmonyConfig) {
+	if cli.IsFlagChanged(cmd, cacheDisabled) {
+		cfg.Cache.Disabled = cli.GetBoolFlagValue(cmd, cacheDisabled)
+	}
+	if cli.IsFlagChanged(cmd, cacheTrieNodeLimit) {
+		cfg.Cache.TrieNodeLimit = cli.GetIntFlagValue(cmd, cacheTrieNodeLimit)
+	}
+	if cli.IsFlagChanged(cmd, cacheTriesInMemory) {
+		value := cli.GetUint64FlagValue(cmd, cacheTriesInMemory)
+		if value <= 2 {
+			panic("Must provide number greater than 2 for Cache.TriesInMemory")
+		}
+		cfg.Cache.TriesInMemory = value
+	}
+	if cli.IsFlagChanged(cmd, cachePreimages) {
+		cfg.Cache.Preimages = cli.GetBoolFlagValue(cmd, cachePreimages)
+	}
+	if cli.IsFlagChanged(cmd, cacheSnapshotLimit) {
+		cfg.Cache.SnapshotLimit = cli.GetIntFlagValue(cmd, cacheSnapshotLimit)
+	}
+	if cli.IsFlagChanged(cmd, cacheSnapshotNoBuild) {
+		cfg.Cache.SnapshotNoBuild = cli.GetBoolFlagValue(cmd, cacheSnapshotNoBuild)
+	}
+	if cli.IsFlagChanged(cmd, cacheSnapshotWait) {
+		cfg.Cache.SnapshotWait = cli.GetBoolFlagValue(cmd, cacheSnapshotWait)
+	}
+}

cmd/harmony/flags_test.go

@@ -37,12 +37,11 @@ func TestHarmonyFlags(t *testing.T) {
 			expConfig: harmonyconfig.HarmonyConfig{
 				Version: tomlConfigVersion,
 				General: harmonyconfig.GeneralConfig{
-					NodeType:      "validator",
+					NodeType:   "validator",
-					NoStaking:     false,
+					NoStaking:  false,
-					ShardID:       -1,
+					ShardID:    -1,
-					IsArchival:    false,
+					IsArchival: false,
-					DataDir:       "./",
+					DataDir:    "./",
-					TriesInMemory: 128,
 				},
 				Network: harmonyconfig.NetworkConfig{
 					NetworkType: "mainnet",
@@ -183,6 +182,16 @@ func TestHarmonyFlags(t *testing.T) {
 					LowUsageThreshold: defaultConfig.GPO.LowUsageThreshold,
 					BlockGasLimit:     defaultConfig.GPO.BlockGasLimit,
 				},
+				Cache: harmonyconfig.CacheConfig{
+					Disabled:        defaultConfig.Cache.Disabled,
+					TrieNodeLimit:   defaultCacheConfig.TrieNodeLimit,
+					TriesInMemory:   defaultConfig.Cache.TriesInMemory,
+					TrieTimeLimit:   defaultConfig.Cache.TrieTimeLimit,
+					SnapshotLimit:   defaultConfig.Cache.SnapshotLimit,
+					SnapshotWait:    defaultConfig.Cache.SnapshotWait,
+					Preimages:       defaultConfig.Cache.Preimages,
+					SnapshotNoBuild: defaultConfig.Cache.SnapshotNoBuild,
+				},
 			},
 		},
 	}
@@ -208,80 +217,63 @@ func TestGeneralFlags(t *testing.T) {
 		{
 			args: []string{},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:      "validator",
+				NodeType:   "validator",
-				NoStaking:     false,
+				NoStaking:  false,
-				ShardID:       -1,
+				ShardID:    -1,
-				IsArchival:    false,
+				IsArchival: false,
-				DataDir:       "./",
+				DataDir:    "./",
-				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--run", "explorer", "--run.legacy", "--run.shard=0",
 				"--run.archive=true", "--datadir=./.hmy"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:      "explorer",
+				NodeType:   "explorer",
-				NoStaking:     true,
+				NoStaking:  true,
-				ShardID:       0,
+				ShardID:    0,
-				IsArchival:    true,
+				IsArchival: true,
-				DataDir:       "./.hmy",
+				DataDir:    "./.hmy",
-				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--node_type", "explorer", "--staking", "--shard_id", "0",
 				"--is_archival", "--db_dir", "./"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:      "explorer",
+				NodeType:   "explorer",
-				NoStaking:     false,
+				NoStaking:  false,
-				ShardID:       0,
+				ShardID:    0,
-				IsArchival:    true,
+				IsArchival: true,
-				DataDir:       "./",
+				DataDir:    "./",
-				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--staking=false", "--is_archival=false"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:      "validator",
+				NodeType:   "validator",
-				NoStaking:     true,
+				NoStaking:  true,
-				ShardID:       -1,
+				ShardID:    -1,
-				IsArchival:    false,
+				IsArchival: false,
-				DataDir:       "./",
+				DataDir:    "./",
-				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--run", "explorer", "--run.shard", "0"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:      "explorer",
+				NodeType:   "explorer",
-				NoStaking:     false,
+				NoStaking:  false,
-				ShardID:       0,
+				ShardID:    0,
-				IsArchival:    false,
+				IsArchival: false,
-				DataDir:       "./",
+				DataDir:    "./",
-				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--run", "explorer", "--run.shard", "0", "--run.archive=false"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:      "explorer",
+				NodeType:   "explorer",
-				NoStaking:     false,
+				NoStaking:  false,
-				ShardID:       0,
+				ShardID:    0,
-				IsArchival:    false,
+				IsArchival: false,
-				DataDir:       "./",
+				DataDir:    "./",
-				TriesInMemory: 128,
-			},
-		},
-		{
-			args: []string{"--blockchain.tries_in_memory", "64"},
-			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:      "validator",
-				NoStaking:     false,
-				ShardID:       -1,
-				IsArchival:    false,
-				DataDir:       "./",
-				TriesInMemory: 64,
 			},
 		},
 	}
@@ -1435,6 +1427,58 @@ func TestGPOFlags(t *testing.T) {
 	}
 }
 
+func TestCacheFlags(t *testing.T) {
+	tests := []struct {
+		args      []string
+		expConfig harmonyconfig.CacheConfig
+		expErr    error
+	}{
+		{
+			args: []string{},
+			expConfig: harmonyconfig.CacheConfig{
+				Disabled:        true, // based on network type
+				TrieNodeLimit:   defaultCacheConfig.TrieNodeLimit,
+				TriesInMemory:   defaultCacheConfig.TriesInMemory,
+				TrieTimeLimit:   defaultCacheConfig.TrieTimeLimit,
+				SnapshotLimit:   defaultCacheConfig.SnapshotLimit,
+				SnapshotWait:    defaultCacheConfig.SnapshotWait,
+				Preimages:       defaultCacheConfig.Preimages, // based on network type
+				SnapshotNoBuild: defaultCacheConfig.SnapshotNoBuild,
+			},
+		},
+		{
+			args: []string{"--cache.disabled=true", "--cache.trie_node_limit", "512", "--cache.tries_in_memory", "256", "--cache.preimages=false", "--cache.snapshot_limit", "512", "--cache.snapshot_no_build=true", "--cache.snapshot_wait=false"},
+			expConfig: harmonyconfig.CacheConfig{
+				Disabled:        true,
+				TrieNodeLimit:   512,
+				TriesInMemory:   256,
+				TrieTimeLimit:   2 * time.Minute,
+				SnapshotLimit:   512,
+				SnapshotWait:    false,
+				Preimages:       false,
+				SnapshotNoBuild: true,
+			},
+		},
+	}
+
+	for i, test := range tests {
+		ts := newFlagTestSuite(t, cacheConfigFlags, applyCacheFlags)
+		hc, err := ts.run(test.args)
+
+		if assErr := assertError(err, test.expErr); assErr != nil {
+			t.Fatalf("Test %v: %v", i, assErr)
+		}
+		if err != nil || test.expErr != nil {
+			continue
+		}
+
+		if !reflect.DeepEqual(hc.Cache, test.expConfig) {
+			t.Errorf("Test %v:\n\t%+v\n\t%+v", i, hc.Cache, test.expConfig)
+		}
+		ts.tearDown()
+	}
+}
+
 func TestDevnetFlags(t *testing.T) {
 	tests := []struct {
 		args      []string

cmd/harmony/main.go

@@ -245,6 +245,7 @@ func applyRootFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
 	applySyncFlags(cmd, config)
 	applyShardDataFlags(cmd, config)
 	applyGPOFlags(cmd, config)
+	applyCacheFlags(cmd, config)
 }
 
 func setupNodeLog(config harmonyconfig.HarmonyConfig) {

internal/configs/harmony/harmony.go

@@ -37,6 +37,7 @@ type HarmonyConfig struct {
 	ShardData  ShardDataConfig
 	GPO        GasPriceOracleConfig
 	Preimage   *PreimageConfig
+	Cache      CacheConfig
 }
 
 func (hc HarmonyConfig) ToRPCServerConfig() nodeconfig.RPCServerConfig {
@@ -138,7 +139,6 @@ type GeneralConfig struct {
 	TraceEnable            bool
 	EnablePruneBeaconChain bool
 	RunElasticMode         bool
-	TriesInMemory          int
 }
 
 type TiKVConfig struct {
@@ -306,6 +306,17 @@ type RevertConfig struct {
 	RevertBefore int
 }
 
+type CacheConfig struct {
+	Disabled        bool          // Whether to disable trie write caching (archive node)
+	TrieNodeLimit   int           // Memory limit (MB) at which to flush the current in-memory trie to disk
+	TrieTimeLimit   time.Duration // Time limit after which to flush the current in-memory trie to disk
+	TriesInMemory   uint64        // Block number from the head stored in disk before exiting
+	Preimages       bool          // Whether to store preimage of trie key to the disk
+	SnapshotLimit   int           // Memory allowance (MB) to use for caching snapshot entries in memory
+	SnapshotNoBuild bool          // Whether the background generation is allowed
+	SnapshotWait    bool          // Wait for snapshot construction on startup
+}
+
 type PreimageConfig struct {
 	ImportFrom    string
 	ExportTo      string

internal/shardchain/shardchains.go

@@ -3,7 +3,6 @@ package shardchain
 import (
 	"math/big"
 	"sync"
-	"time"
 
 	"github.com/harmony-one/harmony/core/state"
 	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
@@ -110,14 +109,19 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c
 			Uint32("shardID", shardID).
 			Msg("disable cache, running in archival mode")
 	} else {
-		cacheConfig = &core.CacheConfig{
+		hc := sc.harmonyconfig
-			TrieNodeLimit: 256,
+		if hc != nil {
-			TrieTimeLimit: 2 * time.Minute,
+			cacheConfig = &core.CacheConfig{
-			TriesInMemory: 128,
+				Disabled:      hc.Cache.Disabled,
-			Preimages:     true,
+				TrieNodeLimit: hc.Cache.TrieNodeLimit,
-		}
+				TrieTimeLimit: hc.Cache.TrieTimeLimit,
-		if sc.harmonyconfig != nil {
+				TriesInMemory: hc.Cache.TriesInMemory,
-			cacheConfig.TriesInMemory = uint64(sc.harmonyconfig.General.TriesInMemory)
+				SnapshotLimit: hc.Cache.SnapshotLimit,
+				SnapshotWait:  hc.Cache.SnapshotWait,
+				Preimages:     hc.Cache.Preimages,
+			}
+		} else {
+			cacheConfig = nil
 		}
 	}