feat: triesInMemory flag

cmd/harmony/config_migrations.go

@@ -332,6 +332,14 @@ func init() {
 		return confTree
 	}
 
+	migrations["2.5.11"] = func(confTree *toml.Tree) *toml.Tree {
+		if confTree.Get("General.TriesInMemory") == nil {
+			confTree.Set("General.TriesInMemory", defaultConfig.General.TriesInMemory)
+		}
+		confTree.Set("Version", "2.5.12")
+		return confTree
+	}
+
 	// check that the latest version here is the same as in default.go
 	largestKey := getNextVersion(migrations)
 	if largestKey != tomlConfigVersion {

cmd/harmony/default.go

@@ -5,7 +5,7 @@ import (
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 )
 
-const tomlConfigVersion = "2.5.11"
+const tomlConfigVersion = "2.5.12"
 
 const (
 	defNetworkType = nodeconfig.Mainnet
@@ -22,6 +22,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
 		IsOffline:        false,
 		DataDir:          "./",
 		TraceEnable:      false,
+		TriesInMemory:    128,
 	},
 	Network: getDefaultNetworkConfig(defNetworkType),
 	P2P: harmonyconfig.P2pConfig{

cmd/harmony/flags.go

@@ -31,6 +31,7 @@ var (
 		legacyDataDirFlag,
 
 		taraceFlag,
+		triesInMemoryFlag,
 	}
 
 	dnsSyncFlags = []cli.Flag{
@@ -322,6 +323,11 @@ var (
 		Usage:    "indicates if full transaction tracing should be enabled",
 		DefValue: defaultConfig.General.TraceEnable,
 	}
+	triesInMemoryFlag = cli.IntFlag{
+		Name:     "blockchain.tries_in_memory",
+		Usage:    "number of blocks from header stored in disk before exiting",
+		DefValue: defaultConfig.General.TriesInMemory,
+	}
 )
 
 func getRootFlags() []cli.Flag {
@@ -399,6 +405,14 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig)
 	if cli.IsFlagChanged(cmd, isBackupFlag) {
 		config.General.IsBackup = cli.GetBoolFlagValue(cmd, isBackupFlag)
 	}
+
+	if cli.IsFlagChanged(cmd, triesInMemoryFlag) {
+		value := cli.GetIntFlagValue(cmd, triesInMemoryFlag)
+		if value <= 1 {
+			panic("Must number greater than 1 for txpool.accountslots")
+		}
+		config.General.TriesInMemory = value
+	}
 }
 
 // network flags

cmd/harmony/flags_test.go

@@ -35,11 +35,12 @@ func TestHarmonyFlags(t *testing.T) {
 			expConfig: harmonyconfig.HarmonyConfig{
 				Version: tomlConfigVersion,
 				General: harmonyconfig.GeneralConfig{
-					NodeType:   "validator",
+					NodeType:      "validator",
-					NoStaking:  false,
+					NoStaking:     false,
-					ShardID:    -1,
+					ShardID:       -1,
-					IsArchival: false,
+					IsArchival:    false,
-					DataDir:    "./",
+					DataDir:       "./",
+					TriesInMemory: 128,
 				},
 				Network: harmonyconfig.NetworkConfig{
 					NetworkType: "mainnet",
@@ -186,63 +187,80 @@ func TestGeneralFlags(t *testing.T) {
 		{
 			args: []string{},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:   "validator",
+				NodeType:      "validator",
-				NoStaking:  false,
+				NoStaking:     false,
-				ShardID:    -1,
+				ShardID:       -1,
-				IsArchival: false,
+				IsArchival:    false,
-				DataDir:    "./",
+				DataDir:       "./",
+				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--run", "explorer", "--run.legacy", "--run.shard=0",
 				"--run.archive=true", "--datadir=./.hmy"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:   "explorer",
+				NodeType:      "explorer",
-				NoStaking:  true,
+				NoStaking:     true,
-				ShardID:    0,
+				ShardID:       0,
-				IsArchival: true,
+				IsArchival:    true,
-				DataDir:    "./.hmy",
+				DataDir:       "./.hmy",
+				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--node_type", "explorer", "--staking", "--shard_id", "0",
 				"--is_archival", "--db_dir", "./"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:   "explorer",
+				NodeType:      "explorer",
-				NoStaking:  false,
+				NoStaking:     false,
-				ShardID:    0,
+				ShardID:       0,
-				IsArchival: true,
+				IsArchival:    true,
-				DataDir:    "./",
+				DataDir:       "./",
+				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--staking=false", "--is_archival=false"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:   "validator",
+				NodeType:      "validator",
-				NoStaking:  true,
+				NoStaking:     true,
-				ShardID:    -1,
+				ShardID:       -1,
-				IsArchival: false,
+				IsArchival:    false,
-				DataDir:    "./",
+				DataDir:       "./",
+				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--run", "explorer", "--run.shard", "0"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:   "explorer",
+				NodeType:      "explorer",
-				NoStaking:  false,
+				NoStaking:     false,
-				ShardID:    0,
+				ShardID:       0,
-				IsArchival: false,
+				IsArchival:    false,
-				DataDir:    "./",
+				DataDir:       "./",
+				TriesInMemory: 128,
 			},
 		},
 		{
 			args: []string{"--run", "explorer", "--run.shard", "0", "--run.archive=false"},
 			expConfig: harmonyconfig.GeneralConfig{
-				NodeType:   "explorer",
+				NodeType:      "explorer",
-				NoStaking:  false,
+				NoStaking:     false,
-				ShardID:    0,
+				ShardID:       0,
-				IsArchival: false,
+				IsArchival:    false,
-				DataDir:    "./",
+				DataDir:       "./",
+				TriesInMemory: 128,
+			},
+		},
+		{
+			args: []string{"--blockchain.tries_in_memory", "64"},
+			expConfig: harmonyconfig.GeneralConfig{
+				NodeType:      "validator",
+				NoStaking:     false,
+				ShardID:       -1,
+				IsArchival:    false,
+				DataDir:       "./",
+				TriesInMemory: 64,
 			},
 		},
 	}

core/blockchain_impl.go

@@ -103,7 +103,6 @@ const (
 	maxFutureBlocks                    = 16
 	maxTimeFutureBlocks                = 30
 	badBlockLimit                      = 10
-	triesInMemory                      = 128
 	triesInRedis                       = 1000
 	shardCacheLimit                    = 10
 	commitsCacheLimit                  = 10
@@ -127,6 +126,7 @@ type CacheConfig struct {
 	Disabled      bool          // Whether to disable trie write caching (archive node)
 	TrieNodeLimit int           // Memory limit (MB) at which to flush the current in-memory trie to disk
 	TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
+	TriesInMemory uint64        // Block number from the head stored in disk before exiting
 }
 
 type BlockChainImpl struct {
@@ -223,12 +223,7 @@ func newBlockChainWithOptions(
 	db ethdb.Database, stateCache state.Database, beaconChain BlockChain,
 	cacheConfig *CacheConfig, chainConfig *params.ChainConfig,
 	engine consensus_engine.Engine, vmConfig vm.Config, options Options) (*BlockChainImpl, error) {
-	if cacheConfig == nil {
+
-		cacheConfig = &CacheConfig{
-			TrieNodeLimit: 256 * 1024 * 1024,
-			TrieTimeLimit: 2 * time.Minute,
-		}
-	}
 	bodyCache, _ := lru.New(bodyCacheLimit)
 	bodyRLPCache, _ := lru.New(bodyCacheLimit)
 	receiptsCache, _ := lru.New(receiptsCacheLimit)
@@ -1080,7 +1075,7 @@ func (bc *BlockChainImpl) Stop() {
 	if !bc.cacheConfig.Disabled {
 		triedb := bc.stateCache.TrieDB()
 
-		for _, offset := range []uint64{0, 1, triesInMemory - 1} {
+		for _, offset := range []uint64{0, 1, bc.cacheConfig.TriesInMemory - 1} {
 			if number := bc.CurrentBlock().NumberU64(); number > offset {
 				recent := bc.GetHeaderByNumber(number - offset)
 				if recent != nil {
@@ -1407,7 +1402,7 @@ func (bc *BlockChainImpl) WriteBlockWithState(
 		triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
 		bc.triegc.Push(root, -int64(block.NumberU64()))
 
-		if current := block.NumberU64(); current > triesInMemory {
+		if current := block.NumberU64(); current > bc.cacheConfig.TriesInMemory {
 			// If we exceeded our memory allowance, flush matured singleton nodes to disk
 			var (
 				nodes, imgs = triedb.Size()
@@ -1417,7 +1412,7 @@ func (bc *BlockChainImpl) WriteBlockWithState(
 				triedb.Cap(limit - ethdb.IdealBatchSize)
 			}
 			// Find the next state trie we need to commit
-			header := bc.GetHeaderByNumber(current - triesInMemory)
+			header := bc.GetHeaderByNumber(current - bc.cacheConfig.TriesInMemory)
 			if header != nil {
 				chosen := header.Number().Uint64()
 
@@ -1425,11 +1420,11 @@ func (bc *BlockChainImpl) WriteBlockWithState(
 				if bc.gcproc > bc.cacheConfig.TrieTimeLimit {
 					// If we're exceeding limits but haven't reached a large enough memory gap,
 					// warn the user that the system is becoming unstable.
-					if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
+					if chosen < lastWrite+bc.cacheConfig.TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
 						utils.Logger().Info().
 							Dur("time", bc.gcproc).
 							Dur("allowance", bc.cacheConfig.TrieTimeLimit).
-							Float64("optimum", float64(chosen-lastWrite)/triesInMemory).
+							Float64("optimum", float64(chosen-lastWrite)/float64(bc.cacheConfig.TriesInMemory)).
 							Msg("State in memory for too long, committing")
 					}
 					// Flush an entire trie and restart the counters

internal/configs/harmony/harmony.go

@@ -74,6 +74,7 @@ type GeneralConfig struct {
 	TraceEnable            bool
 	EnablePruneBeaconChain bool
 	RunElasticMode         bool
+	TriesInMemory          int
 }
 
 type TiKVConfig struct {

internal/shardchain/shardchains.go

@@ -3,6 +3,7 @@ package shardchain
 import (
 	"math/big"
 	"sync"
+	"time"
 
 	"github.com/harmony-one/harmony/core/state"
 	harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
@@ -104,6 +105,15 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c
 		utils.Logger().Info().
 			Uint32("shardID", shardID).
 			Msg("disable cache, running in archival mode")
+	} else {
+		cacheConfig = &core.CacheConfig{
+			TrieNodeLimit: 256 * 1024 * 1024,
+			TrieTimeLimit: 2 * time.Minute,
+			TriesInMemory: 128,
+		}
+		if sc.harmonyconfig != nil {
+			cacheConfig.TriesInMemory = uint64(sc.harmonyconfig.General.TriesInMemory)
+		}
 	}
 
 	chainConfig := *sc.chainConfig

rosetta/infra/harmony-mainnet.conf

@@ -35,6 +35,7 @@ Version = "2.5.11"
   RunElasticMode = false
   ShardID = 0
   TraceEnable = false
+  TriesInMemory = 128
 
 [HTTP]
   AuthPort = 9501

rosetta/infra/harmony-pstn.conf

@@ -35,6 +35,7 @@ Version = "2.5.11"
   RunElasticMode = false
   ShardID = 0
   TraceEnable = false
+  TriesInMemory = 128
 
 [HTTP]
   AuthPort = 9501