From 6dfcd51ba3b44d3fac96bc9f6a4af27fcfa4f4b1 Mon Sep 17 00:00:00 2001 From: Sun Hyuk Ahn Date: Thu, 9 Feb 2023 16:46:46 -0800 Subject: [PATCH] feat: triesInMemory flag --- cmd/harmony/config_migrations.go | 8 +++ cmd/harmony/default.go | 3 +- cmd/harmony/flags.go | 14 +++++ cmd/harmony/flags_test.go | 88 +++++++++++++++++------------ core/blockchain_impl.go | 19 +++---- internal/configs/harmony/harmony.go | 1 + internal/shardchain/shardchains.go | 10 ++++ rosetta/infra/harmony-mainnet.conf | 1 + rosetta/infra/harmony-pstn.conf | 1 + 9 files changed, 97 insertions(+), 48 deletions(-) diff --git a/cmd/harmony/config_migrations.go b/cmd/harmony/config_migrations.go index bdbfefdc9..7c9420a0b 100644 --- a/cmd/harmony/config_migrations.go +++ b/cmd/harmony/config_migrations.go @@ -332,6 +332,14 @@ func init() { return confTree } + migrations["2.5.11"] = func(confTree *toml.Tree) *toml.Tree { + if confTree.Get("General.TriesInMemory") == nil { + confTree.Set("General.TriesInMemory", defaultConfig.General.TriesInMemory) + } + confTree.Set("Version", "2.5.12") + return confTree + } + // check that the latest version here is the same as in default.go largestKey := getNextVersion(migrations) if largestKey != tomlConfigVersion { diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 78d2643b6..3835c4921 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -5,7 +5,7 @@ import ( nodeconfig "github.com/harmony-one/harmony/internal/configs/node" ) -const tomlConfigVersion = "2.5.11" +const tomlConfigVersion = "2.5.12" const ( defNetworkType = nodeconfig.Mainnet @@ -22,6 +22,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ IsOffline: false, DataDir: "./", TraceEnable: false, + TriesInMemory: 128, }, Network: getDefaultNetworkConfig(defNetworkType), P2P: harmonyconfig.P2pConfig{ diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index 947adacc5..c3b03e336 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -31,6 +31,7 @@ var ( legacyDataDirFlag, taraceFlag, + triesInMemoryFlag, } dnsSyncFlags = []cli.Flag{ @@ -322,6 +323,11 @@ var ( Usage: "indicates if full transaction tracing should be enabled", DefValue: defaultConfig.General.TraceEnable, } + triesInMemoryFlag = cli.IntFlag{ + Name: "blockchain.tries_in_memory", + Usage: "number of blocks from header stored in disk before exiting", + DefValue: defaultConfig.General.TriesInMemory, + } ) func getRootFlags() []cli.Flag { @@ -399,6 +405,14 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) if cli.IsFlagChanged(cmd, isBackupFlag) { config.General.IsBackup = cli.GetBoolFlagValue(cmd, isBackupFlag) } + + if cli.IsFlagChanged(cmd, triesInMemoryFlag) { + value := cli.GetIntFlagValue(cmd, triesInMemoryFlag) + if value <= 1 { + panic("Must number greater than 1 for txpool.accountslots") + } + config.General.TriesInMemory = value + } } // network flags diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 7f8cf34a8..15ba9aaf0 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -35,11 +35,12 @@ func TestHarmonyFlags(t *testing.T) { expConfig: harmonyconfig.HarmonyConfig{ Version: tomlConfigVersion, General: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: false, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, Network: harmonyconfig.NetworkConfig{ NetworkType: "mainnet", @@ -186,63 +187,80 @@ func TestGeneralFlags(t *testing.T) { { args: []string{}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: false, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.legacy", "--run.shard=0", "--run.archive=true", "--datadir=./.hmy"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: true, - ShardID: 0, - IsArchival: true, - DataDir: "./.hmy", + NodeType: "explorer", + NoStaking: true, + ShardID: 0, + IsArchival: true, + DataDir: "./.hmy", + TriesInMemory: 128, }, }, { args: []string{"--node_type", "explorer", "--staking", "--shard_id", "0", "--is_archival", "--db_dir", "./"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: true, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: true, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--staking=false", "--is_archival=false"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "validator", - NoStaking: true, - ShardID: -1, - IsArchival: false, - DataDir: "./", + NodeType: "validator", + NoStaking: true, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.shard", "0"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: false, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, }, }, { args: []string{"--run", "explorer", "--run.shard", "0", "--run.archive=false"}, expConfig: harmonyconfig.GeneralConfig{ - NodeType: "explorer", - NoStaking: false, - ShardID: 0, - IsArchival: false, - DataDir: "./", + NodeType: "explorer", + NoStaking: false, + ShardID: 0, + IsArchival: false, + DataDir: "./", + TriesInMemory: 128, + }, + }, + { + args: []string{"--blockchain.tries_in_memory", "64"}, + expConfig: harmonyconfig.GeneralConfig{ + NodeType: "validator", + NoStaking: false, + ShardID: -1, + IsArchival: false, + DataDir: "./", + TriesInMemory: 64, }, }, } diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 489fb3661..01f1b21e6 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -103,7 +103,6 @@ const ( maxFutureBlocks = 16 maxTimeFutureBlocks = 30 badBlockLimit = 10 - triesInMemory = 128 triesInRedis = 1000 shardCacheLimit = 10 commitsCacheLimit = 10 @@ -127,6 +126,7 @@ type CacheConfig struct { Disabled bool // Whether to disable trie write caching (archive node) TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk + TriesInMemory uint64 // Block number from the head stored in disk before exiting } type BlockChainImpl struct { @@ -223,12 +223,7 @@ func newBlockChainWithOptions( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus_engine.Engine, vmConfig vm.Config, options Options) (*BlockChainImpl, error) { - if cacheConfig == nil { - cacheConfig = &CacheConfig{ - TrieNodeLimit: 256 * 1024 * 1024, - TrieTimeLimit: 2 * time.Minute, - } - } + bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) receiptsCache, _ := lru.New(receiptsCacheLimit) @@ -1080,7 +1075,7 @@ func (bc *BlockChainImpl) Stop() { if !bc.cacheConfig.Disabled { triedb := bc.stateCache.TrieDB() - for _, offset := range []uint64{0, 1, triesInMemory - 1} { + for _, offset := range []uint64{0, 1, bc.cacheConfig.TriesInMemory - 1} { if number := bc.CurrentBlock().NumberU64(); number > offset { recent := bc.GetHeaderByNumber(number - offset) if recent != nil { @@ -1407,7 +1402,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) - if current := block.NumberU64(); current > triesInMemory { + if current := block.NumberU64(); current > bc.cacheConfig.TriesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( nodes, imgs = triedb.Size() @@ -1417,7 +1412,7 @@ func (bc *BlockChainImpl) WriteBlockWithState( triedb.Cap(limit - ethdb.IdealBatchSize) } // Find the next state trie we need to commit - header := bc.GetHeaderByNumber(current - triesInMemory) + header := bc.GetHeaderByNumber(current - bc.cacheConfig.TriesInMemory) if header != nil { chosen := header.Number().Uint64() @@ -1425,11 +1420,11 @@ func (bc *BlockChainImpl) WriteBlockWithState( if bc.gcproc > bc.cacheConfig.TrieTimeLimit { // If we're exceeding limits but haven't reached a large enough memory gap, // warn the user that the system is becoming unstable. - if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + if chosen < lastWrite+bc.cacheConfig.TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { utils.Logger().Info(). Dur("time", bc.gcproc). Dur("allowance", bc.cacheConfig.TrieTimeLimit). - Float64("optimum", float64(chosen-lastWrite)/triesInMemory). + Float64("optimum", float64(chosen-lastWrite)/float64(bc.cacheConfig.TriesInMemory)). Msg("State in memory for too long, committing") } // Flush an entire trie and restart the counters diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index c02f3bb7e..67c29f820 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -74,6 +74,7 @@ type GeneralConfig struct { TraceEnable bool EnablePruneBeaconChain bool RunElasticMode bool + TriesInMemory int } type TiKVConfig struct { diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index 8f9b18595..1be2c6841 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -3,6 +3,7 @@ package shardchain import ( "math/big" "sync" + "time" "github.com/harmony-one/harmony/core/state" harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony" @@ -104,6 +105,15 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c utils.Logger().Info(). Uint32("shardID", shardID). Msg("disable cache, running in archival mode") + } else { + cacheConfig = &core.CacheConfig{ + TrieNodeLimit: 256 * 1024 * 1024, + TrieTimeLimit: 2 * time.Minute, + TriesInMemory: 128, + } + if sc.harmonyconfig != nil { + cacheConfig.TriesInMemory = uint64(sc.harmonyconfig.General.TriesInMemory) + } } chainConfig := *sc.chainConfig diff --git a/rosetta/infra/harmony-mainnet.conf b/rosetta/infra/harmony-mainnet.conf index 8dc7909e3..a929eeeab 100644 --- a/rosetta/infra/harmony-mainnet.conf +++ b/rosetta/infra/harmony-mainnet.conf @@ -35,6 +35,7 @@ Version = "2.5.11" RunElasticMode = false ShardID = 0 TraceEnable = false + TriesInMemory = 128 [HTTP] AuthPort = 9501 diff --git a/rosetta/infra/harmony-pstn.conf b/rosetta/infra/harmony-pstn.conf index 0835f70d7..edb911f87 100644 --- a/rosetta/infra/harmony-pstn.conf +++ b/rosetta/infra/harmony-pstn.conf @@ -35,6 +35,7 @@ Version = "2.5.11" RunElasticMode = false ShardID = 0 TraceEnable = false + TriesInMemory = 128 [HTTP] AuthPort = 9501