PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: triesInMemory flag

Browse Source
pull/4366/head
Sun Hyuk Ahn 2 years ago committed by Casey Gardiner
parent e795f457d0
commit 0d994276c2
9 changed files with 97 additions and 48 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 8
      cmd/harmony/config_migrations.go
  2. 3
      cmd/harmony/default.go
  3. 14
      cmd/harmony/flags.go
  4. 88
      cmd/harmony/flags_test.go
  5. 19
      core/blockchain_impl.go
  6. 1
      internal/configs/harmony/harmony.go
  7. 10
      internal/shardchain/shardchains.go
  8. 1
      rosetta/infra/harmony-mainnet.conf
  9. 1
      rosetta/infra/harmony-pstn.conf

8
cmd/harmony/config_migrations.go
Unescape View File

@ -332,6 +332,14 @@ func init() {
return confTree
}
migrations["2.5.11"] = func(confTree *toml.Tree) *toml.Tree {
if confTree.Get("General.TriesInMemory") == nil {
confTree.Set("General.TriesInMemory", defaultConfig.General.TriesInMemory)
}
confTree.Set("Version", "2.5.12")
return confTree
}
// check that the latest version here is the same as in default.go
largestKey := getNextVersion(migrations)
if largestKey != tomlConfigVersion {

3
cmd/harmony/default.go
Unescape View File

@ -5,7 +5,7 @@ import (
nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
)
const tomlConfigVersion = "2.5.11"
const tomlConfigVersion = "2.5.12"
const (
defNetworkType = nodeconfig.Mainnet
@ -22,6 +22,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
IsOffline: false,
DataDir: "./",
TraceEnable: false,
TriesInMemory: 128,
},
Network: getDefaultNetworkConfig(defNetworkType),
P2P: harmonyconfig.P2pConfig{

14
cmd/harmony/flags.go
Unescape View File

@ -31,6 +31,7 @@ var (
legacyDataDirFlag,
taraceFlag,
triesInMemoryFlag,
}
dnsSyncFlags = []cli.Flag{
@ -322,6 +323,11 @@ var (
Usage: "indicates if full transaction tracing should be enabled",
DefValue: defaultConfig.General.TraceEnable,
}
triesInMemoryFlag = cli.IntFlag{
Name: "blockchain.tries_in_memory",
Usage: "number of blocks from header stored in disk before exiting",
DefValue: defaultConfig.General.TriesInMemory,
}
)
func getRootFlags() []cli.Flag {
@ -399,6 +405,14 @@ func applyGeneralFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig)
if cli.IsFlagChanged(cmd, isBackupFlag) {
config.General.IsBackup = cli.GetBoolFlagValue(cmd, isBackupFlag)
}
if cli.IsFlagChanged(cmd, triesInMemoryFlag) {
value := cli.GetIntFlagValue(cmd, triesInMemoryFlag)
if value <= 1 {
panic("Must number greater than 1 for txpool.accountslots")
}
config.General.TriesInMemory = value
}
}
// network flags

88
cmd/harmony/flags_test.go
Unescape View File

@ -35,11 +35,12 @@ func TestHarmonyFlags(t *testing.T) {
expConfig: harmonyconfig.HarmonyConfig{
Version: tomlConfigVersion,
General: harmonyconfig.GeneralConfig{
NodeType: "validator",
NoStaking: false,
ShardID: -1,
IsArchival: false,
DataDir: "./",
NodeType: "validator",
NoStaking: false,
ShardID: -1,
IsArchival: false,
DataDir: "./",
TriesInMemory: 128,
},
Network: harmonyconfig.NetworkConfig{
NetworkType: "mainnet",
@ -186,63 +187,80 @@ func TestGeneralFlags(t *testing.T) {
{
args: []string{},
expConfig: harmonyconfig.GeneralConfig{
NodeType: "validator",
NoStaking: false,
ShardID: -1,
IsArchival: false,
DataDir: "./",
NodeType: "validator",
NoStaking: false,
ShardID: -1,
IsArchival: false,
DataDir: "./",
TriesInMemory: 128,
},
},
{
args: []string{"--run", "explorer", "--run.legacy", "--run.shard=0",
"--run.archive=true", "--datadir=./.hmy"},
expConfig: harmonyconfig.GeneralConfig{
NodeType: "explorer",
NoStaking: true,
ShardID: 0,
IsArchival: true,
DataDir: "./.hmy",
NodeType: "explorer",
NoStaking: true,
ShardID: 0,
IsArchival: true,
DataDir: "./.hmy",
TriesInMemory: 128,
},
},
{
args: []string{"--node_type", "explorer", "--staking", "--shard_id", "0",
"--is_archival", "--db_dir", "./"},
expConfig: harmonyconfig.GeneralConfig{
NodeType: "explorer",
NoStaking: false,
ShardID: 0,
IsArchival: true,
DataDir: "./",
NodeType: "explorer",
NoStaking: false,
ShardID: 0,
IsArchival: true,
DataDir: "./",
TriesInMemory: 128,
},
},
{
args: []string{"--staking=false", "--is_archival=false"},
expConfig: harmonyconfig.GeneralConfig{
NodeType: "validator",
NoStaking: true,
ShardID: -1,
IsArchival: false,
DataDir: "./",
NodeType: "validator",
NoStaking: true,
ShardID: -1,
IsArchival: false,
DataDir: "./",
TriesInMemory: 128,
},
},
{
args: []string{"--run", "explorer", "--run.shard", "0"},
expConfig: harmonyconfig.GeneralConfig{
NodeType: "explorer",
NoStaking: false,
ShardID: 0,
IsArchival: false,
DataDir: "./",
NodeType: "explorer",
NoStaking: false,
ShardID: 0,
IsArchival: false,
DataDir: "./",
TriesInMemory: 128,
},
},
{
args: []string{"--run", "explorer", "--run.shard", "0", "--run.archive=false"},
expConfig: harmonyconfig.GeneralConfig{
NodeType: "explorer",
NoStaking: false,
ShardID: 0,
IsArchival: false,
DataDir: "./",
NodeType: "explorer",
NoStaking: false,
ShardID: 0,
IsArchival: false,
DataDir: "./",
TriesInMemory: 128,
},
},
{
args: []string{"--blockchain.tries_in_memory", "64"},
expConfig: harmonyconfig.GeneralConfig{
NodeType: "validator",
NoStaking: false,
ShardID: -1,
IsArchival: false,
DataDir: "./",
TriesInMemory: 64,
},
},
}

19
core/blockchain_impl.go
Unescape View File

@ -103,7 +103,6 @@ const (
maxFutureBlocks = 16
maxTimeFutureBlocks = 30
badBlockLimit = 10
triesInMemory = 128
triesInRedis = 1000
shardCacheLimit = 10
commitsCacheLimit = 10
@ -127,6 +126,7 @@ type CacheConfig struct {
Disabled bool // Whether to disable trie write caching (archive node)
TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
TriesInMemory uint64 // Block number from the head stored in disk before exiting
}
type BlockChainImpl struct {
@ -223,12 +223,7 @@ func newBlockChainWithOptions(
db ethdb.Database, stateCache state.Database, beaconChain BlockChain,
cacheConfig *CacheConfig, chainConfig *params.ChainConfig,
engine consensus_engine.Engine, vmConfig vm.Config, options Options) (*BlockChainImpl, error) {
if cacheConfig == nil {
cacheConfig = &CacheConfig{
TrieNodeLimit: 256 * 1024 * 1024,
TrieTimeLimit: 2 * time.Minute,
}
}
bodyCache, _ := lru.New(bodyCacheLimit)
bodyRLPCache, _ := lru.New(bodyCacheLimit)
receiptsCache, _ := lru.New(receiptsCacheLimit)
@ -1080,7 +1075,7 @@ func (bc *BlockChainImpl) Stop() {
if !bc.cacheConfig.Disabled {
triedb := bc.stateCache.TrieDB()
for _, offset := range []uint64{0, 1, triesInMemory - 1} {
for _, offset := range []uint64{0, 1, bc.cacheConfig.TriesInMemory - 1} {
if number := bc.CurrentBlock().NumberU64(); number > offset {
recent := bc.GetHeaderByNumber(number - offset)
if recent != nil {
@ -1407,7 +1402,7 @@ func (bc *BlockChainImpl) WriteBlockWithState(
triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
bc.triegc.Push(root, -int64(block.NumberU64()))
if current := block.NumberU64(); current > triesInMemory {
if current := block.NumberU64(); current > bc.cacheConfig.TriesInMemory {
// If we exceeded our memory allowance, flush matured singleton nodes to disk
var (
nodes, imgs = triedb.Size()
@ -1417,7 +1412,7 @@ func (bc *BlockChainImpl) WriteBlockWithState(
triedb.Cap(limit - ethdb.IdealBatchSize)
}
// Find the next state trie we need to commit
header := bc.GetHeaderByNumber(current - triesInMemory)
header := bc.GetHeaderByNumber(current - bc.cacheConfig.TriesInMemory)
if header != nil {
chosen := header.Number().Uint64()
@ -1425,11 +1420,11 @@ func (bc *BlockChainImpl) WriteBlockWithState(
if bc.gcproc > bc.cacheConfig.TrieTimeLimit {
// If we're exceeding limits but haven't reached a large enough memory gap,
// warn the user that the system is becoming unstable.
if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
if chosen < lastWrite+bc.cacheConfig.TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
utils.Logger().Info().
Dur("time", bc.gcproc).
Dur("allowance", bc.cacheConfig.TrieTimeLimit).
Float64("optimum", float64(chosen-lastWrite)/triesInMemory).
Float64("optimum", float64(chosen-lastWrite)/float64(bc.cacheConfig.TriesInMemory)).
Msg("State in memory for too long, committing")
}
// Flush an entire trie and restart the counters

1
internal/configs/harmony/harmony.go
Unescape View File

@ -74,6 +74,7 @@ type GeneralConfig struct {
TraceEnable bool
EnablePruneBeaconChain bool
RunElasticMode bool
TriesInMemory int
}
type TiKVConfig struct {

10
internal/shardchain/shardchains.go
Unescape View File

@ -3,6 +3,7 @@ package shardchain
import (
"math/big"
"sync"
"time"
"github.com/harmony-one/harmony/core/state"
harmonyconfig "github.com/harmony-one/harmony/internal/configs/harmony"
@ -104,6 +105,15 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c
utils.Logger().Info().
Uint32("shardID", shardID).
Msg("disable cache, running in archival mode")
} else {
cacheConfig = &core.CacheConfig{
TrieNodeLimit: 256 * 1024 * 1024,
TrieTimeLimit: 2 * time.Minute,
TriesInMemory: 128,
}
if sc.harmonyconfig != nil {
cacheConfig.TriesInMemory = uint64(sc.harmonyconfig.General.TriesInMemory)
}
}
chainConfig := *sc.chainConfig

1
rosetta/infra/harmony-mainnet.conf
Unescape View File

@ -35,6 +35,7 @@ Version = "2.5.11"
RunElasticMode = false
ShardID = 0
TraceEnable = false
TriesInMemory = 128
[HTTP]
AuthPort = 9501

1
rosetta/infra/harmony-pstn.conf
Unescape View File

@ -35,6 +35,7 @@ Version = "2.5.11"
RunElasticMode = false
ShardID = 0
TraceEnable = false
TriesInMemory = 128
[HTTP]
AuthPort = 9501

Write Preview
Loading…
Cancel
Save