diff --git a/cmd/harmony/default.go b/cmd/harmony/default.go index 4cc20cfdf..cbedc9370 100644 --- a/cmd/harmony/default.go +++ b/cmd/harmony/default.go @@ -65,6 +65,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{ RateLimterEnabled: true, RequestsPerSecond: nodeconfig.DefaultRPCRateLimit, EvmCallTimeout: nodeconfig.DefaultEvmCallTimeout, + PreimagesEnabled: false, }, BLSKeys: harmonyconfig.BlsConfig{ KeyDir: "./.hmy/blskeys", @@ -149,6 +150,13 @@ var defaultRevertConfig = harmonyconfig.RevertConfig{ RevertTo: 0, } +var defaultPreimageConfig = harmonyconfig.PreimageConfig{ + ImportFrom: "", + ExportTo: "", + GenerateStart: 0, + GenerateEnd: 0, +} + var defaultLogContext = harmonyconfig.LogContext{ IP: "127.0.0.1", Port: 9000, @@ -291,6 +299,11 @@ func getDefaultRevertConfigCopy() harmonyconfig.RevertConfig { return config } +func getDefaultPreimageConfigCopy() harmonyconfig.PreimageConfig { + config := defaultPreimageConfig + return config +} + func getDefaultLogContextCopy() harmonyconfig.LogContext { config := defaultLogContext return config diff --git a/cmd/harmony/flags.go b/cmd/harmony/flags.go index acf53d8b3..46a1decb0 100644 --- a/cmd/harmony/flags.go +++ b/cmd/harmony/flags.go @@ -90,6 +90,7 @@ var ( rpcOptFlags = []cli.Flag{ rpcDebugEnabledFlag, + rpcPreimagesEnabledFlag, rpcEthRPCsEnabledFlag, rpcStakingRPCsEnabledFlag, rpcLegacyRPCsEnabledFlag, @@ -205,6 +206,13 @@ var ( revertBeforeFlag, } + preimageFlags = []cli.Flag{ + preimageImportFlag, + preimageExportFlag, + preimageGenerateStartFlag, + preimageGenerateEndFlag, + } + legacyRevertFlags = []cli.Flag{ legacyRevertBeaconFlag, legacyRevertBeforeFlag, @@ -370,6 +378,7 @@ func getRootFlags() []cli.Flag { flags = append(flags, sysFlags...) flags = append(flags, devnetFlags...) flags = append(flags, revertFlags...) + flags = append(flags, preimageFlags...) flags = append(flags, legacyMiscFlags...) flags = append(flags, prometheusFlags...) flags = append(flags, syncFlags...) @@ -827,6 +836,12 @@ var ( DefValue: defaultConfig.RPCOpt.DebugEnabled, Hidden: true, } + rpcPreimagesEnabledFlag = cli.BoolFlag{ + Name: "rpc.preimages", + Usage: "enable preimages export api", + DefValue: defaultConfig.RPCOpt.PreimagesEnabled, + Hidden: true, // not for end users + } rpcEthRPCsEnabledFlag = cli.BoolFlag{ Name: "rpc.eth", @@ -879,6 +894,9 @@ func applyRPCOptFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { if cli.IsFlagChanged(cmd, rpcDebugEnabledFlag) { config.RPCOpt.DebugEnabled = cli.GetBoolFlagValue(cmd, rpcDebugEnabledFlag) } + if cli.IsFlagChanged(cmd, rpcPreimagesEnabledFlag) { + config.RPCOpt.PreimagesEnabled = cli.GetBoolFlagValue(cmd, rpcPreimagesEnabledFlag) + } if cli.IsFlagChanged(cmd, rpcEthRPCsEnabledFlag) { config.RPCOpt.EthRPCsEnabled = cli.GetBoolFlagValue(cmd, rpcEthRPCsEnabledFlag) } @@ -1656,6 +1674,52 @@ func applyRevertFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { } } +var ( + preimageImportFlag = cli.StringFlag{ + Name: "preimage.import", + Usage: "Import pre-images from CSV file", + Hidden: true, + DefValue: defaultPreimageConfig.ImportFrom, + } + preimageExportFlag = cli.StringFlag{ + Name: "preimage.export", + Usage: "Export pre-images to CSV file", + Hidden: true, + DefValue: defaultPreimageConfig.ExportTo, + } + preimageGenerateStartFlag = cli.Uint64Flag{ + Name: "preimage.start", + Usage: "The block number from which pre-images are to be generated", + Hidden: true, + DefValue: defaultPreimageConfig.GenerateStart, + } + preimageGenerateEndFlag = cli.Uint64Flag{ + Name: "preimage.end", + Usage: "The block number upto and including which pre-images are to be generated", + Hidden: true, + DefValue: defaultPreimageConfig.GenerateEnd, + } +) + +func applyPreimageFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { + if cli.HasFlagsChanged(cmd, preimageFlags) { + cfg := getDefaultPreimageConfigCopy() + config.Preimage = &cfg + } + if cli.IsFlagChanged(cmd, preimageImportFlag) { + config.Preimage.ImportFrom = cli.GetStringFlagValue(cmd, preimageImportFlag) + } + if cli.IsFlagChanged(cmd, preimageExportFlag) { + config.Preimage.ExportTo = cli.GetStringFlagValue(cmd, preimageExportFlag) + } + if cli.IsFlagChanged(cmd, preimageGenerateStartFlag) { + config.Preimage.GenerateStart = cli.GetUint64FlagValue(cmd, preimageGenerateStartFlag) + } + if cli.IsFlagChanged(cmd, preimageGenerateEndFlag) { + config.Preimage.GenerateEnd = cli.GetUint64FlagValue(cmd, preimageGenerateEndFlag) + } +} + var ( legacyPortFlag = cli.IntFlag{ Name: "port", diff --git a/cmd/harmony/flags_test.go b/cmd/harmony/flags_test.go index 054c80421..bea0e0eab 100644 --- a/cmd/harmony/flags_test.go +++ b/cmd/harmony/flags_test.go @@ -92,6 +92,7 @@ func TestHarmonyFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 1000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, WS: harmonyconfig.WsConfig{ Enabled: true, @@ -752,6 +753,7 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 1000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, }, @@ -766,6 +768,7 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 1000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, }, @@ -780,6 +783,7 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 1000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, }, @@ -794,6 +798,7 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 1000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, }, @@ -808,6 +813,7 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 1000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, }, @@ -822,6 +828,7 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 1000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, }, @@ -836,6 +843,7 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 2000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, }, @@ -850,6 +858,7 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: false, RequestsPerSecond: 2000, EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, }, }, @@ -864,6 +873,22 @@ func TestRPCOptFlags(t *testing.T) { RateLimterEnabled: true, RequestsPerSecond: 1000, EvmCallTimeout: "10s", + PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled, + }, + }, + + { + args: []string{"--rpc.preimages"}, + expConfig: harmonyconfig.RpcOptConfig{ + DebugEnabled: false, + EthRPCsEnabled: true, + StakingRPCsEnabled: true, + LegacyRPCsEnabled: true, + RpcFilterFile: "./.hmy/rpc_filter.txt", + RateLimterEnabled: true, + RequestsPerSecond: 1000, + EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout, + PreimagesEnabled: true, }, }, } @@ -1509,6 +1534,70 @@ func TestRevertFlags(t *testing.T) { } } +func TestPreimageFlags(t *testing.T) { + tests := []struct { + args []string + expConfig *harmonyconfig.PreimageConfig + expErr error + }{ + { + args: []string{}, + expConfig: nil, + }, + { + args: []string{"--preimage.import", "/path/to/source.csv"}, + expConfig: &harmonyconfig.PreimageConfig{ + ImportFrom: "/path/to/source.csv", + ExportTo: defaultPreimageConfig.ExportTo, + GenerateStart: defaultPreimageConfig.GenerateStart, + GenerateEnd: defaultPreimageConfig.GenerateEnd, + }, + }, + { + args: []string{"--preimage.export", "/path/to/destination.csv"}, + expConfig: &harmonyconfig.PreimageConfig{ + ImportFrom: defaultPreimageConfig.ImportFrom, + ExportTo: "/path/to/destination.csv", + GenerateStart: defaultPreimageConfig.GenerateStart, + GenerateEnd: defaultPreimageConfig.GenerateEnd, + }, + }, + { + args: []string{"--preimage.start", "1"}, + expConfig: &harmonyconfig.PreimageConfig{ + ImportFrom: defaultPreimageConfig.ImportFrom, + ExportTo: defaultPreimageConfig.ExportTo, + GenerateStart: 1, + GenerateEnd: defaultPreimageConfig.GenerateEnd, + }, + }, + { + args: []string{"--preimage.end", "2"}, + expConfig: &harmonyconfig.PreimageConfig{ + ImportFrom: defaultPreimageConfig.ImportFrom, + ExportTo: defaultPreimageConfig.ExportTo, + GenerateStart: defaultPreimageConfig.GenerateStart, + GenerateEnd: 2, + }, + }, + } + for i, test := range tests { + ts := newFlagTestSuite(t, preimageFlags, applyPreimageFlags) + hc, err := ts.run(test.args) + + if assErr := assertError(err, test.expErr); assErr != nil { + t.Fatalf("Test %v: %v", i, assErr) + } + if err != nil || test.expErr != nil { + continue + } + if !reflect.DeepEqual(hc.Preimage, test.expConfig) { + t.Errorf("Test %v:\n\t%+v\n\t%+v", i, hc.Preimage, test.expConfig) + } + ts.tearDown() + } +} + func TestDNSSyncFlags(t *testing.T) { tests := []struct { args []string diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go index b70c41ec8..d90684aed 100644 --- a/cmd/harmony/main.go +++ b/cmd/harmony/main.go @@ -1,7 +1,9 @@ package main import ( + "encoding/csv" "fmt" + "io" "io/ioutil" "math/big" "math/rand" @@ -30,6 +32,7 @@ import ( ethCommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -45,6 +48,7 @@ import ( "github.com/harmony-one/harmony/common/ntp" "github.com/harmony-one/harmony/consensus" "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/hmy/downloader" "github.com/harmony-one/harmony/internal/cli" "github.com/harmony-one/harmony/internal/common" @@ -60,6 +64,7 @@ import ( "github.com/harmony-one/harmony/p2p" "github.com/harmony-one/harmony/shard" "github.com/harmony-one/harmony/webhooks" + prom "github.com/prometheus/client_golang/prometheus" ) // Host @@ -246,6 +251,7 @@ func applyRootFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) { applySysFlags(cmd, config) applyDevnetFlags(cmd, config) applyRevertFlags(cmd, config) + applyPreimageFlags(cmd, config) applyPrometheusFlags(cmd, config) applySyncFlags(cmd, config) applyShardDataFlags(cmd, config) @@ -375,6 +381,122 @@ func setupNodeAndRun(hc harmonyconfig.HarmonyConfig) { } } + //// code to handle pre-image export, import and generation + if hc.Preimage != nil { + if hc.Preimage.ImportFrom != "" { + reader, err := os.Open(hc.Preimage.ImportFrom) + if err != nil { + fmt.Println("Could not open file for reading", err) + os.Exit(1) + } + csvReader := csv.NewReader(reader) + chain := currentNode.Blockchain() + dbReader := chain.ChainDb() + imported := uint64(0) + for { + record, err := csvReader.Read() + if err == io.EOF { + fmt.Println("MyBlockNumber field missing, cannot proceed") + os.Exit(1) + } + if err != nil { + fmt.Println("Could not read from reader", err) + os.Exit(1) + } + // this means the address is a number + if blockNumber, err := strconv.ParseUint(record[1], 10, 64); err == nil { + if record[0] == "MyBlockNumber" { + // set this value in database, and prometheus, if needed + prev, err := rawdb.ReadPreimageImportBlock(dbReader) + if err != nil { + fmt.Println("No prior value found, overwriting") + } + if blockNumber > prev { + if rawdb.WritePreimageImportBlock(dbReader, blockNumber) != nil { + fmt.Println("Error saving last import block", err) + os.Exit(1) + } + // export blockNumber to prometheus + gauge := prom.NewGauge( + prom.GaugeOpts{ + Namespace: "hmy", + Subsystem: "blockchain", + Name: "last_preimage_import", + Help: "the last known block for which preimages were imported", + }, + ) + prometheus.PromRegistry().MustRegister( + gauge, + ) + gauge.Set(float64(blockNumber)) + } + // this is the last record + imported = blockNumber + break + } + } + key := ethCommon.HexToHash(record[0]) + value := ethCommon.Hex2Bytes(record[1]) + // validate + if crypto.Keccak256Hash(value) != key { + fmt.Println("Data mismatch: skipping", record) + continue + } + // add to database + rawdb.WritePreimages( + dbReader, map[ethCommon.Hash][]byte{ + key: value, + }, + ) + } + // now, at this point, we will have to generate missing pre-images + if imported != 0 { + genStart, _ := rawdb.ReadPreImageStartBlock(dbReader) + genEnd, _ := rawdb.ReadPreImageEndBlock(dbReader) + current := chain.CurrentBlock().NumberU64() + toGenStart, toGenEnd := core.FindMissingRange(imported, genStart, genEnd, current) + if toGenStart != 0 && toGenEnd != 0 { + if err := core.GeneratePreimages( + chain, toGenStart, toGenEnd, + ); err != nil { + fmt.Println("Error generating", err) + os.Exit(1) + } + } + } + os.Exit(0) + } else if exportPath := hc.Preimage.ExportTo; exportPath != "" { + if err := core.ExportPreimages( + currentNode.Blockchain(), + exportPath, + ); err != nil { + fmt.Println("Error exporting", err) + os.Exit(1) + } + os.Exit(0) + // both must be set + } else if hc.Preimage.GenerateStart > 0 && hc.Preimage.GenerateEnd > 0 { + chain := currentNode.Blockchain() + end := hc.Preimage.GenerateEnd + if number := chain.CurrentBlock().NumberU64(); number > end { + fmt.Printf( + "Cropping generate endpoint from %d to %d\n", + number, end, + ) + end = number + } + if err := core.GeneratePreimages( + chain, + hc.Preimage.GenerateStart, end, + ); err != nil { + fmt.Println("Error generating", err) + os.Exit(1) + } + os.Exit(0) + } + os.Exit(0) + } + startMsg := "==== New Harmony Node ====" if hc.General.NodeType == nodeTypeExplorer { startMsg = "==== New Explorer Node ====" diff --git a/core/blockchain.go b/core/blockchain.go index f7e956dbb..856b3b6f9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -346,6 +346,7 @@ type BlockChain interface { ) (status WriteStatus, err error) GetLeaderPubKeyFromCoinbase(h *block.Header) (*bls.PublicKeyWrapper, error) + CommitPreimages() error // ========== Only For Tikv Start ========== diff --git a/core/blockchain_impl.go b/core/blockchain_impl.go index 965dccd9a..37fa38582 100644 --- a/core/blockchain_impl.go +++ b/core/blockchain_impl.go @@ -149,6 +149,7 @@ var defaultCacheConfig = &CacheConfig{ TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 256, SnapshotWait: true, + Preimages: true, } type BlockChainImpl struct { @@ -236,7 +237,7 @@ func NewBlockChainWithOptions( // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum validator and -// Processor. +// Processor. As of Aug-23, this is only used by tests func NewBlockChain( db ethdb.Database, stateCache state.Database, beaconChain BlockChain, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus_engine.Engine, vmConfig vm.Config, @@ -366,6 +367,12 @@ func newBlockChainWithOptions( return nil, errors.WithMessage(err, "failed to build leader rotation meta") } + if cacheConfig.Preimages { + if _, _, err := rawdb.WritePreImageStartEndBlock(bc.ChainDb(), curHeader.NumberU64()+1, 0); err != nil { + return nil, errors.WithMessage(err, "failed to write pre-image start end blocks") + } + } + // Take ownership of this particular state go bc.update() return bc, nil @@ -1208,6 +1215,10 @@ func (bc *BlockChainImpl) Stop() { // Flush the collected preimages to disk if err := bc.stateCache.TrieDB().CommitPreimages(); err != nil { utils.Logger().Error().Interface("err", err).Msg("Failed to commit trie preimages") + } else { + if _, _, err := rawdb.WritePreImageStartEndBlock(bc.ChainDb(), 0, bc.CurrentBlock().NumberU64()); err != nil { + utils.Logger().Error().Interface("err", err).Msg("Failed to mark preimages end block") + } } // Ensure all live cached entries be saved into disk, so that we can skip // cache warmup when node restarts. @@ -3694,6 +3705,10 @@ func (bc *BlockChainImpl) InitTiKV(conf *harmonyconfig.TiKVConfig) { go bc.tikvCleanCache() } +func (bc *BlockChainImpl) CommitPreimages() error { + return bc.stateCache.TrieDB().CommitPreimages() +} + var ( leveldbErrSpec = "leveldb" tooManyOpenFilesErrStr = "Too many open files" diff --git a/core/blockchain_stub.go b/core/blockchain_stub.go index 5d83149a6..9b59c9699 100644 --- a/core/blockchain_stub.go +++ b/core/blockchain_stub.go @@ -439,3 +439,7 @@ func (a Stub) InitTiKV(conf *harmonyconfig.TiKVConfig) { func (a Stub) LeaderRotationMeta() (publicKeyBytes []byte, epoch, count, shifts uint64, err error) { return nil, 0, 0, 0, errors.Errorf("method LeaderRotationMeta not implemented for %s", a.Name) } + +func (a Stub) CommitPreimages() error { + return errors.Errorf("method CommitPreimages not implemented for %s", a.Name) +} diff --git a/core/epochchain.go b/core/epochchain.go index bcf00f5a8..7a3c40677 100644 --- a/core/epochchain.go +++ b/core/epochchain.go @@ -323,3 +323,9 @@ func (bc *EpochChain) IsSameLeaderAsPreviousBlock(block *types.Block) bool { func (bc *EpochChain) GetVMConfig() *vm.Config { return bc.vmConfig } + +func (bc *EpochChain) CommitPreimages() error { + // epoch chain just has last block, which does not have any txs + // so no pre-images here + return nil +} diff --git a/core/preimages.go b/core/preimages.go new file mode 100644 index 000000000..88d3dc40b --- /dev/null +++ b/core/preimages.go @@ -0,0 +1,218 @@ +package core + +import ( + "encoding/csv" + "fmt" + "os" + + ethCommon "github.com/ethereum/go-ethereum/common" + "github.com/harmony-one/harmony/api/service/prometheus" + "github.com/harmony-one/harmony/core/rawdb" + "github.com/harmony-one/harmony/core/state" + "github.com/harmony-one/harmony/core/types" + "github.com/harmony-one/harmony/internal/utils" + prom "github.com/prometheus/client_golang/prometheus" +) + +// ExportPreimages is public so `main.go` can call it directly` +func ExportPreimages(chain BlockChain, path string) error { + // set up csv + writer, err := os.Create(path) + if err != nil { + utils.Logger().Error(). + Msgf("unable to create file at %s due to %s", path, err) + return fmt.Errorf( + "unable to create file at %s due to %s", + path, err, + ) + } + csvWriter := csv.NewWriter(writer) + // open trie + block := chain.CurrentBlock() + statedb, err := chain.StateAt(block.Root()) + if err != nil { + utils.Logger().Error(). + Msgf( + "unable to open statedb at %s due to %s", + block.Root(), err, + ) + return fmt.Errorf( + "unable to open statedb at %x due to %s", + block.Root(), err, + ) + } + trie, err := statedb.Database().OpenTrie( + block.Root(), + ) + if err != nil { + utils.Logger().Error(). + Msgf( + "unable to open trie at %x due to %s", + block.Root(), err, + ) + return fmt.Errorf( + "unable to open trie at %x due to %s", + block.Root(), err, + ) + } + accountIterator := trie.NodeIterator(nil) + dbReader := chain.ChainDb() + for accountIterator.Next(true) { + // the leaf nodes of the MPT represent accounts + if accountIterator.Leaf() { + // the leaf key is the hashed address + hashed := accountIterator.LeafKey() + asHash := ethCommon.BytesToHash(hashed) + // obtain the corresponding address + preimage := rawdb.ReadPreimage( + dbReader, asHash, + ) + if len(preimage) == 0 { + utils.Logger().Warn(). + Msgf("Address not found for %x", asHash) + continue + } + address := ethCommon.BytesToAddress(preimage) + // key value format, so hash of value is first + csvWriter.Write([]string{ + fmt.Sprintf("%x", asHash.Bytes()), + fmt.Sprintf("%x", address.Bytes()), + }) + } + } + // lastly, write the block number + csvWriter.Write( + []string{ + "MyBlockNumber", + block.Number().String(), + }, + ) + // to disk + csvWriter.Flush() + if err := csvWriter.Error(); err != nil { + utils.Logger().Error(). + Msgf("unable to write csv due to %s", err) + return fmt.Errorf("unable to write csv due to %s", err) + } + writer.Close() + return nil +} + +func GeneratePreimages(chain BlockChain, start, end uint64) error { + if start < 2 { + return fmt.Errorf("too low starting point %d", start) + } + // fetch all the blocks, from start and end both inclusive + // then execute them - the execution will write the pre-images + // to disk and we are good to go + + // attempt to find a block number for which we have block and state + // with number < start + var startingState *state.DB + var startingBlock *types.Block + for i := start - 1; i > 0; i-- { + startingBlock = chain.GetBlockByNumber(i) + if startingBlock == nil { + // rewound too much in snapdb, so exit loop + // although this is only designed for s2/s3 nodes in mind + // which do not have such a snapdb + break + } + state, err := chain.StateAt(startingBlock.Root()) + if err == nil { + continue + } + startingState = state + break + } + if startingBlock == nil || startingState == nil { + return fmt.Errorf("no eligible starting block with state found") + } + + // now execute block T+1 based on starting state + for i := startingBlock.NumberU64() + 1; i <= end; i++ { + block := chain.GetBlockByNumber(i) + if block == nil { + // because we have startingBlock we must have all following + return fmt.Errorf("block %d not found", i) + } + _, _, _, _, _, _, endingState, err := chain.Processor().Process(block, startingState, *chain.GetVMConfig(), false) + if err == nil { + return fmt.Errorf("error executing block #%d: %s", i, err) + } + startingState = endingState + } + // force any pre-images in memory so far to go to disk, if they haven't already + if err := chain.CommitPreimages(); err != nil { + return fmt.Errorf("error committing preimages %s", err) + } + // save information about generated pre-images start and end nbs + var gauge1, gauge2 uint64 + var err error + if gauge1, gauge2, err = rawdb.WritePreImageStartEndBlock(chain.ChainDb(), startingBlock.NumberU64()+1, end); err != nil { + return fmt.Errorf("error writing pre-image gen blocks %s", err) + } + // add prometheus metrics as well + startGauge := prom.NewGauge( + prom.GaugeOpts{ + Namespace: "hmy", + Subsystem: "blockchain", + Name: "preimage_start", + Help: "the first block for which pre-image generation ran locally", + }, + ) + endGauge := prom.NewGauge( + prom.GaugeOpts{ + Namespace: "hmy", + Subsystem: "blockchain", + Name: "preimage_end", + Help: "the last block for which pre-image generation ran locally", + }, + ) + prometheus.PromRegistry().MustRegister( + startGauge, endGauge, + ) + startGauge.Set(float64(gauge1)) + endGauge.Set(float64(gauge2)) + return nil +} + +func FindMissingRange( + imported, start, end, current uint64, +) (uint64, uint64) { + // both are unset + if start == 0 && end == 0 { + if imported < current { + return imported + 1, current + } else { + return 0, 0 + } + } + // constraints: start <= end <= current + // in regular usage, we should have end == current + // however, with the GenerateFlag usage, we can have end < current + check1 := start <= end + if !check1 { + panic("Start > End") + } + check2 := end <= current + if !check2 { + panic("End > Current") + } + // imported can sit in any of the 4 ranges + if imported < start { + // both inclusive + return imported + 1, start - 1 + } + if imported < end { + return end + 1, current + } + if imported < current { + return imported + 1, current + } + // future data imported + if current < imported { + return 0, 0 + } + return 0, 0 +} diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index 72dbe94fb..81adabbfd 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -147,3 +147,65 @@ func DeleteValidatorCode(db ethdb.KeyValueWriter, hash common.Hash) { utils.Logger().Error().Err(err).Msg("Failed to delete validator code") } } + +func WritePreimageImportBlock(db ethdb.KeyValueWriter, number uint64) error { + return db.Put(preImageImportKey, encodeBlockNumber(number)) +} + +func ReadPreimageImportBlock(db ethdb.KeyValueReader) (uint64, error) { + val, err := db.Get(preImageImportKey) + if err != nil { + return 0, err + } + return decodeBlockNumber(val), nil +} + +func WritePreImageStartEndBlock( + db ethdb.KeyValueStore, + start uint64, + end uint64, +) ( + uint64, + uint64, + error, +) { + returnStart := start + returnEnd := end + if start != 0 { + existingStart, err := ReadPreImageStartBlock(db) + if err != nil || existingStart > start { + if err := db.Put(preImageGenStartKey, encodeBlockNumber(start)); err != nil { + return 0, 0, err + } else { + returnStart = existingStart + } + } + } + if end != 0 { + existingEnd, err := ReadPreImageEndBlock(db) + if err != nil || existingEnd < end { + if err := db.Put(preImageGenEndKey, encodeBlockNumber(end)); err != nil { + return 0, 0, err + } else { + returnEnd = existingEnd + } + } + } + return returnStart, returnEnd, nil +} + +func ReadPreImageStartBlock(db ethdb.KeyValueReader) (uint64, error) { + val, err := db.Get(preImageGenStartKey) + if err != nil { + return 0, err + } + return decodeBlockNumber(val), nil +} + +func ReadPreImageEndBlock(db ethdb.KeyValueReader) (uint64, error) { + val, err := db.Get(preImageGenEndKey) + if err != nil { + return 0, err + } + return decodeBlockNumber(val), nil +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 56147b51d..8251766ef 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -148,6 +148,10 @@ var ( BloomTrieIndexPrefix = []byte("bltIndex-") CliqueSnapshotPrefix = []byte("clique-") + + preImageImportKey = []byte("preimage-import") + preImageGenStartKey = []byte("preimage-gen-start") + preImageGenEndKey = []byte("preimage-gen-end") ) // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary diff --git a/internal/cli/flag.go b/internal/cli/flag.go index 755a6a73b..7d3a93c3d 100644 --- a/internal/cli/flag.go +++ b/internal/cli/flag.go @@ -71,8 +71,7 @@ type Int64Flag struct { Usage string Deprecated string Hidden bool - - DefValue int64 + DefValue int64 } // RegisterTo register the int flag to FlagSet @@ -81,6 +80,22 @@ func (f Int64Flag) RegisterTo(fs *pflag.FlagSet) error { return markHiddenOrDeprecated(fs, f.Name, f.Deprecated, f.Hidden) } +// Uint64Flag is the flag with uint64 value, used for block number configurations +type Uint64Flag struct { + Name string + Shorthand string + Usage string + Deprecated string + Hidden bool + DefValue uint64 +} + +// RegisterTo register the int flag to FlagSet +func (f Uint64Flag) RegisterTo(fs *pflag.FlagSet) error { + fs.Uint64P(f.Name, f.Shorthand, f.DefValue, f.Usage) + return markHiddenOrDeprecated(fs, f.Name, f.Deprecated, f.Hidden) +} + // StringSliceFlag is the flag with string slice value type StringSliceFlag struct { Name string @@ -143,6 +158,8 @@ func getFlagName(flag Flag) string { return f.Name case Int64Flag: return f.Name + case Uint64Flag: + return f.Name } return "" } diff --git a/internal/cli/parse.go b/internal/cli/parse.go index 326c923ad..13fc5bdce 100644 --- a/internal/cli/parse.go +++ b/internal/cli/parse.go @@ -76,6 +76,12 @@ func GetInt64FlagValue(cmd *cobra.Command, flag Int64Flag) int64 { return getInt64FlagValue(cmd.Flags(), flag) } +// GetInt64FlagValue get the int value for the given Int64Flag from the local flags of the +// cobra command. +func GetUint64FlagValue(cmd *cobra.Command, flag Uint64Flag) uint64 { + return getUint64FlagValue(cmd.Flags(), flag) +} + // GetIntPersistentFlagValue get the int value for the given IntFlag from the persistent // flags of the cobra command. func GetIntPersistentFlagValue(cmd *cobra.Command, flag IntFlag) int { @@ -100,6 +106,15 @@ func getInt64FlagValue(fs *pflag.FlagSet, flag Int64Flag) int64 { return val } +func getUint64FlagValue(fs *pflag.FlagSet, flag Uint64Flag) uint64 { + val, err := fs.GetUint64(flag.Name) + if err != nil { + handleParseError(err) + return 0 + } + return val +} + // GetStringSliceFlagValue get the string slice value for the given StringSliceFlag from // the local flags of the cobra command. func GetStringSliceFlagValue(cmd *cobra.Command, flag StringSliceFlag) []string { diff --git a/internal/configs/harmony/harmony.go b/internal/configs/harmony/harmony.go index 5aca663a8..2fcb200c4 100644 --- a/internal/configs/harmony/harmony.go +++ b/internal/configs/harmony/harmony.go @@ -36,6 +36,7 @@ type HarmonyConfig struct { DNSSync DnsSync ShardData ShardDataConfig GPO GasPriceOracleConfig + Preimage *PreimageConfig } func (hc HarmonyConfig) ToRPCServerConfig() nodeconfig.RPCServerConfig { @@ -84,6 +85,7 @@ func (hc HarmonyConfig) ToRPCServerConfig() nodeconfig.RPCServerConfig { WSPort: hc.WS.Port, WSAuthPort: hc.WS.AuthPort, DebugEnabled: hc.RPCOpt.DebugEnabled, + PreimagesEnabled: hc.RPCOpt.PreimagesEnabled, EthRPCsEnabled: hc.RPCOpt.EthRPCsEnabled, StakingRPCsEnabled: hc.RPCOpt.StakingRPCsEnabled, LegacyRPCsEnabled: hc.RPCOpt.LegacyRPCsEnabled, @@ -287,6 +289,7 @@ type RpcOptConfig struct { RateLimterEnabled bool // Enable Rate limiter for RPC RequestsPerSecond int // for RPC rate limiter EvmCallTimeout string // Timeout for eth_call + PreimagesEnabled bool // Expose preimage API } type DevnetConfig struct { @@ -303,6 +306,13 @@ type RevertConfig struct { RevertBefore int } +type PreimageConfig struct { + ImportFrom string + ExportTo string + GenerateStart uint64 + GenerateEnd uint64 +} + type LegacyConfig struct { WebHookConfig *string `toml:",omitempty"` TPBroadcastInvalidTxn *bool `toml:",omitempty"` diff --git a/internal/configs/node/config.go b/internal/configs/node/config.go index 5370a2e52..11dd5ba59 100644 --- a/internal/configs/node/config.go +++ b/internal/configs/node/config.go @@ -153,6 +153,7 @@ type RPCServerConfig struct { DebugEnabled bool + PreimagesEnabled bool EthRPCsEnabled bool StakingRPCsEnabled bool LegacyRPCsEnabled bool diff --git a/internal/shardchain/shardchains.go b/internal/shardchain/shardchains.go index 66cfad220..5da1b9186 100644 --- a/internal/shardchain/shardchains.go +++ b/internal/shardchain/shardchains.go @@ -100,8 +100,12 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c } } var cacheConfig *core.CacheConfig + // archival node if sc.disableCache[shardID] { - cacheConfig = &core.CacheConfig{Disabled: true} + cacheConfig = &core.CacheConfig{ + Disabled: true, + Preimages: true, + } utils.Logger().Info(). Uint32("shardID", shardID). Msg("disable cache, running in archival mode") @@ -110,6 +114,7 @@ func (sc *CollectionImpl) ShardChain(shardID uint32, options ...core.Options) (c TrieNodeLimit: 256, TrieTimeLimit: 2 * time.Minute, TriesInMemory: 128, + Preimages: true, } if sc.harmonyconfig != nil { cacheConfig.TriesInMemory = uint64(sc.harmonyconfig.General.TriesInMemory) diff --git a/rpc/preimages.go b/rpc/preimages.go new file mode 100644 index 000000000..d0ab56f38 --- /dev/null +++ b/rpc/preimages.go @@ -0,0 +1,29 @@ +package rpc + +import ( + "context" + + "github.com/harmony-one/harmony/core" + "github.com/harmony-one/harmony/eth/rpc" + "github.com/harmony-one/harmony/hmy" +) + +type PreimagesService struct { + hmy *hmy.Harmony +} + +// NewPreimagesAPI creates a new API for the RPC interface +func NewPreimagesAPI(hmy *hmy.Harmony, version string) rpc.API { + var service interface{} = &PreimagesService{hmy} + return rpc.API{ + Namespace: version, + Version: APIVersion, + Service: service, + Public: true, + } +} + +func (s *PreimagesService) Export(ctx context.Context, path string) error { + // these are by default not blocking + return core.ExportPreimages(s.hmy.BlockChain, path) +} diff --git a/rpc/rpc.go b/rpc/rpc.go index a8f1e121a..89a0d3532 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -71,6 +71,9 @@ func (n Version) Namespace() string { func StartServers(hmy *hmy.Harmony, apis []rpc.API, config nodeconfig.RPCServerConfig, rpcOpt harmony.RpcOptConfig) error { apis = append(apis, getAPIs(hmy, config)...) authApis := append(apis, getAuthAPIs(hmy, config.DebugEnabled, config.RateLimiterEnabled, config.RequestsPerSecond)...) + if rpcOpt.PreimagesEnabled { + authApis = append(authApis, NewPreimagesAPI(hmy, "preimages")) + } // load method filter from file (if exist) var rmf rpc.RpcMethodFilter rpcFilterFilePath := strings.TrimSpace(rpcOpt.RpcFilterFile)