From 02f8f4963443ab51d9bb7f41564befed5f518be0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?=
<36589218+GheisMohammadi@users.noreply.github.com>
Date: Mon, 24 Apr 2023 06:38:08 +0800
Subject: [PATCH] initial version of offline state pruning
---
cmd/harmony/main.go | 7 +-
cmd/harmony/snapshotdb.go | 168 ++++++++++++
core/state/pruner/bloom.go | 131 +++++++++
core/state/pruner/pruner.go | 516 ++++++++++++++++++++++++++++++++++++
4 files changed, 821 insertions(+), 1 deletion(-)
create mode 100644 cmd/harmony/snapshotdb.go
create mode 100644 core/state/pruner/bloom.go
create mode 100644 core/state/pruner/pruner.go
diff --git a/cmd/harmony/main.go b/cmd/harmony/main.go
index 31332b2e6..05661e37b 100644
--- a/cmd/harmony/main.go
+++ b/cmd/harmony/main.go
@@ -106,7 +106,9 @@ func init() {
rootCmd.AddCommand(dumpConfigLegacyCmd)
rootCmd.AddCommand(dumpDBCmd)
rootCmd.AddCommand(inspectDBCmd)
-
+ rootCmd.AddCommand(snapshotCmd)
+ snapshotCmd.AddCommand(pruneStateCmd)
+ snapshotCmd.AddCommand(verifyStateCmd)
if err := registerRootCmdFlags(); err != nil {
os.Exit(2)
}
@@ -119,6 +121,9 @@ func init() {
if err := registerInspectionFlags(); err != nil {
os.Exit(2)
}
+ if err := registerSnapshotCmdFlags(); err != nil {
+ os.Exit(2)
+ }
}
func main() {
diff --git a/cmd/harmony/snapshotdb.go b/cmd/harmony/snapshotdb.go
new file mode 100644
index 000000000..6edf7d897
--- /dev/null
+++ b/cmd/harmony/snapshotdb.go
@@ -0,0 +1,168 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/ethereum/go-ethereum/trie"
+ "github.com/spf13/cobra"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/harmony-one/harmony/core/rawdb"
+ "github.com/harmony-one/harmony/core/state/pruner"
+ "github.com/harmony-one/harmony/core/state/snapshot"
+ "github.com/harmony-one/harmony/internal/cli"
+ "github.com/harmony-one/harmony/internal/utils"
+)
+
+var bloomFilterSizeFlag = cli.IntFlag{
+ Name: "bloomfilter.size",
+ Shorthand: "b",
+ Usage: "Megabytes of memory allocated to bloom-filter for pruning",
+ DefValue: 2048,
+}
+
+var stateRootFlag = cli.StringFlag{
+ Name: "stateroot",
+ Shorthand: "r",
+ Usage: "state root hash",
+ DefValue: "",
+}
+
+var snapshotCmd = &cobra.Command{
+ Use: "snapshot",
+ Short: "A set of commands based on the snapshot",
+ Long: "A set of commands based on the snapshot",
+ Run: func(cmd *cobra.Command, args []string) {
+ fmt.Println("Error: must also specify a subcommand (prune-state, verify, ...)")
+ },
+}
+
+var pruneStateCmd = &cobra.Command{
+ Use: "prune-state srcdb cachedir",
+ Short: "prune stale harmony state data based on snapshot",
+ Long: "will prune historical state data with the help of state snapshot. All trie nodes that do not belong to the specified version state will be deleted from the database",
+ Example: "harmony prune-state /srcDir/harmony_db_0 /prune_cache",
+ Args: cobra.RangeArgs(1, 2),
+ Run: func(cmd *cobra.Command, args []string) {
+ srcDBDir, cachedir := args[0], args[1]
+ bloomFilterSize := cli.GetIntFlagValue(cmd, bloomFilterSizeFlag)
+ stateRoot := cli.GetStringFlagValue(cmd, stateRootFlag)
+
+ chaindb, err := rawdb.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false)
+ if err != nil {
+ fmt.Println("open src db error:", err)
+ os.Exit(-1)
+ }
+ defer chaindb.Close()
+
+ prunerconfig := pruner.Config{
+ Datadir: ResolvePath(""),
+ Cachedir: ResolvePath(cachedir),
+ BloomSize: uint64(bloomFilterSize),
+ }
+ pruner, err := pruner.NewPruner(chaindb, prunerconfig)
+ if err != nil {
+ utils.Logger().Error().Err(err).Msg("Failed to open snapshot tree")
+ return
+ }
+
+ var targetRoot common.Hash
+ if len(stateRoot) >= 3 {
+ targetRoot, err = parseRoot(stateRoot)
+ if err != nil {
+ utils.Logger().Error().Err(err).Msg("Failed to resolve state root")
+ return
+ }
+ } else {
+ targetRoot = rawdb.ReadHeadBlockHash(chaindb)
+ }
+
+ if err = pruner.Prune(targetRoot); err != nil {
+ utils.Logger().Error().Err(err).Msg("Failed to prune state")
+ return
+ }
+
+ return
+ },
+}
+
+var verifyStateCmd = &cobra.Command{
+ Use: "verify-state srcdb",
+ Short: "Recalculate state hash based on snapshot for verification",
+ Long: "Recalculate state hash based on snapshot for verification",
+ Example: "harmony verify-state /srcDir/harmony_db_0",
+ Args: cobra.ExactArgs(1),
+ Run: func(cmd *cobra.Command, args []string) {
+ if len(args) > 1 {
+ fmt.Println("too many arguments")
+ return
+ }
+ srcDBDir := args[0]
+ chaindb, err := rawdb.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false)
+ if err != nil {
+ fmt.Println("open src db error:", err)
+ os.Exit(-1)
+ }
+ defer chaindb.Close()
+
+ headRoot := rawdb.ReadHeadBlockHash(chaindb)
+ stateRoot := cli.GetStringFlagValue(cmd, stateRootFlag)
+ var targetRoot common.Hash
+ if len(stateRoot) >= 3 {
+ var err error
+ if targetRoot, err = parseRoot(stateRoot); err != nil {
+ utils.Logger().Error().Err(err).Msg("Failed to resolve state root")
+ return
+ }
+ } else {
+ targetRoot = headRoot
+ }
+ snapconfig := snapshot.Config{
+ CacheSize: 256,
+ Recovery: false,
+ NoBuild: true,
+ AsyncBuild: false,
+ }
+ snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb), headRoot)
+ if err != nil {
+ utils.Logger().Error().Err(err).Msg("Failed to open snapshot tree")
+ return
+ }
+ if err := snaptree.Verify(targetRoot); err != nil {
+ utils.Logger().Error().Err(err).Interface("root", targetRoot).Msg("Failed to verify state")
+ return
+ }
+ utils.Logger().Info().Interface("root", targetRoot).Msg("Verified the state")
+ if err := snapshot.CheckDanglingStorage(chaindb); err != nil {
+ utils.Logger().Error().Err(err).Interface("root", targetRoot).Msg("Failed to check dangling storage")
+ }
+ return
+ },
+}
+
+func ResolvePath(filename string) string {
+ if filepath.IsAbs(filename) {
+ return filename
+ }
+ return filepath.Join(filepath.Dir("."), filename)
+}
+
+func parseRoot(input string) (common.Hash, error) {
+ var h common.Hash
+ if err := h.UnmarshalText([]byte(input)); err != nil {
+ return h, err
+ }
+ return h, nil
+}
+
+func registerSnapshotCmdFlags() error {
+ if err := cli.RegisterFlags(pruneStateCmd, []cli.Flag{bloomFilterSizeFlag, stateRootFlag}); err != nil {
+ return err
+ }
+ if err := cli.RegisterFlags(verifyStateCmd, []cli.Flag{stateRootFlag}); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/core/state/pruner/bloom.go b/core/state/pruner/bloom.go
new file mode 100644
index 000000000..513f507e8
--- /dev/null
+++ b/core/state/pruner/bloom.go
@@ -0,0 +1,131 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package pruner
+
+import (
+ "encoding/binary"
+ "errors"
+ "os"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/harmony-one/harmony/core/rawdb"
+ "github.com/ethereum/go-ethereum/log"
+ bloomfilter "github.com/holiman/bloomfilter/v2"
+)
+
+// stateBloomHasher is a wrapper around a byte blob to satisfy the interface API
+// requirements of the bloom library used. It's used to convert a trie hash or
+// contract code hash into a 64 bit mini hash.
+type stateBloomHasher []byte
+
+func (f stateBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
+func (f stateBloomHasher) Sum(b []byte) []byte { panic("not implemented") }
+func (f stateBloomHasher) Reset() { panic("not implemented") }
+func (f stateBloomHasher) BlockSize() int { panic("not implemented") }
+func (f stateBloomHasher) Size() int { return 8 }
+func (f stateBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) }
+
+// stateBloom is a bloom filter used during the state conversion(snapshot->state).
+// The keys of all generated entries will be recorded here so that in the pruning
+// stage the entries belong to the specific version can be avoided for deletion.
+//
+// The false-positive is allowed here. The "false-positive" entries means they
+// actually don't belong to the specific version but they are not deleted in the
+// pruning. The downside of the false-positive allowance is we may leave some "dangling"
+// nodes in the disk. But in practice the it's very unlike the dangling node is
+// state root. So in theory this pruned state shouldn't be visited anymore. Another
+// potential issue is for fast sync. If we do another fast sync upon the pruned
+// database, it's problematic which will stop the expansion during the syncing.
+//
+// After the entire state is generated, the bloom filter should be persisted into
+// the disk. It indicates the whole generation procedure is finished.
+type stateBloom struct {
+ bloom *bloomfilter.Filter
+}
+
+// newStateBloomWithSize creates a brand new state bloom for state generation.
+// The bloom filter will be created by the passing bloom filter size. According
+// to the https://hur.st/bloomfilter/?n=600000000&p=&m=2048MB&k=4, the parameters
+// are picked so that the false-positive rate for mainnet is low enough.
+func newStateBloomWithSize(size uint64) (*stateBloom, error) {
+ bloom, err := bloomfilter.New(size*1024*1024*8, 4)
+ if err != nil {
+ return nil, err
+ }
+ log.Info("Initialized state bloom", "size", common.StorageSize(float64(bloom.M()/8)))
+ return &stateBloom{bloom: bloom}, nil
+}
+
+// NewStateBloomFromDisk loads the state bloom from the given file.
+// In this case the assumption is held the bloom filter is complete.
+func NewStateBloomFromDisk(filename string) (*stateBloom, error) {
+ bloom, _, err := bloomfilter.ReadFile(filename)
+ if err != nil {
+ return nil, err
+ }
+ return &stateBloom{bloom: bloom}, nil
+}
+
+// Commit flushes the bloom filter content into the disk and marks the bloom
+// as complete.
+func (bloom *stateBloom) Commit(filename, tempname string) error {
+ // Write the bloom out into a temporary file
+ _, err := bloom.bloom.WriteFile(tempname)
+ if err != nil {
+ return err
+ }
+ // Ensure the file is synced to disk
+ f, err := os.OpenFile(tempname, os.O_RDWR, 0666)
+ if err != nil {
+ return err
+ }
+ if err := f.Sync(); err != nil {
+ f.Close()
+ return err
+ }
+ f.Close()
+
+ // Move the temporary file into it's final location
+ return os.Rename(tempname, filename)
+}
+
+// Put implements the KeyValueWriter interface. But here only the key is needed.
+func (bloom *stateBloom) Put(key []byte, value []byte) error {
+ // If the key length is not 32bytes, ensure it's contract code
+ // entry with new scheme.
+ if len(key) != common.HashLength {
+ isCode, codeKey := rawdb.IsCodeKey(key)
+ if !isCode {
+ return errors.New("invalid entry")
+ }
+ bloom.bloom.Add(stateBloomHasher(codeKey))
+ return nil
+ }
+ bloom.bloom.Add(stateBloomHasher(key))
+ return nil
+}
+
+// Delete removes the key from the key-value data store.
+func (bloom *stateBloom) Delete(key []byte) error { panic("not supported") }
+
+// Contain is the wrapper of the underlying contains function which
+// reports whether the key is contained.
+// - If it says yes, the key may be contained
+// - If it says no, the key is definitely not contained.
+func (bloom *stateBloom) Contain(key []byte) (bool, error) {
+ return bloom.bloom.Contains(stateBloomHasher(key)), nil
+}
diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go
new file mode 100644
index 000000000..9c767e326
--- /dev/null
+++ b/core/state/pruner/pruner.go
@@ -0,0 +1,516 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package pruner
+
+import (
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "math"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/harmony-one/harmony/core/rawdb"
+ "github.com/harmony-one/harmony/core/state/snapshot"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/trie"
+ "github.com/harmony-one/harmony/block"
+)
+
+const (
+ // stateBloomFilePrefix is the filename prefix of state bloom filter.
+ stateBloomFilePrefix = "statebloom"
+
+ // stateBloomFilePrefix is the filename suffix of state bloom filter.
+ stateBloomFileSuffix = "bf.gz"
+
+ // stateBloomFileTempSuffix is the filename suffix of state bloom filter
+ // while it is being written out to detect write aborts.
+ stateBloomFileTempSuffix = ".tmp"
+
+ // rangeCompactionThreshold is the minimal deleted entry number for
+ // triggering range compaction. It's a quite arbitrary number but just
+ // to avoid triggering range compaction because of small deletion.
+ rangeCompactionThreshold = 100000
+)
+
+// Config includes all the configurations for pruning.
+type Config struct {
+ Datadir string // The directory of the state database
+ Cachedir string // The directory of state clean cache
+ BloomSize uint64 // The Megabytes of memory allocated to bloom-filter
+}
+
+// Pruner is an offline tool to prune the stale state with the
+// help of the snapshot. The workflow of pruner is very simple:
+//
+// - iterate the snapshot, reconstruct the relevant state
+// - iterate the database, delete all other state entries which
+// don't belong to the target state and the genesis state
+//
+// It can take several hours(around 2 hours for mainnet) to finish
+// the whole pruning work. It's recommended to run this offline tool
+// periodically in order to release the disk usage and improve the
+// disk read performance to some extent.
+type Pruner struct {
+ config Config
+ chainHeader *block.Header
+ db ethdb.Database
+ stateBloom *stateBloom
+ snaptree *snapshot.Tree
+}
+
+// NewPruner creates the pruner instance.
+func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
+ headBlock := rawdb.ReadHeadBlock(db)
+ if headBlock == nil {
+ return nil, errors.New("failed to load head block")
+ }
+ snapconfig := snapshot.Config{
+ CacheSize: 256,
+ Recovery: false,
+ NoBuild: true,
+ AsyncBuild: false,
+ }
+ snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
+ if err != nil {
+ return nil, err // The relevant snapshot(s) might not exist
+ }
+ // Sanitize the bloom filter size if it's too small.
+ if config.BloomSize < 256 {
+ log.Warn("Sanitizing bloomfilter size", "provided(MB)", config.BloomSize, "updated(MB)", 256)
+ config.BloomSize = 256
+ }
+ stateBloom, err := newStateBloomWithSize(config.BloomSize)
+ if err != nil {
+ return nil, err
+ }
+ return &Pruner{
+ config: config,
+ chainHeader: headBlock.Header(),
+ db: db,
+ stateBloom: stateBloom,
+ snaptree: snaptree,
+ }, nil
+}
+
+func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, middleStateRoots map[common.Hash]struct{}, start time.Time) error {
+ // Delete all stale trie nodes in the disk. With the help of state bloom
+ // the trie nodes(and codes) belong to the active state will be filtered
+ // out. A very small part of stale tries will also be filtered because of
+ // the false-positive rate of bloom filter. But the assumption is held here
+ // that the false-positive is low enough(~0.05%). The probablity of the
+ // dangling node is the state root is super low. So the dangling nodes in
+ // theory will never ever be visited again.
+ var (
+ count int
+ size common.StorageSize
+ pstart = time.Now()
+ logged = time.Now()
+ batch = maindb.NewBatch()
+ iter = maindb.NewIterator(nil, nil)
+ )
+ for iter.Next() {
+ key := iter.Key()
+
+ // All state entries don't belong to specific state and genesis are deleted here
+ // - trie node
+ // - legacy contract code
+ // - new-scheme contract code
+ isCode, codeKey := rawdb.IsCodeKey(key)
+ if len(key) == common.HashLength || isCode {
+ checkKey := key
+ if isCode {
+ checkKey = codeKey
+ }
+ if _, exist := middleStateRoots[common.BytesToHash(checkKey)]; exist {
+ log.Debug("Forcibly delete the middle state roots", "hash", common.BytesToHash(checkKey))
+ } else {
+ if ok, err := stateBloom.Contain(checkKey); err != nil {
+ return err
+ } else if ok {
+ continue
+ }
+ }
+ count += 1
+ size += common.StorageSize(len(key) + len(iter.Value()))
+ batch.Delete(key)
+
+ var eta time.Duration // Realistically will never remain uninited
+ if done := binary.BigEndian.Uint64(key[:8]); done > 0 {
+ var (
+ left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
+ speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero
+ )
+ eta = time.Duration(left/speed) * time.Millisecond
+ }
+ if time.Since(logged) > 8*time.Second {
+ log.Info("Pruning state data", "nodes", count, "size", size,
+ "elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta))
+ logged = time.Now()
+ }
+ // Recreate the iterator after every batch commit in order
+ // to allow the underlying compactor to delete the entries.
+ if batch.ValueSize() >= ethdb.IdealBatchSize {
+ batch.Write()
+ batch.Reset()
+
+ iter.Release()
+ iter = maindb.NewIterator(nil, key)
+ }
+ }
+ }
+ if batch.ValueSize() > 0 {
+ batch.Write()
+ batch.Reset()
+ }
+ iter.Release()
+ log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart)))
+
+ // Pruning is done, now drop the "useless" layers from the snapshot.
+ // Firstly, flushing the target layer into the disk. After that all
+ // diff layers below the target will all be merged into the disk.
+ if err := snaptree.Cap(root, 0); err != nil {
+ return err
+ }
+ // Secondly, flushing the snapshot journal into the disk. All diff
+ // layers upon are dropped silently. Eventually the entire snapshot
+ // tree is converted into a single disk layer with the pruning target
+ // as the root.
+ if _, err := snaptree.Journal(root); err != nil {
+ return err
+ }
+ // Delete the state bloom, it marks the entire pruning procedure is
+ // finished. If any crashes or manual exit happens before this,
+ // `RecoverPruning` will pick it up in the next restarts to redo all
+ // the things.
+ os.RemoveAll(bloomPath)
+
+ // Start compactions, will remove the deleted data from the disk immediately.
+ // Note for small pruning, the compaction is skipped.
+ if count >= rangeCompactionThreshold {
+ cstart := time.Now()
+ for b := 0x00; b <= 0xf0; b += 0x10 {
+ var (
+ start = []byte{byte(b)}
+ end = []byte{byte(b + 0x10)}
+ )
+ if b == 0xf0 {
+ end = nil
+ }
+ log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart)))
+ if err := maindb.Compact(start, end); err != nil {
+ log.Error("Database compaction failed", "error", err)
+ return err
+ }
+ }
+ log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart)))
+ }
+ log.Info("State pruning successful", "pruned", size, "elapsed", common.PrettyDuration(time.Since(start)))
+ return nil
+}
+
+// Prune deletes all historical state nodes except the nodes belong to the
+// specified state version. If user doesn't specify the state version, use
+// the bottom-most snapshot diff layer as the target.
+func (p *Pruner) Prune(root common.Hash) error {
+ // If the state bloom filter is already committed previously,
+ // reuse it for pruning instead of generating a new one. It's
+ // mandatory because a part of state may already be deleted,
+ // the recovery procedure is necessary.
+ _, stateBloomRoot, err := findBloomFilter(p.config.Datadir)
+ if err != nil {
+ return err
+ }
+ if stateBloomRoot != (common.Hash{}) {
+ return RecoverPruning(p.config.Datadir, p.db, p.config.Cachedir)
+ }
+ // If the target state root is not specified, use the HEAD-127 as the
+ // target. The reason for picking it is:
+ // - in most of the normal cases, the related state is available
+ // - the probability of this layer being reorg is very low
+ var layers []snapshot.Snapshot
+ if root == (common.Hash{}) {
+ // Retrieve all snapshot layers from the current HEAD.
+ // In theory there are 128 difflayers + 1 disk layer present,
+ // so 128 diff layers are expected to be returned.
+ layers = p.snaptree.Snapshots(p.chainHeader.Root(), 128, true)
+ if len(layers) != 128 {
+ // Reject if the accumulated diff layers are less than 128. It
+ // means in most of normal cases, there is no associated state
+ // with bottom-most diff layer.
+ return fmt.Errorf("snapshot not old enough yet: need %d more blocks", 128-len(layers))
+ }
+ // Use the bottom-most diff layer as the target
+ root = layers[len(layers)-1].Root()
+ }
+ // Ensure the root is really present. The weak assumption
+ // is the presence of root can indicate the presence of the
+ // entire trie.
+ if !rawdb.HasLegacyTrieNode(p.db, root) {
+ // The special case is for clique based networks(rinkeby, goerli
+ // and some other private networks), it's possible that two
+ // consecutive blocks will have same root. In this case snapshot
+ // difflayer won't be created. So HEAD-127 may not paired with
+ // head-127 layer. Instead the paired layer is higher than the
+ // bottom-most diff layer. Try to find the bottom-most snapshot
+ // layer with state available.
+ //
+ // Note HEAD and HEAD-1 is ignored. Usually there is the associated
+ // state available, but we don't want to use the topmost state
+ // as the pruning target.
+ var found bool
+ for i := len(layers) - 2; i >= 2; i-- {
+ if rawdb.HasLegacyTrieNode(p.db, layers[i].Root()) {
+ root = layers[i].Root()
+ found = true
+ log.Info("Selecting middle-layer as the pruning target", "root", root, "depth", i)
+ break
+ }
+ }
+ if !found {
+ if len(layers) > 0 {
+ return errors.New("no snapshot paired state")
+ }
+ return fmt.Errorf("associated state[%x] is not present", root)
+ }
+ } else {
+ if len(layers) > 0 {
+ log.Info("Selecting bottom-most difflayer as the pruning target", "root", root, "height", p.chainHeader.Number().Uint64()-127)
+ } else {
+ log.Info("Selecting user-specified state as the pruning target", "root", root)
+ }
+ }
+ // Before start the pruning, delete the clean trie cache first.
+ // It's necessary otherwise in the next restart we will hit the
+ // deleted state root in the "clean cache" so that the incomplete
+ // state is picked for usage.
+ deleteCleanTrieCache(p.config.Cachedir)
+
+ // All the state roots of the middle layer should be forcibly pruned,
+ // otherwise the dangling state will be left.
+ middleRoots := make(map[common.Hash]struct{})
+ for _, layer := range layers {
+ if layer.Root() == root {
+ break
+ }
+ middleRoots[layer.Root()] = struct{}{}
+ }
+ // Traverse the target state, re-construct the whole state trie and
+ // commit to the given bloom filter.
+ start := time.Now()
+ if err := snapshot.GenerateTrie(p.snaptree, root, p.db, p.stateBloom); err != nil {
+ return err
+ }
+ // Traverse the genesis, put all genesis state entries into the
+ // bloom filter too.
+ if err := extractGenesis(p.db, p.stateBloom); err != nil {
+ return err
+ }
+ filterName := bloomFilterName(p.config.Datadir, root)
+
+ log.Info("Writing state bloom to disk", "name", filterName)
+ if err := p.stateBloom.Commit(filterName, filterName+stateBloomFileTempSuffix); err != nil {
+ return err
+ }
+ log.Info("State bloom filter committed", "name", filterName)
+ return prune(p.snaptree, root, p.db, p.stateBloom, filterName, middleRoots, start)
+}
+
+// RecoverPruning will resume the pruning procedure during the system restart.
+// This function is used in this case: user tries to prune state data, but the
+// system was interrupted midway because of crash or manual-kill. In this case
+// if the bloom filter for filtering active state is already constructed, the
+// pruning can be resumed. What's more if the bloom filter is constructed, the
+// pruning **has to be resumed**. Otherwise a lot of dangling nodes may be left
+// in the disk.
+func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) error {
+ stateBloomPath, stateBloomRoot, err := findBloomFilter(datadir)
+ if err != nil {
+ return err
+ }
+ if stateBloomPath == "" {
+ return nil // nothing to recover
+ }
+ headBlock := rawdb.ReadHeadBlock(db)
+ if headBlock == nil {
+ return errors.New("failed to load head block")
+ }
+ // Initialize the snapshot tree in recovery mode to handle this special case:
+ // - Users run the `prune-state` command multiple times
+ // - Neither these `prune-state` running is finished(e.g. interrupted manually)
+ // - The state bloom filter is already generated, a part of state is deleted,
+ // so that resuming the pruning here is mandatory
+ // - The state HEAD is rewound already because of multiple incomplete `prune-state`
+ // In this case, even the state HEAD is not exactly matched with snapshot, it
+ // still feasible to recover the pruning correctly.
+ snapconfig := snapshot.Config{
+ CacheSize: 256,
+ Recovery: true,
+ NoBuild: true,
+ AsyncBuild: false,
+ }
+ snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
+ if err != nil {
+ return err // The relevant snapshot(s) might not exist
+ }
+ stateBloom, err := NewStateBloomFromDisk(stateBloomPath)
+ if err != nil {
+ return err
+ }
+ log.Info("Loaded state bloom filter", "path", stateBloomPath)
+
+ // Before start the pruning, delete the clean trie cache first.
+ // It's necessary otherwise in the next restart we will hit the
+ // deleted state root in the "clean cache" so that the incomplete
+ // state is picked for usage.
+ deleteCleanTrieCache(trieCachePath)
+
+ // All the state roots of the middle layers should be forcibly pruned,
+ // otherwise the dangling state will be left.
+ var (
+ found bool
+ layers = snaptree.Snapshots(headBlock.Root(), 128, true)
+ middleRoots = make(map[common.Hash]struct{})
+ )
+ for _, layer := range layers {
+ if layer.Root() == stateBloomRoot {
+ found = true
+ break
+ }
+ middleRoots[layer.Root()] = struct{}{}
+ }
+ if !found {
+ log.Error("Pruning target state is not existent")
+ return errors.New("non-existent target state")
+ }
+ return prune(snaptree, stateBloomRoot, db, stateBloom, stateBloomPath, middleRoots, time.Now())
+}
+
+// extractGenesis loads the genesis state and commits all the state entries
+// into the given bloomfilter.
+func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
+ genesisHash := rawdb.ReadCanonicalHash(db, 0)
+ if genesisHash == (common.Hash{}) {
+ return errors.New("missing genesis hash")
+ }
+ genesis := rawdb.ReadBlock(db, genesisHash, 0)
+ if genesis == nil {
+ return errors.New("missing genesis block")
+ }
+ t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
+ if err != nil {
+ return err
+ }
+ accIter := t.NodeIterator(nil)
+ for accIter.Next(true) {
+ hash := accIter.Hash()
+
+ // Embedded nodes don't have hash.
+ if hash != (common.Hash{}) {
+ stateBloom.Put(hash.Bytes(), nil)
+ }
+ // If it's a leaf node, yes we are touching an account,
+ // dig into the storage trie further.
+ if accIter.Leaf() {
+ var acc types.StateAccount
+ if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil {
+ return err
+ }
+ if acc.Root != types.EmptyRootHash {
+ id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)
+ storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db))
+ if err != nil {
+ return err
+ }
+ storageIter := storageTrie.NodeIterator(nil)
+ for storageIter.Next(true) {
+ hash := storageIter.Hash()
+ if hash != (common.Hash{}) {
+ stateBloom.Put(hash.Bytes(), nil)
+ }
+ }
+ if storageIter.Error() != nil {
+ return storageIter.Error()
+ }
+ }
+ if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash.Bytes()) {
+ stateBloom.Put(acc.CodeHash, nil)
+ }
+ }
+ }
+ return accIter.Error()
+}
+
+func bloomFilterName(datadir string, hash common.Hash) string {
+ return filepath.Join(datadir, fmt.Sprintf("%s.%s.%s", stateBloomFilePrefix, hash.Hex(), stateBloomFileSuffix))
+}
+
+func isBloomFilter(filename string) (bool, common.Hash) {
+ filename = filepath.Base(filename)
+ if strings.HasPrefix(filename, stateBloomFilePrefix) && strings.HasSuffix(filename, stateBloomFileSuffix) {
+ return true, common.HexToHash(filename[len(stateBloomFilePrefix)+1 : len(filename)-len(stateBloomFileSuffix)-1])
+ }
+ return false, common.Hash{}
+}
+
+func findBloomFilter(datadir string) (string, common.Hash, error) {
+ var (
+ stateBloomPath string
+ stateBloomRoot common.Hash
+ )
+ if err := filepath.Walk(datadir, func(path string, info os.FileInfo, err error) error {
+ if info != nil && !info.IsDir() {
+ ok, root := isBloomFilter(path)
+ if ok {
+ stateBloomPath = path
+ stateBloomRoot = root
+ }
+ }
+ return nil
+ }); err != nil {
+ return "", common.Hash{}, err
+ }
+ return stateBloomPath, stateBloomRoot, nil
+}
+
+const warningLog = `
+
+WARNING!
+
+The clean trie cache is not found. Please delete it by yourself after the
+pruning. Remember don't start the Geth without deleting the clean trie cache
+otherwise the entire database may be damaged!
+
+Check the command description "harmony prune-state --help" for more details.
+`
+
+func deleteCleanTrieCache(path string) {
+ if !common.FileExist(path) {
+ log.Warn(warningLog)
+ return
+ }
+ os.RemoveAll(path)
+ log.Info("Deleted trie clean cache", "path", path)
+}