From 51d2a00d24427217256ad4b1374394a3769dbb93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGheisMohammadi=E2=80=9D?= <36589218+GheisMohammadi@users.noreply.github.com> Date: Tue, 6 Feb 2024 21:32:09 +0800 Subject: [PATCH] refactor snapshot --- core/state/snapshot/account.go | 14 +-- core/state/snapshot/difflayer.go | 34 +++++-- core/state/snapshot/disklayer.go | 4 +- core/state/snapshot/generate.go | 2 +- core/state/snapshot/generate_test.go | 144 +++++++++++++-------------- core/state/snapshot/iterator_fast.go | 36 +++---- core/state/snapshot/snapshot.go | 32 ++++-- core/state/snapshot/snapshot_test.go | 13 ++- core/state/snapshot/utils.go | 9 +- 9 files changed, 158 insertions(+), 130 deletions(-) diff --git a/core/state/snapshot/account.go b/core/state/snapshot/account.go index b5634972a..3b027af7e 100644 --- a/core/state/snapshot/account.go +++ b/core/state/snapshot/account.go @@ -29,7 +29,7 @@ import ( // with a byte slice. This format can be used to represent full-consensus format // or slim-snapshot format which replaces the empty root and code hash as nil // byte slice. -type Account struct { +type SlimAccount struct { Nonce uint64 Balance *big.Int Root []byte @@ -37,8 +37,8 @@ type Account struct { } // SlimAccount converts a state.Account content into a slim snapshot account -func SlimAccount(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) Account { - slim := Account{ +func toSlimAccount(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) SlimAccount { + slim := SlimAccount{ Nonce: nonce, Balance: balance, } @@ -54,7 +54,7 @@ func SlimAccount(nonce uint64, balance *big.Int, root common.Hash, codehash []by // SlimAccountRLP converts a state.Account content into a slim snapshot // version RLP encoded. func SlimAccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) []byte { - data, err := rlp.EncodeToBytes(SlimAccount(nonce, balance, root, codehash)) + data, err := rlp.EncodeToBytes(toSlimAccount(nonce, balance, root, codehash)) if err != nil { panic(err) } @@ -63,10 +63,10 @@ func SlimAccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash [ // FullAccount decodes the data on the 'slim RLP' format and return // the consensus format account. -func FullAccount(data []byte) (Account, error) { - var account Account +func FullAccount(data []byte) (SlimAccount, error) { + var account SlimAccount if err := rlp.DecodeBytes(data, &account); err != nil { - return Account{}, err + return SlimAccount{}, err } if len(account.Root) == 0 { account.Root = types.EmptyRootHash[:] diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index f916a020e..e81b7eef6 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -17,11 +17,11 @@ package snapshot import ( + "bytes" "encoding/binary" "fmt" "math" "math/rand" - "sort" "sync" "sync/atomic" "time" @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" bloomfilter "github.com/holiman/bloomfilter/v2" + "golang.org/x/exp/slices" ) var ( @@ -103,7 +104,7 @@ type diffLayer struct { memory uint64 // Approximate guess as to how much memory we use root common.Hash // Root hash to which this snapshot diff belongs to - stale uint32 // Signals that the layer became stale (state progressed) + stale atomic.Bool // Signals that the layer became stale (state progressed) // destructSet is a very special helper marker. If an account is marked as // deleted, then it's recorded in this set. However it's allowed that an account @@ -267,12 +268,12 @@ func (dl *diffLayer) Parent() snapshot { // Stale return whether this layer has become stale (was flattened across) or if // it's still live. func (dl *diffLayer) Stale() bool { - return atomic.LoadUint32(&dl.stale) != 0 + return dl.stale.Load() } // Account directly retrieves the account associated with a particular hash in // the snapshot slim data format. -func (dl *diffLayer) Account(hash common.Hash) (*Account, error) { +func (dl *diffLayer) Account(hash common.Hash) (*SlimAccount, error) { data, err := dl.AccountRLP(hash) if err != nil { return nil, err @@ -280,7 +281,7 @@ func (dl *diffLayer) Account(hash common.Hash) (*Account, error) { if len(data) == 0 { // can be both nil and []byte{} return nil, nil } - account := new(Account) + account := new(SlimAccount) if err := rlp.DecodeBytes(data, account); err != nil { panic(err) } @@ -292,9 +293,14 @@ func (dl *diffLayer) Account(hash common.Hash) (*Account, error) { // // Note the returned account is not a copy, please don't modify it. func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { + // Check staleness before reaching further. + dl.lock.RLock() + if dl.Stale() { + dl.lock.RUnlock() + return nil, ErrSnapshotStale + } // Check the bloom filter first whether there's even a point in reaching into // all the maps in all the layers below - dl.lock.RLock() hit := dl.diffed.Contains(accountBloomHasher(hash)) if !hit { hit = dl.diffed.Contains(destructBloomHasher(hash)) @@ -361,6 +367,11 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro // Check the bloom filter first whether there's even a point in reaching into // all the maps in all the layers below dl.lock.RLock() + // Check staleness before reaching further. + if dl.Stale() { + dl.lock.RUnlock() + return nil, ErrSnapshotStale + } hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash}) if !hit { hit = dl.diffed.Contains(destructBloomHasher(accountHash)) @@ -449,7 +460,7 @@ func (dl *diffLayer) flatten() snapshot { // Before actually writing all our data to the parent, first ensure that the // parent hasn't been 'corrupted' by someone else already flattening into it - if atomic.SwapUint32(&parent.stale, 1) != 0 { + if parent.stale.Swap(true) { panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo } // Overwrite all the updated accounts blindly, merge the sorted list @@ -488,6 +499,11 @@ func (dl *diffLayer) flatten() snapshot { } } +// Cmp compares two hashes. +func Cmp(h common.Hash, other common.Hash) int { + return bytes.Compare(h[:], other[:]) +} + // AccountList returns a sorted list of all accounts in this diffLayer, including // the deleted ones. // @@ -514,7 +530,7 @@ func (dl *diffLayer) AccountList() []common.Hash { dl.accountList = append(dl.accountList, hash) } } - sort.Sort(hashes(dl.accountList)) + slices.SortFunc(dl.accountList, Cmp) dl.memory += uint64(len(dl.accountList) * common.HashLength) return dl.accountList } @@ -552,7 +568,7 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) for k := range storageMap { storageList = append(storageList, k) } - sort.Sort(hashes(storageList)) + slices.SortFunc(storageList, Cmp) dl.storageList[accountHash] = storageList dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) return storageList, destructed diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index 39879e143..ad651f897 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -65,7 +65,7 @@ func (dl *diskLayer) Stale() bool { // Account directly retrieves the account associated with a particular hash in // the snapshot slim data format. -func (dl *diskLayer) Account(hash common.Hash) (*Account, error) { +func (dl *diskLayer) Account(hash common.Hash) (*SlimAccount, error) { data, err := dl.AccountRLP(hash) if err != nil { return nil, err @@ -73,7 +73,7 @@ func (dl *diskLayer) Account(hash common.Hash) (*Account, error) { if len(data) == 0 { // can be both nil and []byte{} return nil, nil } - account := new(Account) + account := new(SlimAccount) if err := rlp.DecodeBytes(data, account); err != nil { panic(err) } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index e5376429b..99e291437 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -230,7 +230,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [ if origin == nil && !diskMore { stackTr := trie.NewStackTrie(nil) for i, key := range keys { - stackTr.TryUpdate(key, vals[i]) + stackTr.Update(key, vals[i]) } if gotRoot := stackTr.Hash(); gotRoot != root { return &proofResult{ diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index c5a6725f4..21f0e3097 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -48,9 +48,9 @@ func TestGeneration(t *testing.T) { var helper = newHelper() stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) - helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -82,16 +82,16 @@ func TestGenerateExistentState(t *testing.T) { var helper = newHelper() stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) - helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) root, snap := helper.CommitAndGenerate() @@ -157,18 +157,18 @@ func newHelper() *testHelper { } } -func (t *testHelper) addTrieAccount(acckey string, acc *Account) { +func (t *testHelper) addTrieAccount(acckey string, acc *SlimAccount) { val, _ := rlp.EncodeToBytes(acc) t.accTrie.Update([]byte(acckey), val) } -func (t *testHelper) addSnapAccount(acckey string, acc *Account) { +func (t *testHelper) addSnapAccount(acckey string, acc *SlimAccount) { val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte(acckey)) rawdb.WriteAccountSnapshot(t.diskdb, key, val) } -func (t *testHelper) addAccount(acckey string, acc *Account) { +func (t *testHelper) addAccount(acckey string, acc *SlimAccount) { t.addTrieAccount(acckey, acc) t.addSnapAccount(acckey, acc) } @@ -233,70 +233,70 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { func TestGenerateExistentStateWithWrongStorage(t *testing.T) { helper := newHelper() - // Account one, empty root but non-empty database - helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + // SlimAccount one, empty root but non-empty database + helper.addAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) - // Account two, non empty root but empty database + // SlimAccount two, non empty root but empty database stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-2", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Miss slots { - // Account three, non empty root but misses slots in the beginning + // SlimAccount three, non empty root but misses slots in the beginning helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-3", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) - // Account four, non empty root but misses slots in the middle + // SlimAccount four, non empty root but misses slots in the middle helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-4", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) - // Account five, non empty root but misses slots in the end + // SlimAccount five, non empty root but misses slots in the end helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-5", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) } // Wrong storage slots { - // Account six, non empty root but wrong slots in the beginning + // SlimAccount six, non empty root but wrong slots in the beginning helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-6", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) - // Account seven, non empty root but wrong slots in the middle + // SlimAccount seven, non empty root but wrong slots in the middle helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-7", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) - // Account eight, non empty root but wrong slots in the end + // SlimAccount eight, non empty root but wrong slots in the end helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-8", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) - // Account 9, non empty root but rotated slots + // SlimAccount 9, non empty root but rotated slots helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-9", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) } // Extra storage slots { - // Account 10, non empty root but extra slots in the beginning + // SlimAccount 10, non empty root but extra slots in the beginning helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-10", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) - // Account 11, non empty root but extra slots in the middle + // SlimAccount 11, non empty root but extra slots in the middle helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-11", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) - // Account 12, non empty root but extra slots in the end + // SlimAccount 12, non empty root but extra slots in the end helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-12", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) } @@ -336,25 +336,25 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { // Missing accounts, only in the trie { - helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning - helper.addTrieAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle - helper.addTrieAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End + helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning + helper.addTrieAccount("acc-4", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle + helper.addTrieAccount("acc-6", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End } // Wrong accounts { - helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) + helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-2", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) - helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addSnapAccount("acc-3", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) } // Extra accounts, only in the snap { - helper.addSnapAccount("acc-0", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning - helper.addSnapAccount("acc-5", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: common.Hex2Bytes("0x1234")}) // Middle - helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // after the end + helper.addSnapAccount("acc-0", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning + helper.addSnapAccount("acc-5", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: common.Hex2Bytes("0x1234")}) // Middle + helper.addSnapAccount("acc-7", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // after the end } root, snap := helper.CommitAndGenerate() @@ -383,9 +383,9 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { // without any storage slots to keep the test smaller. helper := newHelper() - helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 - helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 + helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 @@ -418,10 +418,10 @@ func TestGenerateMissingStorageTrie(t *testing.T) { helper := newHelper() stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 root := helper.Commit() @@ -452,10 +452,10 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { helper := newHelper() stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 root := helper.Commit() @@ -481,13 +481,13 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { func TestGenerateWithExtraAccounts(t *testing.T) { helper := newHelper() { - // Account one in the trie + // SlimAccount one in the trie stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, true, ) - acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -501,13 +501,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) { rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-5")), []byte("val-5")) } { - // Account two exists only in the snapshot + // SlimAccount two exists only in the snapshot stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, true, ) - acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte("acc-2")) rawdb.WriteAccountSnapshot(helper.diskdb, key, val) @@ -545,13 +545,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) { func TestGenerateWithManyExtraAccounts(t *testing.T) { helper := newHelper() { - // Account one in the trie + // SlimAccount one in the trie stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true, ) - acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} + acc := &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -565,7 +565,7 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { { // 100 accounts exist only in snapshot for i := 0; i < 1000; i++ { - acc := &Account{Balance: big.NewInt(int64(i)), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} + acc := &SlimAccount{Balance: big.NewInt(int64(i)), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte(fmt.Sprintf("acc-%d", i))) rawdb.WriteAccountSnapshot(helper.diskdb, key, val) @@ -599,7 +599,7 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { accountCheckRange = 3 helper := newHelper() { - acc := &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} + acc := &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val) @@ -633,7 +633,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { accountCheckRange = 3 helper := newHelper() { - acc := &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} + acc := &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) @@ -672,7 +672,7 @@ func TestGenerateFromEmptySnap(t *testing.T) { for i := 0; i < 400; i++ { stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount(fmt.Sprintf("acc-%d", i), - &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) } root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 @@ -709,7 +709,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) { for i := 0; i < 8; i++ { accKey := fmt.Sprintf("acc-%d", i) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true) - helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount(accKey, &SlimAccount{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) var moddedKeys []string var moddedVals []string for ii := 0; ii < 8; ii++ { @@ -801,11 +801,11 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-2", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addAccount("acc-3", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) @@ -836,11 +836,11 @@ func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) - helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) - helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) + helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) populateDangling(helper.diskdb) diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 1a042c7cd..0502d9cf8 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -22,6 +22,7 @@ import ( "sort" "github.com/ethereum/go-ethereum/common" + "golang.org/x/exp/slices" ) // weightedIterator is a iterator with an assigned weight. It is used to prioritise @@ -32,32 +33,25 @@ type weightedIterator struct { priority int } -// weightedIterators is a set of iterators implementing the sort.Interface. -type weightedIterators []*weightedIterator - -// Len implements sort.Interface, returning the number of active iterators. -func (its weightedIterators) Len() int { return len(its) } - -// Less implements sort.Interface, returning which of two iterators in the stack -// is before the other. -func (its weightedIterators) Less(i, j int) bool { +func (it *weightedIterator) Cmp(other *weightedIterator) int { // Order the iterators primarily by the account hashes - hashI := its[i].it.Hash() - hashJ := its[j].it.Hash() + hashI := it.it.Hash() + hashJ := other.it.Hash() switch bytes.Compare(hashI[:], hashJ[:]) { case -1: - return true + return -1 case 1: - return false + return 1 } // Same account/storage-slot in multiple layers, split by priority - return its[i].priority < its[j].priority -} - -// Swap implements sort.Interface, swapping two entries in the iterator stack. -func (its weightedIterators) Swap(i, j int) { - its[i], its[j] = its[j], its[i] + if it.priority < other.priority { + return -1 + } + if it.priority > other.priority { + return 1 + } + return 0 } // fastIterator is a more optimized multi-layer iterator which maintains a @@ -69,7 +63,7 @@ type fastIterator struct { curAccount []byte curSlot []byte - iterators weightedIterators + iterators []*weightedIterator initiated bool account bool fail error @@ -167,7 +161,7 @@ func (fi *fastIterator) init() { } } // Re-sort the entire list - sort.Sort(fi.iterators) + slices.SortFunc(fi.iterators, func(a, b *weightedIterator) int { return a.Cmp(b) }) fi.initiated = false } diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index d294ff185..9d6fd0a26 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "sync" - "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" @@ -103,7 +102,7 @@ type Snapshot interface { // Account directly retrieves the account associated with a particular hash in // the snapshot slim data format. - Account(hash common.Hash) (*Account, error) + Account(hash common.Hash) (*SlimAccount, error) // AccountRLP directly retrieves the account RLP associated with a particular // hash in the snapshot slim data format. @@ -206,8 +205,7 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root utils.Logger().Warn().Err(err).Msg("Snapshot maintenance disabled (syncing)") return snap, nil } - - // Create the building waiter if the background generation is allowed + // Create the building waiter iff the background generation is allowed if !config.NoBuild && !config.AsyncBuild { defer snap.waitBuild() } @@ -273,7 +271,7 @@ func (t *Tree) Disable() { case *diffLayer: // If the layer is a simple diff, simply mark as stale layer.lock.Lock() - atomic.StoreUint32(&layer.stale, 1) + layer.stale.Store(true) layer.lock.Unlock() default: @@ -565,7 +563,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer { // Ensure we don't delete too much data blindly (contract can be // huge). It's ok to flush, the root will go missing in case of a // crash and we'll detect and regenerate the snapshot. - if batch.ValueSize() > ethdb.IdealBatchSize { + if batch.ValueSize() > 64*1024*1024 { if err := batch.Write(); err != nil { utils.Logger().Fatal().Err(err).Msg("Failed to write storage deletions") } @@ -591,7 +589,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer { // Ensure we don't write too much data blindly. It's ok to flush, the // root will go missing in case of a crash and we'll detect and regen // the snapshot. - if batch.ValueSize() > ethdb.IdealBatchSize { + if batch.ValueSize() > 64*1024*1024 { if err := batch.Write(); err != nil { utils.Logger().Fatal().Err(err).Msg("Failed to write storage deletions") } @@ -727,7 +725,7 @@ func (t *Tree) Rebuild(root common.Hash) { case *diffLayer: // If the layer is a simple diff, simply mark as stale layer.lock.Lock() - atomic.StoreUint32(&layer.stale, 1) + layer.stale.Store(true) layer.lock.Unlock() default: @@ -853,3 +851,21 @@ func (t *Tree) DiskRoot() common.Hash { return t.diskRoot() } + +// Size returns the memory usage of the diff layers above the disk layer and the +// dirty nodes buffered in the disk layer. Currently, the implementation uses a +// special diff layer (the first) as an aggregator simulating a dirty buffer, so +// the second return will always be 0. However, this will be made consistent with +// the pathdb, which will require a second return. +func (t *Tree) Size() (diffs common.StorageSize, buf common.StorageSize) { + t.lock.RLock() + defer t.lock.RUnlock() + + var size common.StorageSize + for _, layer := range t.layers { + if layer, ok := layer.(*diffLayer); ok { + size += common.StorageSize(layer.memory) + } + } + return size, 0 +} diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index 951576bc8..f77c6a911 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -43,11 +43,10 @@ func randomHash() common.Hash { // randomAccount generates a random account and returns it RLP encoded. func randomAccount() []byte { - root := randomHash() - a := Account{ + a := &types.StateAccount{ Balance: big.NewInt(rand.Int63()), Nonce: rand.Uint64(), - Root: root[:], + Root: randomHash(), CodeHash: types.EmptyCodeHash[:], } data, _ := rlp.EncodeToBytes(a) @@ -118,7 +117,7 @@ func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { if err := snaps.Cap(common.HexToHash("0x02"), 0); err != nil { t.Fatalf("failed to merge diff layer onto disk: %v", err) } - // Since the base layer was modified, ensure that data retrieval on the external reference fail + // Since the base layer was modified, ensure that data retrievals on the external reference fail if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) } @@ -185,6 +184,10 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { // be returned with junk data. This version of the test retains the bottom diff // layer to check the usual mode of operation where the accumulator is retained. func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { + // Un-commenting this triggers the bloom set to be deterministic. The values below + // were used to trigger the flaw described in https://github.com/ethereum/go-ethereum/issues/27254. + // bloomDestructHasherOffset, bloomAccountHasherOffset, bloomStorageHasherOffset = 14, 24, 5 + // Create an empty base layer and a snapshot tree out of it base := &diskLayer{ diskdb: rawdb.NewMemoryDatabase(), @@ -461,7 +464,7 @@ func TestReadStateDuringFlattening(t *testing.T) { snap := snaps.Snapshot(common.HexToHash("0xa3")) // Register the testing hook to access the state after flattening - var result = make(chan *Account) + var result = make(chan *SlimAccount) snaps.onFlatten = func() { // Spin up a thread to read the account from the pre-created // snapshot handler. It's expected to be blocked. diff --git a/core/state/snapshot/utils.go b/core/state/snapshot/utils.go index 68c049a42..358c3f32d 100644 --- a/core/state/snapshot/utils.go +++ b/core/state/snapshot/utils.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/rlp" "github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/internal/utils" ) @@ -111,8 +110,8 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { baseRoot := rawdb.ReadSnapshotRoot(db) fmt.Printf("Disklayer: Root: %x\n", baseRoot) if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { - account := new(Account) - if err := rlp.DecodeBytes(data, account); err != nil { + account, err := FullAccount(data) + if err != nil { panic(err) } fmt.Printf("\taccount.nonce: %d\n", account.Nonce) @@ -142,8 +141,8 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { } fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) if data, ok := accounts[hash]; ok { - account := new(Account) - if err := rlp.DecodeBytes(data, account); err != nil { + account, err := FullAccount(data) + if err != nil { panic(err) } fmt.Printf("\taccount.nonce: %d\n", account.Nonce)