PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor snapshot

Browse Source
feature/state_pruning_codes
“GheisMohammadi” 10 months ago
parent 6e65269336
commit 51d2a00d24
No known key found for this signature in database
GPG Key ID: 15073AED3829FE90
9 changed files with 158 additions and 130 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 14
      core/state/snapshot/account.go
  2. 34
      core/state/snapshot/difflayer.go
  3. 4
      core/state/snapshot/disklayer.go
  4. 2
      core/state/snapshot/generate.go
  5. 144
      core/state/snapshot/generate_test.go
  6. 36
      core/state/snapshot/iterator_fast.go
  7. 32
      core/state/snapshot/snapshot.go
  8. 13
      core/state/snapshot/snapshot_test.go
  9. 9
      core/state/snapshot/utils.go

14
core/state/snapshot/account.go
Unescape View File

@ -29,7 +29,7 @@ import (
// with a byte slice. This format can be used to represent full-consensus format // with a byte slice. This format can be used to represent full-consensus format
// or slim-snapshot format which replaces the empty root and code hash as nil // or slim-snapshot format which replaces the empty root and code hash as nil
// byte slice. // byte slice.
type Account struct { type SlimAccount struct {
Nonce uint64 Nonce uint64
Balance *big.Int Balance *big.Int
Root []byte Root []byte
@ -37,8 +37,8 @@ type Account struct {
} }
// SlimAccount converts a state.Account content into a slim snapshot account // SlimAccount converts a state.Account content into a slim snapshot account
func SlimAccount(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) Account { func toSlimAccount(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) SlimAccount {
slim := Account{ slim := SlimAccount{
Nonce: nonce, Nonce: nonce,
Balance: balance, Balance: balance,
} }
@ -54,7 +54,7 @@ func SlimAccount(nonce uint64, balance *big.Int, root common.Hash, codehash []by
// SlimAccountRLP converts a state.Account content into a slim snapshot // SlimAccountRLP converts a state.Account content into a slim snapshot
// version RLP encoded. // version RLP encoded.
func SlimAccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) []byte { func SlimAccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) []byte {
data, err := rlp.EncodeToBytes(SlimAccount(nonce, balance, root, codehash)) data, err := rlp.EncodeToBytes(toSlimAccount(nonce, balance, root, codehash))
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -63,10 +63,10 @@ func SlimAccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash [
// FullAccount decodes the data on the 'slim RLP' format and return // FullAccount decodes the data on the 'slim RLP' format and return
// the consensus format account. // the consensus format account.
func FullAccount(data []byte) (Account, error) { func FullAccount(data []byte) (SlimAccount, error) {
var account Account var account SlimAccount
if err := rlp.DecodeBytes(data, &account); err != nil { if err := rlp.DecodeBytes(data, &account); err != nil {
return Account{}, err return SlimAccount{}, err
} }
if len(account.Root) == 0 { if len(account.Root) == 0 {
account.Root = types.EmptyRootHash[:] account.Root = types.EmptyRootHash[:]

34
core/state/snapshot/difflayer.go
Unescape View File

@ -17,11 +17,11 @@
package snapshot package snapshot
import ( import (
"bytes"
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"math" "math"
"math/rand" "math/rand"
"sort"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
bloomfilter "github.com/holiman/bloomfilter/v2" bloomfilter "github.com/holiman/bloomfilter/v2"
"golang.org/x/exp/slices"
) )
var ( var (
@ -103,7 +104,7 @@ type diffLayer struct {
memory uint64 // Approximate guess as to how much memory we use memory uint64 // Approximate guess as to how much memory we use
root common.Hash // Root hash to which this snapshot diff belongs to root common.Hash // Root hash to which this snapshot diff belongs to
stale uint32 // Signals that the layer became stale (state progressed) stale atomic.Bool // Signals that the layer became stale (state progressed)
// destructSet is a very special helper marker. If an account is marked as // destructSet is a very special helper marker. If an account is marked as
// deleted, then it's recorded in this set. However it's allowed that an account // deleted, then it's recorded in this set. However it's allowed that an account
@ -267,12 +268,12 @@ func (dl *diffLayer) Parent() snapshot {
// Stale return whether this layer has become stale (was flattened across) or if // Stale return whether this layer has become stale (was flattened across) or if
// it's still live. // it's still live.
func (dl *diffLayer) Stale() bool { func (dl *diffLayer) Stale() bool {
return atomic.LoadUint32(&dl.stale) != 0 return dl.stale.Load()
} }
// Account directly retrieves the account associated with a particular hash in // Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format. // the snapshot slim data format.
func (dl *diffLayer) Account(hash common.Hash) (*Account, error) { func (dl *diffLayer) Account(hash common.Hash) (*SlimAccount, error) {
data, err := dl.AccountRLP(hash) data, err := dl.AccountRLP(hash)
if err != nil { if err != nil {
return nil, err return nil, err
@ -280,7 +281,7 @@ func (dl *diffLayer) Account(hash common.Hash) (*Account, error) {
if len(data) == 0 { // can be both nil and []byte{} if len(data) == 0 { // can be both nil and []byte{}
return nil, nil return nil, nil
} }
account := new(Account) account := new(SlimAccount)
if err := rlp.DecodeBytes(data, account); err != nil { if err := rlp.DecodeBytes(data, account); err != nil {
panic(err) panic(err)
} }
@ -292,9 +293,14 @@ func (dl *diffLayer) Account(hash common.Hash) (*Account, error) {
// //
// Note the returned account is not a copy, please don't modify it. // Note the returned account is not a copy, please don't modify it.
func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) {
// Check staleness before reaching further.
dl.lock.RLock()
if dl.Stale() {
dl.lock.RUnlock()
return nil, ErrSnapshotStale
}
// Check the bloom filter first whether there's even a point in reaching into // Check the bloom filter first whether there's even a point in reaching into
// all the maps in all the layers below // all the maps in all the layers below
dl.lock.RLock()
hit := dl.diffed.Contains(accountBloomHasher(hash)) hit := dl.diffed.Contains(accountBloomHasher(hash))
if !hit { if !hit {
hit = dl.diffed.Contains(destructBloomHasher(hash)) hit = dl.diffed.Contains(destructBloomHasher(hash))
@ -361,6 +367,11 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
// Check the bloom filter first whether there's even a point in reaching into // Check the bloom filter first whether there's even a point in reaching into
// all the maps in all the layers below // all the maps in all the layers below
dl.lock.RLock() dl.lock.RLock()
// Check staleness before reaching further.
if dl.Stale() {
dl.lock.RUnlock()
return nil, ErrSnapshotStale
}
hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash}) hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash})
if !hit { if !hit {
hit = dl.diffed.Contains(destructBloomHasher(accountHash)) hit = dl.diffed.Contains(destructBloomHasher(accountHash))
@ -449,7 +460,7 @@ func (dl *diffLayer) flatten() snapshot {
// Before actually writing all our data to the parent, first ensure that the // Before actually writing all our data to the parent, first ensure that the
// parent hasn't been 'corrupted' by someone else already flattening into it // parent hasn't been 'corrupted' by someone else already flattening into it
if atomic.SwapUint32(&parent.stale, 1) != 0 { if parent.stale.Swap(true) {
panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo
} }
// Overwrite all the updated accounts blindly, merge the sorted list // Overwrite all the updated accounts blindly, merge the sorted list
@ -488,6 +499,11 @@ func (dl *diffLayer) flatten() snapshot {
} }
} }
// Cmp compares two hashes.
func Cmp(h common.Hash, other common.Hash) int {
return bytes.Compare(h[:], other[:])
}
// AccountList returns a sorted list of all accounts in this diffLayer, including // AccountList returns a sorted list of all accounts in this diffLayer, including
// the deleted ones. // the deleted ones.
// //
@ -514,7 +530,7 @@ func (dl *diffLayer) AccountList() []common.Hash {
dl.accountList = append(dl.accountList, hash) dl.accountList = append(dl.accountList, hash)
} }
} }
sort.Sort(hashes(dl.accountList)) slices.SortFunc(dl.accountList, Cmp)
dl.memory += uint64(len(dl.accountList) * common.HashLength) dl.memory += uint64(len(dl.accountList) * common.HashLength)
return dl.accountList return dl.accountList
} }
@ -552,7 +568,7 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool)
for k := range storageMap { for k := range storageMap {
storageList = append(storageList, k) storageList = append(storageList, k)
} }
sort.Sort(hashes(storageList)) slices.SortFunc(storageList, Cmp)
dl.storageList[accountHash] = storageList dl.storageList[accountHash] = storageList
dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength)
return storageList, destructed return storageList, destructed

4
core/state/snapshot/disklayer.go
Unescape View File

@ -65,7 +65,7 @@ func (dl *diskLayer) Stale() bool {
// Account directly retrieves the account associated with a particular hash in // Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format. // the snapshot slim data format.
func (dl *diskLayer) Account(hash common.Hash) (*Account, error) { func (dl *diskLayer) Account(hash common.Hash) (*SlimAccount, error) {
data, err := dl.AccountRLP(hash) data, err := dl.AccountRLP(hash)
if err != nil { if err != nil {
return nil, err return nil, err
@ -73,7 +73,7 @@ func (dl *diskLayer) Account(hash common.Hash) (*Account, error) {
if len(data) == 0 { // can be both nil and []byte{} if len(data) == 0 { // can be both nil and []byte{}
return nil, nil return nil, nil
} }
account := new(Account) account := new(SlimAccount)
if err := rlp.DecodeBytes(data, account); err != nil { if err := rlp.DecodeBytes(data, account); err != nil {
panic(err) panic(err)
} }

2
core/state/snapshot/generate.go
Unescape View File

@ -230,7 +230,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [
if origin == nil && !diskMore { if origin == nil && !diskMore {
stackTr := trie.NewStackTrie(nil) stackTr := trie.NewStackTrie(nil)
for i, key := range keys { for i, key := range keys {
stackTr.TryUpdate(key, vals[i]) stackTr.Update(key, vals[i])
} }
if gotRoot := stackTr.Hash(); gotRoot != root { if gotRoot := stackTr.Hash(); gotRoot != root {
return &proofResult{ return &proofResult{

144
core/state/snapshot/generate_test.go
Unescape View File

@ -48,9 +48,9 @@ func TestGeneration(t *testing.T) {
var helper = newHelper() var helper = newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()})
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@ -82,16 +82,16 @@ func TestGenerateExistentState(t *testing.T) {
var helper = newHelper() var helper = newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()})
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
root, snap := helper.CommitAndGenerate() root, snap := helper.CommitAndGenerate()
@ -157,18 +157,18 @@ func newHelper() *testHelper {
} }
} }
func (t *testHelper) addTrieAccount(acckey string, acc *Account) { func (t *testHelper) addTrieAccount(acckey string, acc *SlimAccount) {
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
t.accTrie.Update([]byte(acckey), val) t.accTrie.Update([]byte(acckey), val)
} }
func (t *testHelper) addSnapAccount(acckey string, acc *Account) { func (t *testHelper) addSnapAccount(acckey string, acc *SlimAccount) {
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte(acckey)) key := hashData([]byte(acckey))
rawdb.WriteAccountSnapshot(t.diskdb, key, val) rawdb.WriteAccountSnapshot(t.diskdb, key, val)
} }
func (t *testHelper) addAccount(acckey string, acc *Account) { func (t *testHelper) addAccount(acckey string, acc *SlimAccount) {
t.addTrieAccount(acckey, acc) t.addTrieAccount(acckey, acc)
t.addSnapAccount(acckey, acc) t.addSnapAccount(acckey, acc)
} }
@ -233,70 +233,70 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
func TestGenerateExistentStateWithWrongStorage(t *testing.T) { func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
helper := newHelper() helper := newHelper()
// Account one, empty root but non-empty database // SlimAccount one, empty root but non-empty database
helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Account two, non empty root but empty database // SlimAccount two, non empty root but empty database
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-2", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
// Miss slots // Miss slots
{ {
// Account three, non empty root but misses slots in the beginning // SlimAccount three, non empty root but misses slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-3", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"})
// Account four, non empty root but misses slots in the middle // SlimAccount four, non empty root but misses slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-4", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"})
// Account five, non empty root but misses slots in the end // SlimAccount five, non empty root but misses slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-5", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"})
} }
// Wrong storage slots // Wrong storage slots
{ {
// Account six, non empty root but wrong slots in the beginning // SlimAccount six, non empty root but wrong slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-6", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"})
// Account seven, non empty root but wrong slots in the middle // SlimAccount seven, non empty root but wrong slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-7", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"})
// Account eight, non empty root but wrong slots in the end // SlimAccount eight, non empty root but wrong slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-8", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"})
// Account 9, non empty root but rotated slots // SlimAccount 9, non empty root but rotated slots
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-9", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"})
} }
// Extra storage slots // Extra storage slots
{ {
// Account 10, non empty root but extra slots in the beginning // SlimAccount 10, non empty root but extra slots in the beginning
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-10", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"})
// Account 11, non empty root but extra slots in the middle // SlimAccount 11, non empty root but extra slots in the middle
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-11", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"})
// Account 12, non empty root but extra slots in the end // SlimAccount 12, non empty root but extra slots in the end
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-12", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"})
} }
@ -336,25 +336,25 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
// Missing accounts, only in the trie // Missing accounts, only in the trie
{ {
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning
helper.addTrieAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle helper.addTrieAccount("acc-4", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle
helper.addTrieAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End helper.addTrieAccount("acc-6", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End
} }
// Wrong accounts // Wrong accounts
{ {
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) helper.addSnapAccount("acc-2", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")})
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.addSnapAccount("acc-3", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()})
} }
// Extra accounts, only in the snap // Extra accounts, only in the snap
{ {
helper.addSnapAccount("acc-0", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning helper.addSnapAccount("acc-0", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning
helper.addSnapAccount("acc-5", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: common.Hex2Bytes("0x1234")}) // Middle helper.addSnapAccount("acc-5", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: common.Hex2Bytes("0x1234")}) // Middle
helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // after the end helper.addSnapAccount("acc-7", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // after the end
} }
root, snap := helper.CommitAndGenerate() root, snap := helper.CommitAndGenerate()
@ -383,9 +383,9 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
// without any storage slots to keep the test smaller. // without any storage slots to keep the test smaller.
helper := newHelper() helper := newHelper()
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
@ -418,10 +418,10 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
helper := newHelper() helper := newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
root := helper.Commit() root := helper.Commit()
@ -452,10 +452,10 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
helper := newHelper() helper := newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
root := helper.Commit() root := helper.Commit()
@ -481,13 +481,13 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
func TestGenerateWithExtraAccounts(t *testing.T) { func TestGenerateWithExtraAccounts(t *testing.T) {
helper := newHelper() helper := newHelper()
{ {
// Account one in the trie // SlimAccount one in the trie
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")),
[]string{"key-1", "key-2", "key-3", "key-4", "key-5"}, []string{"key-1", "key-2", "key-3", "key-4", "key-5"},
[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, []string{"val-1", "val-2", "val-3", "val-4", "val-5"},
true, true,
) )
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} acc := &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
@ -501,13 +501,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-5")), []byte("val-5")) rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-5")), []byte("val-5"))
} }
{ {
// Account two exists only in the snapshot // SlimAccount two exists only in the snapshot
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")),
[]string{"key-1", "key-2", "key-3", "key-4", "key-5"}, []string{"key-1", "key-2", "key-3", "key-4", "key-5"},
[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, []string{"val-1", "val-2", "val-3", "val-4", "val-5"},
true, true,
) )
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} acc := &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte("acc-2")) key := hashData([]byte("acc-2"))
rawdb.WriteAccountSnapshot(helper.diskdb, key, val) rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
@ -545,13 +545,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
func TestGenerateWithManyExtraAccounts(t *testing.T) { func TestGenerateWithManyExtraAccounts(t *testing.T) {
helper := newHelper() helper := newHelper()
{ {
// Account one in the trie // SlimAccount one in the trie
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")),
[]string{"key-1", "key-2", "key-3"}, []string{"key-1", "key-2", "key-3"},
[]string{"val-1", "val-2", "val-3"}, []string{"val-1", "val-2", "val-3"},
true, true,
) )
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()} acc := &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
@ -565,7 +565,7 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
{ {
// 100 accounts exist only in snapshot // 100 accounts exist only in snapshot
for i := 0; i < 1000; i++ { for i := 0; i < 1000; i++ {
acc := &Account{Balance: big.NewInt(int64(i)), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} acc := &SlimAccount{Balance: big.NewInt(int64(i)), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte(fmt.Sprintf("acc-%d", i))) key := hashData([]byte(fmt.Sprintf("acc-%d", i)))
rawdb.WriteAccountSnapshot(helper.diskdb, key, val) rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
@ -599,7 +599,7 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
accountCheckRange = 3 accountCheckRange = 3
helper := newHelper() helper := newHelper()
{ {
acc := &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} acc := &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val)
helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val) helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val)
@ -633,7 +633,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
accountCheckRange = 3 accountCheckRange = 3
helper := newHelper() helper := newHelper()
{ {
acc := &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()} acc := &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc) val, _ := rlp.EncodeToBytes(acc)
helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val)
@ -672,7 +672,7 @@ func TestGenerateFromEmptySnap(t *testing.T) {
for i := 0; i < 400; i++ { for i := 0; i < 400; i++ {
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount(fmt.Sprintf("acc-%d", i), helper.addTrieAccount(fmt.Sprintf("acc-%d", i),
&Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
} }
root, snap := helper.CommitAndGenerate() root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4
@ -709,7 +709,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) {
for i := 0; i < 8; i++ { for i := 0; i < 8; i++ {
accKey := fmt.Sprintf("acc-%d", i) accKey := fmt.Sprintf("acc-%d", i)
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true)
helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount(accKey, &SlimAccount{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
var moddedKeys []string var moddedKeys []string
var moddedVals []string var moddedVals []string
for ii := 0; ii < 8; ii++ { for ii := 0; ii < 8; ii++ {
@ -801,11 +801,11 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper() var helper = newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-2", &SlimAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()})
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addAccount("acc-3", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
@ -836,11 +836,11 @@ func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper() var helper = newHelper()
stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-1", &SlimAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-2", &SlimAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()})
helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) helper.addTrieAccount("acc-3", &SlimAccount{Balance: big.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
populateDangling(helper.diskdb) populateDangling(helper.diskdb)

36
core/state/snapshot/iterator_fast.go
Unescape View File

@ -22,6 +22,7 @@ import (
"sort" "sort"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"golang.org/x/exp/slices"
) )
// weightedIterator is a iterator with an assigned weight. It is used to prioritise // weightedIterator is a iterator with an assigned weight. It is used to prioritise
@ -32,32 +33,25 @@ type weightedIterator struct {
priority int priority int
} }
// weightedIterators is a set of iterators implementing the sort.Interface. func (it *weightedIterator) Cmp(other *weightedIterator) int {
type weightedIterators []*weightedIterator
// Len implements sort.Interface, returning the number of active iterators.
func (its weightedIterators) Len() int { return len(its) }
// Less implements sort.Interface, returning which of two iterators in the stack
// is before the other.
func (its weightedIterators) Less(i, j int) bool {
// Order the iterators primarily by the account hashes // Order the iterators primarily by the account hashes
hashI := its[i].it.Hash() hashI := it.it.Hash()
hashJ := its[j].it.Hash() hashJ := other.it.Hash()
switch bytes.Compare(hashI[:], hashJ[:]) { switch bytes.Compare(hashI[:], hashJ[:]) {
case -1: case -1:
return true return -1
case 1: case 1:
return false return 1
} }
// Same account/storage-slot in multiple layers, split by priority // Same account/storage-slot in multiple layers, split by priority
return its[i].priority < its[j].priority if it.priority < other.priority {
} return -1
}
// Swap implements sort.Interface, swapping two entries in the iterator stack. if it.priority > other.priority {
func (its weightedIterators) Swap(i, j int) { return 1
its[i], its[j] = its[j], its[i] }
return 0
} }
// fastIterator is a more optimized multi-layer iterator which maintains a // fastIterator is a more optimized multi-layer iterator which maintains a
@ -69,7 +63,7 @@ type fastIterator struct {
curAccount []byte curAccount []byte
curSlot []byte curSlot []byte
iterators weightedIterators iterators []*weightedIterator
initiated bool initiated bool
account bool account bool
fail error fail error
@ -167,7 +161,7 @@ func (fi *fastIterator) init() {
} }
} }
// Re-sort the entire list // Re-sort the entire list
sort.Sort(fi.iterators) slices.SortFunc(fi.iterators, func(a, b *weightedIterator) int { return a.Cmp(b) })
fi.initiated = false fi.initiated = false
} }

32
core/state/snapshot/snapshot.go
Unescape View File

@ -22,7 +22,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"sync" "sync"
"sync/atomic"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
@ -103,7 +102,7 @@ type Snapshot interface {
// Account directly retrieves the account associated with a particular hash in // Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format. // the snapshot slim data format.
Account(hash common.Hash) (*Account, error) Account(hash common.Hash) (*SlimAccount, error)
// AccountRLP directly retrieves the account RLP associated with a particular // AccountRLP directly retrieves the account RLP associated with a particular
// hash in the snapshot slim data format. // hash in the snapshot slim data format.
@ -206,8 +205,7 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root
utils.Logger().Warn().Err(err).Msg("Snapshot maintenance disabled (syncing)") utils.Logger().Warn().Err(err).Msg("Snapshot maintenance disabled (syncing)")
return snap, nil return snap, nil
} }
// Create the building waiter iff the background generation is allowed
// Create the building waiter if the background generation is allowed
if !config.NoBuild && !config.AsyncBuild { if !config.NoBuild && !config.AsyncBuild {
defer snap.waitBuild() defer snap.waitBuild()
} }
@ -273,7 +271,7 @@ func (t *Tree) Disable() {
case *diffLayer: case *diffLayer:
// If the layer is a simple diff, simply mark as stale // If the layer is a simple diff, simply mark as stale
layer.lock.Lock() layer.lock.Lock()
atomic.StoreUint32(&layer.stale, 1) layer.stale.Store(true)
layer.lock.Unlock() layer.lock.Unlock()
default: default:
@ -565,7 +563,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
// Ensure we don't delete too much data blindly (contract can be // Ensure we don't delete too much data blindly (contract can be
// huge). It's ok to flush, the root will go missing in case of a // huge). It's ok to flush, the root will go missing in case of a
// crash and we'll detect and regenerate the snapshot. // crash and we'll detect and regenerate the snapshot.
if batch.ValueSize() > ethdb.IdealBatchSize { if batch.ValueSize() > 64*1024*1024 {
if err := batch.Write(); err != nil { if err := batch.Write(); err != nil {
utils.Logger().Fatal().Err(err).Msg("Failed to write storage deletions") utils.Logger().Fatal().Err(err).Msg("Failed to write storage deletions")
} }
@ -591,7 +589,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
// Ensure we don't write too much data blindly. It's ok to flush, the // Ensure we don't write too much data blindly. It's ok to flush, the
// root will go missing in case of a crash and we'll detect and regen // root will go missing in case of a crash and we'll detect and regen
// the snapshot. // the snapshot.
if batch.ValueSize() > ethdb.IdealBatchSize { if batch.ValueSize() > 64*1024*1024 {
if err := batch.Write(); err != nil { if err := batch.Write(); err != nil {
utils.Logger().Fatal().Err(err).Msg("Failed to write storage deletions") utils.Logger().Fatal().Err(err).Msg("Failed to write storage deletions")
} }
@ -727,7 +725,7 @@ func (t *Tree) Rebuild(root common.Hash) {
case *diffLayer: case *diffLayer:
// If the layer is a simple diff, simply mark as stale // If the layer is a simple diff, simply mark as stale
layer.lock.Lock() layer.lock.Lock()
atomic.StoreUint32(&layer.stale, 1) layer.stale.Store(true)
layer.lock.Unlock() layer.lock.Unlock()
default: default:
@ -853,3 +851,21 @@ func (t *Tree) DiskRoot() common.Hash {
return t.diskRoot() return t.diskRoot()
} }
// Size returns the memory usage of the diff layers above the disk layer and the
// dirty nodes buffered in the disk layer. Currently, the implementation uses a
// special diff layer (the first) as an aggregator simulating a dirty buffer, so
// the second return will always be 0. However, this will be made consistent with
// the pathdb, which will require a second return.
func (t *Tree) Size() (diffs common.StorageSize, buf common.StorageSize) {
t.lock.RLock()
defer t.lock.RUnlock()
var size common.StorageSize
for _, layer := range t.layers {
if layer, ok := layer.(*diffLayer); ok {
size += common.StorageSize(layer.memory)
}
}
return size, 0
}

13
core/state/snapshot/snapshot_test.go
Unescape View File

@ -43,11 +43,10 @@ func randomHash() common.Hash {
// randomAccount generates a random account and returns it RLP encoded. // randomAccount generates a random account and returns it RLP encoded.
func randomAccount() []byte { func randomAccount() []byte {
root := randomHash() a := &types.StateAccount{
a := Account{
Balance: big.NewInt(rand.Int63()), Balance: big.NewInt(rand.Int63()),
Nonce: rand.Uint64(), Nonce: rand.Uint64(),
Root: root[:], Root: randomHash(),
CodeHash: types.EmptyCodeHash[:], CodeHash: types.EmptyCodeHash[:],
} }
data, _ := rlp.EncodeToBytes(a) data, _ := rlp.EncodeToBytes(a)
@ -118,7 +117,7 @@ func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) {
if err := snaps.Cap(common.HexToHash("0x02"), 0); err != nil { if err := snaps.Cap(common.HexToHash("0x02"), 0); err != nil {
t.Fatalf("failed to merge diff layer onto disk: %v", err) t.Fatalf("failed to merge diff layer onto disk: %v", err)
} }
// Since the base layer was modified, ensure that data retrieval on the external reference fail // Since the base layer was modified, ensure that data retrievals on the external reference fail
if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale {
t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) t.Errorf("stale reference returned account: %#x (err: %v)", acc, err)
} }
@ -185,6 +184,10 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) {
// be returned with junk data. This version of the test retains the bottom diff // be returned with junk data. This version of the test retains the bottom diff
// layer to check the usual mode of operation where the accumulator is retained. // layer to check the usual mode of operation where the accumulator is retained.
func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) {
// Un-commenting this triggers the bloom set to be deterministic. The values below
// were used to trigger the flaw described in https://github.com/ethereum/go-ethereum/issues/27254.
// bloomDestructHasherOffset, bloomAccountHasherOffset, bloomStorageHasherOffset = 14, 24, 5
// Create an empty base layer and a snapshot tree out of it // Create an empty base layer and a snapshot tree out of it
base := &diskLayer{ base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(), diskdb: rawdb.NewMemoryDatabase(),
@ -461,7 +464,7 @@ func TestReadStateDuringFlattening(t *testing.T) {
snap := snaps.Snapshot(common.HexToHash("0xa3")) snap := snaps.Snapshot(common.HexToHash("0xa3"))
// Register the testing hook to access the state after flattening // Register the testing hook to access the state after flattening
var result = make(chan *Account) var result = make(chan *SlimAccount)
snaps.onFlatten = func() { snaps.onFlatten = func() {
// Spin up a thread to read the account from the pre-created // Spin up a thread to read the account from the pre-created
// snapshot handler. It's expected to be blocked. // snapshot handler. It's expected to be blocked.

9
core/state/snapshot/utils.go
Unescape View File

@ -23,7 +23,6 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/harmony-one/harmony/core/rawdb" "github.com/harmony-one/harmony/core/rawdb"
"github.com/harmony-one/harmony/internal/utils" "github.com/harmony-one/harmony/internal/utils"
) )
@ -111,8 +110,8 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error {
baseRoot := rawdb.ReadSnapshotRoot(db) baseRoot := rawdb.ReadSnapshotRoot(db)
fmt.Printf("Disklayer: Root: %x\n", baseRoot) fmt.Printf("Disklayer: Root: %x\n", baseRoot)
if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { if data := rawdb.ReadAccountSnapshot(db, hash); data != nil {
account := new(Account) account, err := FullAccount(data)
if err := rlp.DecodeBytes(data, account); err != nil { if err != nil {
panic(err) panic(err)
} }
fmt.Printf("\taccount.nonce: %d\n", account.Nonce) fmt.Printf("\taccount.nonce: %d\n", account.Nonce)
@ -142,8 +141,8 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error {
} }
fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot)
if data, ok := accounts[hash]; ok { if data, ok := accounts[hash]; ok {
account := new(Account) account, err := FullAccount(data)
if err := rlp.DecodeBytes(data, account); err != nil { if err != nil {
panic(err) panic(err)
} }
fmt.Printf("\taccount.nonce: %d\n", account.Nonce) fmt.Printf("\taccount.nonce: %d\n", account.Nonce)

Write Preview
Loading…
Cancel
Save