Upgrade rawdb and statedb codes to add the latest functionalities of ethdb (#4374)
* added bloom filter * upgrade rawdb and statedb * change var name and remove extra comments * return back fake storage in case if we need it for test later * add the previous change back * remove some extra entries from go.mod * fix WritePreimages to use batch * mark unused functions which are ported over from eth --------- Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com>pull/4395/head
Gheis Mohammadi
2 years ago
committed by
GitHub
parent
a1775465d7
commit
d21dc7f200
@ -0,0 +1,210 @@ |
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
) |
||||
|
||||
// ReadSnapshotDisabled retrieves if the snapshot maintenance is disabled.
|
||||
func ReadSnapshotDisabled(db ethdb.KeyValueReader) bool { |
||||
disabled, _ := db.Has(snapshotDisabledKey) |
||||
return disabled |
||||
} |
||||
|
||||
// WriteSnapshotDisabled stores the snapshot pause flag.
|
||||
func WriteSnapshotDisabled(db ethdb.KeyValueWriter) { |
||||
if err := db.Put(snapshotDisabledKey, []byte("42")); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store snapshot disabled flag") |
||||
} |
||||
} |
||||
|
||||
// DeleteSnapshotDisabled deletes the flag keeping the snapshot maintenance disabled.
|
||||
func DeleteSnapshotDisabled(db ethdb.KeyValueWriter) { |
||||
if err := db.Delete(snapshotDisabledKey); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to remove snapshot disabled flag") |
||||
} |
||||
} |
||||
|
||||
// ReadSnapshotRoot retrieves the root of the block whose state is contained in
|
||||
// the persisted snapshot.
|
||||
func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash { |
||||
data, _ := db.Get(SnapshotRootKey) |
||||
if len(data) != common.HashLength { |
||||
return common.Hash{} |
||||
} |
||||
return common.BytesToHash(data) |
||||
} |
||||
|
||||
// WriteSnapshotRoot stores the root of the block whose state is contained in
|
||||
// the persisted snapshot.
|
||||
func WriteSnapshotRoot(db ethdb.KeyValueWriter, root common.Hash) { |
||||
if err := db.Put(SnapshotRootKey, root[:]); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store snapshot root") |
||||
} |
||||
} |
||||
|
||||
// DeleteSnapshotRoot deletes the hash of the block whose state is contained in
|
||||
// the persisted snapshot. Since snapshots are not immutable, this method can
|
||||
// be used during updates, so a crash or failure will mark the entire snapshot
|
||||
// invalid.
|
||||
func DeleteSnapshotRoot(db ethdb.KeyValueWriter) { |
||||
if err := db.Delete(SnapshotRootKey); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to remove snapshot root") |
||||
} |
||||
} |
||||
|
||||
// ReadAccountSnapshot retrieves the snapshot entry of an account trie leaf.
|
||||
func ReadAccountSnapshot(db ethdb.KeyValueReader, hash common.Hash) []byte { |
||||
data, _ := db.Get(accountSnapshotKey(hash)) |
||||
return data |
||||
} |
||||
|
||||
// WriteAccountSnapshot stores the snapshot entry of an account trie leaf.
|
||||
func WriteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash, entry []byte) { |
||||
if err := db.Put(accountSnapshotKey(hash), entry); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store account snapshot") |
||||
} |
||||
} |
||||
|
||||
// DeleteAccountSnapshot removes the snapshot entry of an account trie leaf.
|
||||
func DeleteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash) { |
||||
if err := db.Delete(accountSnapshotKey(hash)); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to delete account snapshot") |
||||
} |
||||
} |
||||
|
||||
// ReadStorageSnapshot retrieves the snapshot entry of an storage trie leaf.
|
||||
func ReadStorageSnapshot(db ethdb.KeyValueReader, accountHash, storageHash common.Hash) []byte { |
||||
data, _ := db.Get(storageSnapshotKey(accountHash, storageHash)) |
||||
return data |
||||
} |
||||
|
||||
// WriteStorageSnapshot stores the snapshot entry of an storage trie leaf.
|
||||
func WriteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash, entry []byte) { |
||||
if err := db.Put(storageSnapshotKey(accountHash, storageHash), entry); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store storage snapshot") |
||||
} |
||||
} |
||||
|
||||
// DeleteStorageSnapshot removes the snapshot entry of an storage trie leaf.
|
||||
func DeleteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash) { |
||||
if err := db.Delete(storageSnapshotKey(accountHash, storageHash)); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to delete storage snapshot") |
||||
} |
||||
} |
||||
|
||||
// IterateStorageSnapshots returns an iterator for walking the entire storage
|
||||
// space of a specific account.
|
||||
func IterateStorageSnapshots(db ethdb.Iteratee, accountHash common.Hash) ethdb.Iterator { |
||||
return NewKeyLengthIterator(db.NewIterator(storageSnapshotsKey(accountHash), nil), len(SnapshotStoragePrefix)+2*common.HashLength) |
||||
} |
||||
|
||||
// ReadSnapshotJournal retrieves the serialized in-memory diff layers saved at
|
||||
// the last shutdown. The blob is expected to be max a few 10s of megabytes.
|
||||
func ReadSnapshotJournal(db ethdb.KeyValueReader) []byte { |
||||
data, _ := db.Get(snapshotJournalKey) |
||||
return data |
||||
} |
||||
|
||||
// WriteSnapshotJournal stores the serialized in-memory diff layers to save at
|
||||
// shutdown. The blob is expected to be max a few 10s of megabytes.
|
||||
func WriteSnapshotJournal(db ethdb.KeyValueWriter, journal []byte) { |
||||
if err := db.Put(snapshotJournalKey, journal); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store snapshot journal") |
||||
} |
||||
} |
||||
|
||||
// DeleteSnapshotJournal deletes the serialized in-memory diff layers saved at
|
||||
// the last shutdown
|
||||
func DeleteSnapshotJournal(db ethdb.KeyValueWriter) { |
||||
if err := db.Delete(snapshotJournalKey); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to remove snapshot journal") |
||||
} |
||||
} |
||||
|
||||
// ReadSnapshotGenerator retrieves the serialized snapshot generator saved at
|
||||
// the last shutdown.
|
||||
func ReadSnapshotGenerator(db ethdb.KeyValueReader) []byte { |
||||
data, _ := db.Get(snapshotGeneratorKey) |
||||
return data |
||||
} |
||||
|
||||
// WriteSnapshotGenerator stores the serialized snapshot generator to save at
|
||||
// shutdown.
|
||||
func WriteSnapshotGenerator(db ethdb.KeyValueWriter, generator []byte) { |
||||
if err := db.Put(snapshotGeneratorKey, generator); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store snapshot generator") |
||||
} |
||||
} |
||||
|
||||
// DeleteSnapshotGenerator deletes the serialized snapshot generator saved at
|
||||
// the last shutdown
|
||||
func DeleteSnapshotGenerator(db ethdb.KeyValueWriter) { |
||||
if err := db.Delete(snapshotGeneratorKey); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to remove snapshot generator") |
||||
} |
||||
} |
||||
|
||||
// ReadSnapshotRecoveryNumber retrieves the block number of the last persisted
|
||||
// snapshot layer.
|
||||
func ReadSnapshotRecoveryNumber(db ethdb.KeyValueReader) *uint64 { |
||||
data, _ := db.Get(snapshotRecoveryKey) |
||||
if len(data) == 0 { |
||||
return nil |
||||
} |
||||
if len(data) != 8 { |
||||
return nil |
||||
} |
||||
number := binary.BigEndian.Uint64(data) |
||||
return &number |
||||
} |
||||
|
||||
// WriteSnapshotRecoveryNumber stores the block number of the last persisted
|
||||
// snapshot layer.
|
||||
func WriteSnapshotRecoveryNumber(db ethdb.KeyValueWriter, number uint64) { |
||||
var buf [8]byte |
||||
binary.BigEndian.PutUint64(buf[:], number) |
||||
if err := db.Put(snapshotRecoveryKey, buf[:]); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store snapshot recovery number") |
||||
} |
||||
} |
||||
|
||||
// DeleteSnapshotRecoveryNumber deletes the block number of the last persisted
|
||||
// snapshot layer.
|
||||
func DeleteSnapshotRecoveryNumber(db ethdb.KeyValueWriter) { |
||||
if err := db.Delete(snapshotRecoveryKey); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to remove snapshot recovery number") |
||||
} |
||||
} |
||||
|
||||
// ReadSnapshotSyncStatus retrieves the serialized sync status saved at shutdown.
|
||||
func ReadSnapshotSyncStatus(db ethdb.KeyValueReader) []byte { |
||||
data, _ := db.Get(snapshotSyncStatusKey) |
||||
return data |
||||
} |
||||
|
||||
// WriteSnapshotSyncStatus stores the serialized sync status to save at shutdown.
|
||||
func WriteSnapshotSyncStatus(db ethdb.KeyValueWriter, status []byte) { |
||||
if err := db.Put(snapshotSyncStatusKey, status); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store snapshot sync status") |
||||
} |
||||
} |
||||
@ -0,0 +1,95 @@ |
||||
// Copyright 2020 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
) |
||||
|
||||
// ReadPreimage retrieves a single preimage of the provided hash.
|
||||
func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte { |
||||
data, _ := db.Get(preimageKey(hash)) |
||||
return data |
||||
} |
||||
|
||||
// WritePreimages writes the provided set of preimages to the database.
|
||||
func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) error { |
||||
for hash, preimage := range preimages { |
||||
if err := db.Put(preimageKey(hash), preimage); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store trie preimage") |
||||
} |
||||
} |
||||
preimageCounter.Inc(int64(len(preimages))) |
||||
preimageHitCounter.Inc(int64(len(preimages))) |
||||
return nil |
||||
} |
||||
|
||||
// ReadCode retrieves the contract code of the provided code hash.
|
||||
func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte { |
||||
// Try with the prefixed code scheme first, if not then try with legacy
|
||||
// scheme.
|
||||
data := ReadCodeWithPrefix(db, hash) |
||||
if len(data) != 0 { |
||||
return data |
||||
} |
||||
data, _ = db.Get(hash.Bytes()) |
||||
return data |
||||
} |
||||
|
||||
// ReadCodeWithPrefix retrieves the contract code of the provided code hash.
|
||||
// The main difference between this function and ReadCode is this function
|
||||
// will only check the existence with latest scheme(with prefix).
|
||||
func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte { |
||||
data, _ := db.Get(codeKey(hash)) |
||||
return data |
||||
} |
||||
|
||||
// HasCode checks if the contract code corresponding to the
|
||||
// provided code hash is present in the db.
|
||||
func HasCode(db ethdb.KeyValueReader, hash common.Hash) bool { |
||||
// Try with the prefixed code scheme first, if not then try with legacy
|
||||
// scheme.
|
||||
if ok := HasCodeWithPrefix(db, hash); ok { |
||||
return true |
||||
} |
||||
ok, _ := db.Has(hash.Bytes()) |
||||
return ok |
||||
} |
||||
|
||||
// HasCodeWithPrefix checks if the contract code corresponding to the
|
||||
// provided code hash is present in the db. This function will only check
|
||||
// presence using the prefix-scheme.
|
||||
func HasCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool { |
||||
ok, _ := db.Has(codeKey(hash)) |
||||
return ok |
||||
} |
||||
|
||||
// WriteCode writes the provided contract code database.
|
||||
func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) { |
||||
if err := db.Put(codeKey(hash), code); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store contract code") |
||||
} |
||||
} |
||||
|
||||
// DeleteCode deletes the specified contract code from the database.
|
||||
func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) { |
||||
if err := db.Delete(codeKey(hash)); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to delete contract code") |
||||
} |
||||
} |
||||
@ -0,0 +1,80 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"bytes" |
||||
|
||||
"github.com/ethereum/go-ethereum/core/types" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/ethereum/go-ethereum/rlp" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
) |
||||
|
||||
// ReadSkeletonSyncStatus retrieves the serialized sync status saved at shutdown.
|
||||
func ReadSkeletonSyncStatus(db ethdb.KeyValueReader) []byte { |
||||
data, _ := db.Get(skeletonSyncStatusKey) |
||||
return data |
||||
} |
||||
|
||||
// WriteSkeletonSyncStatus stores the serialized sync status to save at shutdown.
|
||||
func WriteSkeletonSyncStatus(db ethdb.KeyValueWriter, status []byte) { |
||||
if err := db.Put(skeletonSyncStatusKey, status); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store skeleton sync status") |
||||
} |
||||
} |
||||
|
||||
// DeleteSkeletonSyncStatus deletes the serialized sync status saved at the last
|
||||
// shutdown
|
||||
func DeleteSkeletonSyncStatus(db ethdb.KeyValueWriter) { |
||||
if err := db.Delete(skeletonSyncStatusKey); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to remove skeleton sync status") |
||||
} |
||||
} |
||||
|
||||
// ReadSkeletonHeader retrieves a block header from the skeleton sync store,
|
||||
func ReadSkeletonHeader(db ethdb.KeyValueReader, number uint64) *types.Header { |
||||
data, _ := db.Get(skeletonHeaderKey(number)) |
||||
if len(data) == 0 { |
||||
return nil |
||||
} |
||||
header := new(types.Header) |
||||
if err := rlp.Decode(bytes.NewReader(data), header); err != nil { |
||||
utils.Logger().Error().Err(err).Uint64("number", number).Msg("Invalid skeleton header RLP") |
||||
return nil |
||||
} |
||||
return header |
||||
} |
||||
|
||||
// WriteSkeletonHeader stores a block header into the skeleton sync store.
|
||||
func WriteSkeletonHeader(db ethdb.KeyValueWriter, header *types.Header) { |
||||
data, err := rlp.EncodeToBytes(header) |
||||
if err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to RLP encode header") |
||||
} |
||||
key := skeletonHeaderKey(header.Number.Uint64()) |
||||
if err := db.Put(key, data); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store skeleton header") |
||||
} |
||||
} |
||||
|
||||
// DeleteSkeletonHeader removes all block header data associated with a hash.
|
||||
func DeleteSkeletonHeader(db ethdb.KeyValueWriter, number uint64) { |
||||
if err := db.Delete(skeletonHeaderKey(number)); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to delete skeleton header") |
||||
} |
||||
} |
||||
@ -0,0 +1,263 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sync" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/crypto" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
"golang.org/x/crypto/sha3" |
||||
) |
||||
|
||||
// HashScheme is the legacy hash-based state scheme with which trie nodes are
|
||||
// stored in the disk with node hash as the database key. The advantage of this
|
||||
// scheme is that different versions of trie nodes can be stored in disk, which
|
||||
// is very beneficial for constructing archive nodes. The drawback is it will
|
||||
// store different trie nodes on the same path to different locations on the disk
|
||||
// with no data locality, and it's unfriendly for designing state pruning.
|
||||
//
|
||||
// Now this scheme is still kept for backward compatibility, and it will be used
|
||||
// for archive node and some other tries(e.g. light trie).
|
||||
const HashScheme = "hashScheme" |
||||
|
||||
// PathScheme is the new path-based state scheme with which trie nodes are stored
|
||||
// in the disk with node path as the database key. This scheme will only store one
|
||||
// version of state data in the disk, which means that the state pruning operation
|
||||
// is native. At the same time, this scheme will put adjacent trie nodes in the same
|
||||
// area of the disk with good data locality property. But this scheme needs to rely
|
||||
// on extra state diffs to survive deep reorg.
|
||||
const PathScheme = "pathScheme" |
||||
|
||||
// nodeHasher used to derive the hash of trie node.
|
||||
type nodeHasher struct{ sha crypto.KeccakState } |
||||
|
||||
var hasherPool = sync.Pool{ |
||||
New: func() interface{} { return &nodeHasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} }, |
||||
} |
||||
|
||||
func newNodeHasher() *nodeHasher { return hasherPool.Get().(*nodeHasher) } |
||||
func returnHasherToPool(h *nodeHasher) { hasherPool.Put(h) } |
||||
|
||||
func (h *nodeHasher) hashData(data []byte) (n common.Hash) { |
||||
h.sha.Reset() |
||||
h.sha.Write(data) |
||||
h.sha.Read(n[:]) |
||||
return n |
||||
} |
||||
|
||||
// ReadAccountTrieNode retrieves the account trie node and the associated node
|
||||
// hash with the specified node path.
|
||||
func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) { |
||||
data, err := db.Get(accountTrieNodeKey(path)) |
||||
if err != nil { |
||||
return nil, common.Hash{} |
||||
} |
||||
hasher := newNodeHasher() |
||||
defer returnHasherToPool(hasher) |
||||
return data, hasher.hashData(data) |
||||
} |
||||
|
||||
// HasAccountTrieNode checks the account trie node presence with the specified
|
||||
// node path and the associated node hash.
|
||||
func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash) bool { |
||||
data, err := db.Get(accountTrieNodeKey(path)) |
||||
if err != nil { |
||||
return false |
||||
} |
||||
hasher := newNodeHasher() |
||||
defer returnHasherToPool(hasher) |
||||
return hasher.hashData(data) == hash |
||||
} |
||||
|
||||
// WriteAccountTrieNode writes the provided account trie node into database.
|
||||
func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) { |
||||
if err := db.Put(accountTrieNodeKey(path), node); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store account trie node") |
||||
} |
||||
} |
||||
|
||||
// DeleteAccountTrieNode deletes the specified account trie node from the database.
|
||||
func DeleteAccountTrieNode(db ethdb.KeyValueWriter, path []byte) { |
||||
if err := db.Delete(accountTrieNodeKey(path)); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to delete account trie node") |
||||
} |
||||
} |
||||
|
||||
// ReadStorageTrieNode retrieves the storage trie node and the associated node
|
||||
// hash with the specified node path.
|
||||
func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) ([]byte, common.Hash) { |
||||
data, err := db.Get(storageTrieNodeKey(accountHash, path)) |
||||
if err != nil { |
||||
return nil, common.Hash{} |
||||
} |
||||
hasher := newNodeHasher() |
||||
defer returnHasherToPool(hasher) |
||||
return data, hasher.hashData(data) |
||||
} |
||||
|
||||
// HasStorageTrieNode checks the storage trie node presence with the provided
|
||||
// node path and the associated node hash.
|
||||
func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte, hash common.Hash) bool { |
||||
data, err := db.Get(storageTrieNodeKey(accountHash, path)) |
||||
if err != nil { |
||||
return false |
||||
} |
||||
hasher := newNodeHasher() |
||||
defer returnHasherToPool(hasher) |
||||
return hasher.hashData(data) == hash |
||||
} |
||||
|
||||
// WriteStorageTrieNode writes the provided storage trie node into database.
|
||||
func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) { |
||||
if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store storage trie node") |
||||
} |
||||
} |
||||
|
||||
// DeleteStorageTrieNode deletes the specified storage trie node from the database.
|
||||
func DeleteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte) { |
||||
if err := db.Delete(storageTrieNodeKey(accountHash, path)); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to delete storage trie node") |
||||
} |
||||
} |
||||
|
||||
// ReadLegacyTrieNode retrieves the legacy trie node with the given
|
||||
// associated node hash.
|
||||
func ReadLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte { |
||||
data, err := db.Get(hash.Bytes()) |
||||
if err != nil { |
||||
return nil |
||||
} |
||||
return data |
||||
} |
||||
|
||||
// HasLegacyTrieNode checks if the trie node with the provided hash is present in db.
|
||||
func HasLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool { |
||||
ok, _ := db.Has(hash.Bytes()) |
||||
return ok |
||||
} |
||||
|
||||
// WriteLegacyTrieNode writes the provided legacy trie node to database.
|
||||
func WriteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) { |
||||
if err := db.Put(hash.Bytes(), node); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to store legacy trie node") |
||||
} |
||||
} |
||||
|
||||
// DeleteLegacyTrieNode deletes the specified legacy trie node from database.
|
||||
func DeleteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash) { |
||||
if err := db.Delete(hash.Bytes()); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to delete legacy trie node") |
||||
} |
||||
} |
||||
|
||||
// HasTrieNode checks the trie node presence with the provided node info and
|
||||
// the associated node hash.
|
||||
func HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) bool { |
||||
switch scheme { |
||||
case HashScheme: |
||||
return HasLegacyTrieNode(db, hash) |
||||
case PathScheme: |
||||
if owner == (common.Hash{}) { |
||||
return HasAccountTrieNode(db, path, hash) |
||||
} |
||||
return HasStorageTrieNode(db, owner, path, hash) |
||||
default: |
||||
panic(fmt.Sprintf("Unknown scheme %v", scheme)) |
||||
} |
||||
} |
||||
|
||||
// ReadTrieNode retrieves the trie node from database with the provided node info
|
||||
// and associated node hash.
|
||||
// hashScheme-based lookup requires the following:
|
||||
// - hash
|
||||
//
|
||||
// pathScheme-based lookup requires the following:
|
||||
// - owner
|
||||
// - path
|
||||
func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) []byte { |
||||
switch scheme { |
||||
case HashScheme: |
||||
return ReadLegacyTrieNode(db, hash) |
||||
case PathScheme: |
||||
var ( |
||||
blob []byte |
||||
nHash common.Hash |
||||
) |
||||
if owner == (common.Hash{}) { |
||||
blob, nHash = ReadAccountTrieNode(db, path) |
||||
} else { |
||||
blob, nHash = ReadStorageTrieNode(db, owner, path) |
||||
} |
||||
if nHash != hash { |
||||
return nil |
||||
} |
||||
return blob |
||||
default: |
||||
panic(fmt.Sprintf("Unknown scheme %v", scheme)) |
||||
} |
||||
} |
||||
|
||||
// WriteTrieNode writes the trie node into database with the provided node info
|
||||
// and associated node hash.
|
||||
// hashScheme-based lookup requires the following:
|
||||
// - hash
|
||||
//
|
||||
// pathScheme-based lookup requires the following:
|
||||
// - owner
|
||||
// - path
|
||||
func WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte, scheme string) { |
||||
switch scheme { |
||||
case HashScheme: |
||||
WriteLegacyTrieNode(db, hash, node) |
||||
case PathScheme: |
||||
if owner == (common.Hash{}) { |
||||
WriteAccountTrieNode(db, path, node) |
||||
} else { |
||||
WriteStorageTrieNode(db, owner, path, node) |
||||
} |
||||
default: |
||||
panic(fmt.Sprintf("Unknown scheme %v", scheme)) |
||||
} |
||||
} |
||||
|
||||
// DeleteTrieNode deletes the trie node from database with the provided node info
|
||||
// and associated node hash.
|
||||
// hashScheme-based lookup requires the following:
|
||||
// - hash
|
||||
//
|
||||
// pathScheme-based lookup requires the following:
|
||||
// - owner
|
||||
// - path
|
||||
func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, scheme string) { |
||||
switch scheme { |
||||
case HashScheme: |
||||
DeleteLegacyTrieNode(db, hash) |
||||
case PathScheme: |
||||
if owner == (common.Hash{}) { |
||||
DeleteAccountTrieNode(db, path) |
||||
} else { |
||||
DeleteStorageTrieNode(db, owner, path) |
||||
} |
||||
default: |
||||
panic(fmt.Sprintf("Unknown scheme %v", scheme)) |
||||
} |
||||
} |
||||
@ -0,0 +1,55 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
// The list of table names of chain freezer.
|
||||
// This variables is NOT used, just ported over from the Ethereum
|
||||
const ( |
||||
// ChainFreezerHeaderTable indicates the name of the freezer header table.
|
||||
ChainFreezerHeaderTable = "headers" |
||||
|
||||
// ChainFreezerHashTable indicates the name of the freezer canonical hash table.
|
||||
ChainFreezerHashTable = "hashes" |
||||
|
||||
// ChainFreezerBodiesTable indicates the name of the freezer block body table.
|
||||
ChainFreezerBodiesTable = "bodies" |
||||
|
||||
// ChainFreezerReceiptTable indicates the name of the freezer receipts table.
|
||||
ChainFreezerReceiptTable = "receipts" |
||||
|
||||
// ChainFreezerDifficultyTable indicates the name of the freezer total difficulty table.
|
||||
ChainFreezerDifficultyTable = "diffs" |
||||
) |
||||
|
||||
// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
|
||||
// Hashes and difficulties don't compress well.
|
||||
// This function is NOT used, just ported over from the Ethereum
|
||||
var chainFreezerNoSnappy = map[string]bool{ |
||||
ChainFreezerHeaderTable: false, |
||||
ChainFreezerHashTable: true, |
||||
ChainFreezerBodiesTable: false, |
||||
ChainFreezerReceiptTable: false, |
||||
ChainFreezerDifficultyTable: true, |
||||
} |
||||
|
||||
// The list of identifiers of ancient stores.
|
||||
var ( |
||||
chainFreezerName = "chain" // the folder name of chain segment ancient store.
|
||||
) |
||||
|
||||
// freezers the collections of all builtin freezers.
|
||||
var freezers = []string{chainFreezerName} |
||||
@ -0,0 +1,91 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
type tableSize struct { |
||||
name string |
||||
size common.StorageSize |
||||
} |
||||
|
||||
// freezerInfo contains the basic information of the freezer.
|
||||
type freezerInfo struct { |
||||
name string // The identifier of freezer
|
||||
head uint64 // The number of last stored item in the freezer
|
||||
tail uint64 // The number of first stored item in the freezer
|
||||
sizes []tableSize // The storage size per table
|
||||
} |
||||
|
||||
// count returns the number of stored items in the freezer.
|
||||
func (info *freezerInfo) count() uint64 { |
||||
return info.head - info.tail + 1 |
||||
} |
||||
|
||||
// size returns the storage size of the entire freezer.
|
||||
func (info *freezerInfo) size() common.StorageSize { |
||||
var total common.StorageSize |
||||
for _, table := range info.sizes { |
||||
total += table.size |
||||
} |
||||
return total |
||||
} |
||||
|
||||
// inspectFreezers inspects all freezers registered in the system.
|
||||
// This function is NOT used, just ported over from the Ethereum
|
||||
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { |
||||
var infos []freezerInfo |
||||
for _, freezer := range freezers { |
||||
switch freezer { |
||||
case chainFreezerName: |
||||
// Chain ancient store is a bit special. It's always opened along
|
||||
// with the key-value store, inspect the chain store directly.
|
||||
info := freezerInfo{name: freezer} |
||||
// Retrieve storage size of every contained table.
|
||||
for table := range chainFreezerNoSnappy { |
||||
size, err := db.AncientSize(table) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)}) |
||||
} |
||||
// Retrieve the number of last stored item
|
||||
ancients, err := db.Ancients() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
info.head = ancients - 1 |
||||
|
||||
// Retrieve the number of first stored item
|
||||
tail, err := db.Tail() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
info.tail = tail |
||||
infos = append(infos, info) |
||||
|
||||
default: |
||||
return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers) |
||||
} |
||||
} |
||||
return infos, nil |
||||
} |
||||
@ -0,0 +1,358 @@ |
||||
// Copyright 2020 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"runtime" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/common/prque" |
||||
"github.com/ethereum/go-ethereum/core/types" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/ethereum/go-ethereum/log" |
||||
"github.com/ethereum/go-ethereum/rlp" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
) |
||||
|
||||
// InitDatabaseFromFreezer reinitializes an empty database from a previous batch
|
||||
// of frozen ancient blocks. The method iterates over all the frozen blocks and
|
||||
// injects into the database the block hash->number mappings.
|
||||
// This function is NOT used, just ported over from the Ethereum
|
||||
func InitDatabaseFromFreezer(db ethdb.Database) { |
||||
// If we can't access the freezer or it's empty, abort
|
||||
frozen, err := db.Ancients() |
||||
if err != nil || frozen == 0 { |
||||
return |
||||
} |
||||
var ( |
||||
batch = db.NewBatch() |
||||
start = time.Now() |
||||
logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log
|
||||
hash common.Hash |
||||
) |
||||
for i := uint64(0); i < frozen; { |
||||
// We read 100K hashes at a time, for a total of 3.2M
|
||||
count := uint64(100_000) |
||||
if i+count > frozen { |
||||
count = frozen - i |
||||
} |
||||
data, err := db.AncientRange(ChainFreezerHashTable, i, count, 32*count) |
||||
if err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to init database from freezer") |
||||
} |
||||
for j, h := range data { |
||||
number := i + uint64(j) |
||||
hash = common.BytesToHash(h) |
||||
WriteHeaderNumber(batch, hash, number) |
||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||
if batch.ValueSize() > ethdb.IdealBatchSize { |
||||
if err := batch.Write(); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to write data to db") |
||||
} |
||||
batch.Reset() |
||||
} |
||||
} |
||||
i += uint64(len(data)) |
||||
// If we've spent too much time already, notify the user of what we're doing
|
||||
if time.Since(logged) > 8*time.Second { |
||||
log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
logged = time.Now() |
||||
} |
||||
} |
||||
if err := batch.Write(); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed to write data to db") |
||||
} |
||||
batch.Reset() |
||||
|
||||
WriteHeadHeaderHash(db, hash) |
||||
WriteHeadFastBlockHash(db, hash) |
||||
log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
} |
||||
|
||||
type blockTxHashes struct { |
||||
number uint64 |
||||
hashes []common.Hash |
||||
} |
||||
|
||||
// iterateTransactions iterates over all transactions in the (canon) block
|
||||
// number(s) given, and yields the hashes on a channel. If there is a signal
|
||||
// received from interrupt channel, the iteration will be aborted and result
|
||||
// channel will be closed.
|
||||
func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes { |
||||
// One thread sequentially reads data from db
|
||||
type numberRlp struct { |
||||
number uint64 |
||||
rlp rlp.RawValue |
||||
} |
||||
if to == from { |
||||
return nil |
||||
} |
||||
threads := to - from |
||||
if cpus := runtime.NumCPU(); threads > uint64(cpus) { |
||||
threads = uint64(cpus) |
||||
} |
||||
var ( |
||||
rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel
|
||||
hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh
|
||||
) |
||||
// lookup runs in one instance
|
||||
lookup := func() { |
||||
n, end := from, to |
||||
if reverse { |
||||
n, end = to-1, from-1 |
||||
} |
||||
defer close(rlpCh) |
||||
for n != end { |
||||
data := ReadCanonicalBodyRLP(db, n) |
||||
// Feed the block to the aggregator, or abort on interrupt
|
||||
select { |
||||
case rlpCh <- &numberRlp{n, data}: |
||||
case <-interrupt: |
||||
return |
||||
} |
||||
if reverse { |
||||
n-- |
||||
} else { |
||||
n++ |
||||
} |
||||
} |
||||
} |
||||
// process runs in parallel
|
||||
nThreadsAlive := int32(threads) |
||||
process := func() { |
||||
defer func() { |
||||
// Last processor closes the result channel
|
||||
if atomic.AddInt32(&nThreadsAlive, -1) == 0 { |
||||
close(hashesCh) |
||||
} |
||||
}() |
||||
for data := range rlpCh { |
||||
var body types.Body |
||||
if err := rlp.DecodeBytes(data.rlp, &body); err != nil { |
||||
utils.Logger().Warn().Err(err).Uint64("block", data.number).Msg("Failed to decode block body") |
||||
return |
||||
} |
||||
var hashes []common.Hash |
||||
for _, tx := range body.Transactions { |
||||
hashes = append(hashes, tx.Hash()) |
||||
} |
||||
result := &blockTxHashes{ |
||||
hashes: hashes, |
||||
number: data.number, |
||||
} |
||||
// Feed the block to the aggregator, or abort on interrupt
|
||||
select { |
||||
case hashesCh <- result: |
||||
case <-interrupt: |
||||
return |
||||
} |
||||
} |
||||
} |
||||
go lookup() // start the sequential db accessor
|
||||
for i := 0; i < int(threads); i++ { |
||||
go process() |
||||
} |
||||
return hashesCh |
||||
} |
||||
|
||||
// indexTransactions creates txlookup indices of the specified block range.
|
||||
//
|
||||
// This function iterates canonical chain in reverse order, it has one main advantage:
|
||||
// We can write tx index tail flag periodically even without the whole indexing
|
||||
// procedure is finished. So that we can resume indexing procedure next time quickly.
|
||||
//
|
||||
// There is a passed channel, the whole procedure will be interrupted if any
|
||||
// signal received.
|
||||
// This function is NOT used, just ported over from the Ethereum
|
||||
func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { |
||||
// short circuit for invalid range
|
||||
if from >= to { |
||||
return |
||||
} |
||||
var ( |
||||
hashesCh = iterateTransactions(db, from, to, true, interrupt) |
||||
batch = db.NewBatch() |
||||
start = time.Now() |
||||
logged = start.Add(-7 * time.Second) |
||||
// Since we iterate in reverse, we expect the first number to come
|
||||
// in to be [to-1]. Therefore, setting lastNum to means that the
|
||||
// prqueue gap-evaluation will work correctly
|
||||
lastNum = to |
||||
queue = prque.New[int64, *blockTxHashes](nil) |
||||
// for stats reporting
|
||||
blocks, txs = 0, 0 |
||||
) |
||||
for chanDelivery := range hashesCh { |
||||
// Push the delivery into the queue and process contiguous ranges.
|
||||
// Since we iterate in reverse, so lower numbers have lower prio, and
|
||||
// we can use the number directly as prio marker
|
||||
queue.Push(chanDelivery, int64(chanDelivery.number)) |
||||
for !queue.Empty() { |
||||
// If the next available item is gapped, return
|
||||
if _, priority := queue.Peek(); priority != int64(lastNum-1) { |
||||
break |
||||
} |
||||
// For testing
|
||||
if hook != nil && !hook(lastNum-1) { |
||||
break |
||||
} |
||||
// Next block available, pop it off and index it
|
||||
delivery := queue.PopItem() |
||||
lastNum = delivery.number |
||||
WriteTxLookupEntries(batch, delivery.number, delivery.hashes) |
||||
blocks++ |
||||
txs += len(delivery.hashes) |
||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||
if batch.ValueSize() > ethdb.IdealBatchSize { |
||||
WriteTxIndexTail(batch, lastNum) // Also write the tail here
|
||||
if err := batch.Write(); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed writing batch to db") |
||||
return |
||||
} |
||||
batch.Reset() |
||||
} |
||||
// If we've spent too much time already, notify the user of what we're doing
|
||||
if time.Since(logged) > 8*time.Second { |
||||
log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
logged = time.Now() |
||||
} |
||||
} |
||||
} |
||||
// Flush the new indexing tail and the last committed data. It can also happen
|
||||
// that the last batch is empty because nothing to index, but the tail has to
|
||||
// be flushed anyway.
|
||||
WriteTxIndexTail(batch, lastNum) |
||||
if err := batch.Write(); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed writing batch to db") |
||||
return |
||||
} |
||||
select { |
||||
case <-interrupt: |
||||
log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
default: |
||||
log.Debug("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
} |
||||
} |
||||
|
||||
// IndexTransactions creates txlookup indices of the specified block range. The from
|
||||
// is included while to is excluded.
|
||||
//
|
||||
// This function iterates canonical chain in reverse order, it has one main advantage:
|
||||
// We can write tx index tail flag periodically even without the whole indexing
|
||||
// procedure is finished. So that we can resume indexing procedure next time quickly.
|
||||
//
|
||||
// There is a passed channel, the whole procedure will be interrupted if any
|
||||
// signal received.
|
||||
func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { |
||||
indexTransactions(db, from, to, interrupt, nil) |
||||
} |
||||
|
||||
// indexTransactionsForTesting is the internal debug version with an additional hook.
|
||||
func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { |
||||
indexTransactions(db, from, to, interrupt, hook) |
||||
} |
||||
|
||||
// unindexTransactions removes txlookup indices of the specified block range.
|
||||
//
|
||||
// There is a passed channel, the whole procedure will be interrupted if any
|
||||
// signal received.
|
||||
// This function is NOT used, just ported over from the Ethereum
|
||||
func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { |
||||
// short circuit for invalid range
|
||||
if from >= to { |
||||
return |
||||
} |
||||
var ( |
||||
hashesCh = iterateTransactions(db, from, to, false, interrupt) |
||||
batch = db.NewBatch() |
||||
start = time.Now() |
||||
logged = start.Add(-7 * time.Second) |
||||
// we expect the first number to come in to be [from]. Therefore, setting
|
||||
// nextNum to from means that the prqueue gap-evaluation will work correctly
|
||||
nextNum = from |
||||
queue = prque.New[int64, *blockTxHashes](nil) |
||||
// for stats reporting
|
||||
blocks, txs = 0, 0 |
||||
) |
||||
// Otherwise spin up the concurrent iterator and unindexer
|
||||
for delivery := range hashesCh { |
||||
// Push the delivery into the queue and process contiguous ranges.
|
||||
queue.Push(delivery, -int64(delivery.number)) |
||||
for !queue.Empty() { |
||||
// If the next available item is gapped, return
|
||||
if _, priority := queue.Peek(); -priority != int64(nextNum) { |
||||
break |
||||
} |
||||
// For testing
|
||||
if hook != nil && !hook(nextNum) { |
||||
break |
||||
} |
||||
delivery := queue.PopItem() |
||||
nextNum = delivery.number + 1 |
||||
DeleteTxLookupEntries(batch, delivery.hashes) |
||||
txs += len(delivery.hashes) |
||||
blocks++ |
||||
|
||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||
// A batch counts the size of deletion as '1', so we need to flush more
|
||||
// often than that.
|
||||
if blocks%1000 == 0 { |
||||
WriteTxIndexTail(batch, nextNum) |
||||
if err := batch.Write(); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed writing batch to db") |
||||
return |
||||
} |
||||
batch.Reset() |
||||
} |
||||
// If we've spent too much time already, notify the user of what we're doing
|
||||
if time.Since(logged) > 8*time.Second { |
||||
log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
logged = time.Now() |
||||
} |
||||
} |
||||
} |
||||
// Flush the new indexing tail and the last committed data. It can also happen
|
||||
// that the last batch is empty because nothing to unindex, but the tail has to
|
||||
// be flushed anyway.
|
||||
WriteTxIndexTail(batch, nextNum) |
||||
if err := batch.Write(); err != nil { |
||||
utils.Logger().Error().Err(err).Msg("Failed writing batch to db") |
||||
return |
||||
} |
||||
select { |
||||
case <-interrupt: |
||||
log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
default: |
||||
log.Debug("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
} |
||||
} |
||||
|
||||
// UnindexTransactions removes txlookup indices of the specified block range.
|
||||
// The from is included while to is excluded.
|
||||
//
|
||||
// There is a passed channel, the whole procedure will be interrupted if any
|
||||
// signal received.
|
||||
func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { |
||||
unindexTransactions(db, from, to, interrupt, nil) |
||||
} |
||||
|
||||
// unindexTransactionsForTesting is the internal debug version with an additional hook.
|
||||
func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { |
||||
unindexTransactions(db, from, to, interrupt, hook) |
||||
} |
||||
@ -0,0 +1,464 @@ |
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"bytes" |
||||
"errors" |
||||
"fmt" |
||||
"os" |
||||
"path" |
||||
"path/filepath" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/ethereum/go-ethereum/ethdb/leveldb" |
||||
"github.com/ethereum/go-ethereum/ethdb/memorydb" |
||||
"github.com/ethereum/go-ethereum/log" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
"github.com/olekukonko/tablewriter" |
||||
) |
||||
|
||||
var errNotSupported = errors.New("not supported") |
||||
|
||||
// convertLegacyFn takes a raw freezer entry in an older format and
|
||||
// returns it in the new format.
|
||||
type convertLegacyFn = func([]byte) ([]byte, error) |
||||
|
||||
// freezerdb is a database wrapper that enabled freezer data retrievals.
|
||||
type freezerdb struct { |
||||
ancientRoot string |
||||
ethdb.KeyValueStore |
||||
ethdb.AncientStore |
||||
} |
||||
|
||||
// AncientDatadir returns the path of root ancient directory.
|
||||
func (frdb *freezerdb) AncientDatadir() (string, error) { |
||||
return frdb.ancientRoot, nil |
||||
} |
||||
|
||||
// Close implements io.Closer, closing both the fast key-value store as well as
|
||||
// the slow ancient tables.
|
||||
func (frdb *freezerdb) Close() error { |
||||
var errs []error |
||||
if err := frdb.AncientStore.Close(); err != nil { |
||||
errs = append(errs, err) |
||||
} |
||||
if err := frdb.KeyValueStore.Close(); err != nil { |
||||
errs = append(errs, err) |
||||
} |
||||
if len(errs) != 0 { |
||||
return fmt.Errorf("%v", errs) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// nofreezedb is a database wrapper that disables freezer data retrievals.
|
||||
type nofreezedb struct { |
||||
ethdb.KeyValueStore |
||||
} |
||||
|
||||
// HasAncient returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) { |
||||
return false, errNotSupported |
||||
} |
||||
|
||||
// Ancient returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { |
||||
return nil, errNotSupported |
||||
} |
||||
|
||||
// AncientRange returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) { |
||||
return nil, errNotSupported |
||||
} |
||||
|
||||
// Ancients returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) Ancients() (uint64, error) { |
||||
return 0, errNotSupported |
||||
} |
||||
|
||||
// Tail returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) Tail() (uint64, error) { |
||||
return 0, errNotSupported |
||||
} |
||||
|
||||
// AncientSize returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) AncientSize(kind string) (uint64, error) { |
||||
return 0, errNotSupported |
||||
} |
||||
|
||||
// ModifyAncients is not supported.
|
||||
func (db *nofreezedb) ModifyAncients(func(ethdb.AncientWriteOp) error) (int64, error) { |
||||
return 0, errNotSupported |
||||
} |
||||
|
||||
// TruncateHead returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) TruncateHead(items uint64) error { |
||||
return errNotSupported |
||||
} |
||||
|
||||
// TruncateTail returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) TruncateTail(items uint64) error { |
||||
return errNotSupported |
||||
} |
||||
|
||||
// Sync returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) Sync() error { |
||||
return errNotSupported |
||||
} |
||||
|
||||
func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { |
||||
// Unlike other ancient-related methods, this method does not return
|
||||
// errNotSupported when invoked.
|
||||
// The reason for this is that the caller might want to do several things:
|
||||
// 1. Check if something is in freezer,
|
||||
// 2. If not, check leveldb.
|
||||
//
|
||||
// This will work, since the ancient-checks inside 'fn' will return errors,
|
||||
// and the leveldb work will continue.
|
||||
//
|
||||
// If we instead were to return errNotSupported here, then the caller would
|
||||
// have to explicitly check for that, having an extra clause to do the
|
||||
// non-ancient operations.
|
||||
return fn(db) |
||||
} |
||||
|
||||
// MigrateTable processes the entries in a given table in sequence
|
||||
// converting them to a new format if they're of an old format.
|
||||
func (db *nofreezedb) MigrateTable(kind string, convert convertLegacyFn) error { |
||||
return errNotSupported |
||||
} |
||||
|
||||
// AncientDatadir returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) AncientDatadir() (string, error) { |
||||
return "", errNotSupported |
||||
} |
||||
|
||||
// NewDatabase creates a high level database on top of a given key-value data
|
||||
// store without a freezer moving immutable chain segments into cold storage.
|
||||
func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { |
||||
return &nofreezedb{KeyValueStore: db} |
||||
} |
||||
|
||||
// resolveChainFreezerDir is a helper function which resolves the absolute path
|
||||
// of chain freezer by considering backward compatibility.
|
||||
// This function is NOT used, just ported over from the Ethereum
|
||||
func resolveChainFreezerDir(ancient string) string { |
||||
// Check if the chain freezer is already present in the specified
|
||||
// sub folder, if not then two possibilities:
|
||||
// - chain freezer is not initialized
|
||||
// - chain freezer exists in legacy location (root ancient folder)
|
||||
freezer := path.Join(ancient, chainFreezerName) |
||||
if !common.FileExist(freezer) { |
||||
if !common.FileExist(ancient) { |
||||
// The entire ancient store is not initialized, still use the sub
|
||||
// folder for initialization.
|
||||
} else { |
||||
// Ancient root is already initialized, then we hold the assumption
|
||||
// that chain freezer is also initialized and located in root folder.
|
||||
// In this case fallback to legacy location.
|
||||
freezer = ancient |
||||
log.Info("Found legacy ancient chain path", "location", ancient) |
||||
} |
||||
} |
||||
return freezer |
||||
} |
||||
|
||||
// NewMemoryDatabase creates an ephemeral in-memory key-value database without a
|
||||
// freezer moving immutable chain segments into cold storage.
|
||||
func NewMemoryDatabase() ethdb.Database { |
||||
return NewDatabase(memorydb.New()) |
||||
} |
||||
|
||||
// NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database
|
||||
// with an initial starting capacity, but without a freezer moving immutable
|
||||
// chain segments into cold storage.
|
||||
func NewMemoryDatabaseWithCap(size int) ethdb.Database { |
||||
return NewDatabase(memorydb.NewWithCap(size)) |
||||
} |
||||
|
||||
// NewLevelDBDatabase creates a persistent key-value database without a freezer
|
||||
// moving immutable chain segments into cold storage.
|
||||
func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { |
||||
db, err := leveldb.New(file, cache, handles, namespace, readonly) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
log.Info("Using LevelDB as the backing database") |
||||
return NewDatabase(db), nil |
||||
} |
||||
|
||||
const ( |
||||
dbPebble = "pebble" |
||||
dbLeveldb = "leveldb" |
||||
) |
||||
|
||||
// hasPreexistingDb checks the given data directory whether a database is already
|
||||
// instantiated at that location, and if so, returns the type of database (or the
|
||||
// empty string).
|
||||
func hasPreexistingDb(path string) string { |
||||
if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { |
||||
return "" // No pre-existing db
|
||||
} |
||||
if matches, err := filepath.Glob(filepath.Join(path, "OPTIONS*")); len(matches) > 0 || err != nil { |
||||
if err != nil { |
||||
panic(err) // only possible if the pattern is malformed
|
||||
} |
||||
return dbPebble |
||||
} |
||||
return dbLeveldb |
||||
} |
||||
|
||||
// OpenOptions contains the options to apply when opening a database.
|
||||
// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
|
||||
type OpenOptions struct { |
||||
Type string // "leveldb" | "pebble"
|
||||
Directory string // the datadir
|
||||
AncientsDirectory string // the ancients-dir
|
||||
Namespace string // the namespace for database relevant metrics
|
||||
Cache int // the capacity(in megabytes) of the data caching
|
||||
Handles int // number of files to be open simultaneously
|
||||
ReadOnly bool |
||||
} |
||||
|
||||
// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
|
||||
//
|
||||
// type == null type != null
|
||||
// +----------------------------------------
|
||||
// db is non-existent | leveldb default | specified type
|
||||
// db is existent | from db | specified type (if compatible)
|
||||
func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { |
||||
existingDb := hasPreexistingDb(o.Directory) |
||||
if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb { |
||||
return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb) |
||||
} |
||||
if o.Type == dbPebble || existingDb == dbPebble { |
||||
if PebbleEnabled { |
||||
log.Info("Using pebble as the backing database") |
||||
return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) |
||||
} else { |
||||
return nil, errors.New("db.engine 'pebble' not supported on this platform") |
||||
} |
||||
} |
||||
if len(o.Type) != 0 && o.Type != dbLeveldb { |
||||
return nil, fmt.Errorf("unknown db.engine %v", o.Type) |
||||
} |
||||
log.Info("Using leveldb as the backing database") |
||||
// Use leveldb, either as default (no explicit choice), or pre-existing, or chosen explicitly
|
||||
return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) |
||||
} |
||||
|
||||
type counter uint64 |
||||
|
||||
func (c counter) String() string { |
||||
return fmt.Sprintf("%d", c) |
||||
} |
||||
|
||||
func (c counter) Percentage(current uint64) string { |
||||
return fmt.Sprintf("%d", current*100/uint64(c)) |
||||
} |
||||
|
||||
// stat stores sizes and count for a parameter
|
||||
type stat struct { |
||||
size common.StorageSize |
||||
count counter |
||||
} |
||||
|
||||
// Add size to the stat and increase the counter by 1
|
||||
func (s *stat) Add(size common.StorageSize) { |
||||
s.size += size |
||||
s.count++ |
||||
} |
||||
|
||||
func (s *stat) Size() string { |
||||
return s.size.String() |
||||
} |
||||
|
||||
func (s *stat) Count() string { |
||||
return s.count.String() |
||||
} |
||||
|
||||
// InspectDatabase traverses the entire database and checks the size
|
||||
// of all different categories of data.
|
||||
// This function is NOT used, just ported over from the Ethereum
|
||||
func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { |
||||
it := db.NewIterator(keyPrefix, keyStart) |
||||
defer it.Release() |
||||
|
||||
var ( |
||||
count int64 |
||||
start = time.Now() |
||||
logged = time.Now() |
||||
|
||||
// Key-value store statistics
|
||||
headers stat |
||||
bodies stat |
||||
receipts stat |
||||
tds stat |
||||
numHashPairings stat |
||||
hashNumPairings stat |
||||
tries stat |
||||
codes stat |
||||
txLookups stat |
||||
accountSnaps stat |
||||
storageSnaps stat |
||||
preimages stat |
||||
bloomBits stat |
||||
beaconHeaders stat |
||||
cliqueSnaps stat |
||||
|
||||
// Les statistic
|
||||
chtTrieNodes stat |
||||
bloomTrieNodes stat |
||||
|
||||
// Meta- and unaccounted data
|
||||
metadata stat |
||||
unaccounted stat |
||||
|
||||
// Totals
|
||||
total common.StorageSize |
||||
) |
||||
// Inspect key-value database first.
|
||||
for it.Next() { |
||||
var ( |
||||
key = it.Key() |
||||
size = common.StorageSize(len(key) + len(it.Value())) |
||||
) |
||||
total += size |
||||
switch { |
||||
case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): |
||||
headers.Add(size) |
||||
case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): |
||||
bodies.Add(size) |
||||
case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): |
||||
receipts.Add(size) |
||||
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): |
||||
tds.Add(size) |
||||
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): |
||||
numHashPairings.Add(size) |
||||
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): |
||||
hashNumPairings.Add(size) |
||||
case len(key) == common.HashLength: |
||||
tries.Add(size) |
||||
case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: |
||||
codes.Add(size) |
||||
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): |
||||
txLookups.Add(size) |
||||
case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): |
||||
accountSnaps.Add(size) |
||||
case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): |
||||
storageSnaps.Add(size) |
||||
case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): |
||||
preimages.Add(size) |
||||
case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): |
||||
metadata.Add(size) |
||||
case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): |
||||
metadata.Add(size) |
||||
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): |
||||
bloomBits.Add(size) |
||||
case bytes.HasPrefix(key, BloomBitsIndexPrefix): |
||||
bloomBits.Add(size) |
||||
case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): |
||||
beaconHeaders.Add(size) |
||||
case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: |
||||
cliqueSnaps.Add(size) |
||||
case bytes.HasPrefix(key, ChtTablePrefix) || |
||||
bytes.HasPrefix(key, ChtIndexTablePrefix) || |
||||
bytes.HasPrefix(key, ChtPrefix): // Canonical hash trie
|
||||
chtTrieNodes.Add(size) |
||||
case bytes.HasPrefix(key, BloomTrieTablePrefix) || |
||||
bytes.HasPrefix(key, BloomTrieIndexPrefix) || |
||||
bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub
|
||||
bloomTrieNodes.Add(size) |
||||
default: |
||||
var accounted bool |
||||
for _, meta := range [][]byte{ |
||||
databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, headFinalizedBlockKey, |
||||
lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, |
||||
snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, |
||||
uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, |
||||
} { |
||||
if bytes.Equal(key, meta) { |
||||
metadata.Add(size) |
||||
accounted = true |
||||
break |
||||
} |
||||
} |
||||
if !accounted { |
||||
unaccounted.Add(size) |
||||
} |
||||
} |
||||
count++ |
||||
if count%1000 == 0 && time.Since(logged) > 8*time.Second { |
||||
log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) |
||||
logged = time.Now() |
||||
} |
||||
} |
||||
// Display the database statistic of key-value store.
|
||||
stats := [][]string{ |
||||
{"Key-Value store", "Headers", headers.Size(), headers.Count()}, |
||||
{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, |
||||
{"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, |
||||
{"Key-Value store", "Difficulties", tds.Size(), tds.Count()}, |
||||
{"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, |
||||
{"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, |
||||
{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, |
||||
{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, |
||||
{"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, |
||||
{"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, |
||||
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, |
||||
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, |
||||
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, |
||||
{"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, |
||||
{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, |
||||
{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, |
||||
{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, |
||||
{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, |
||||
} |
||||
// Inspect all registered append-only file store then.
|
||||
ancients, err := inspectFreezers(db) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
for _, ancient := range ancients { |
||||
for _, table := range ancient.sizes { |
||||
stats = append(stats, []string{ |
||||
fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)), |
||||
strings.Title(table.name), |
||||
table.size.String(), |
||||
fmt.Sprintf("%d", ancient.count()), |
||||
}) |
||||
} |
||||
total += ancient.size() |
||||
} |
||||
table := tablewriter.NewWriter(os.Stdout) |
||||
table.SetHeader([]string{"Database", "Category", "Size", "Items"}) |
||||
table.SetFooter([]string{"", "Total", total.String(), " "}) |
||||
table.AppendBulk(stats) |
||||
table.Render() |
||||
|
||||
if unaccounted.size > 0 { |
||||
utils.Logger().Error(). |
||||
Interface("size", unaccounted.size). |
||||
Interface("count", unaccounted.count). |
||||
Msg("Database contains unaccounted data") |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,17 @@ |
||||
// Copyright 2017 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
@ -0,0 +1,37 @@ |
||||
// Copyright 2023 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
|
||||
|
||||
//go:build arm64 || amd64
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/ethereum/go-ethereum/ethdb/pebble" |
||||
) |
||||
|
||||
// Pebble is unsuported on 32bit architecture
|
||||
const PebbleEnabled = true |
||||
|
||||
// NewPebbleDBDatabase creates a persistent key-value database without a freezer
|
||||
// moving immutable chain segments into cold storage.
|
||||
func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { |
||||
db, err := pebble.New(file, cache, handles, namespace, readonly) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return NewDatabase(db), nil |
||||
} |
||||
@ -0,0 +1,34 @@ |
||||
// Copyright 2023 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//go:build !(arm64 || amd64)
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"errors" |
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
// Pebble is unsuported on 32bit architecture
|
||||
const PebbleEnabled = false |
||||
|
||||
// NewPebbleDBDatabase creates a persistent key-value database without a freezer
|
||||
// moving immutable chain segments into cold storage.
|
||||
func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { |
||||
return nil, errors.New("pebble is not supported on this platform") |
||||
} |
||||
@ -0,0 +1,47 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import "github.com/ethereum/go-ethereum/ethdb" |
||||
|
||||
// KeyLengthIterator is a wrapper for a database iterator that ensures only key-value pairs
|
||||
// with a specific key length will be returned.
|
||||
type KeyLengthIterator struct { |
||||
requiredKeyLength int |
||||
ethdb.Iterator |
||||
} |
||||
|
||||
// NewKeyLengthIterator returns a wrapped version of the iterator that will only return key-value
|
||||
// pairs where keys with a specific key length will be returned.
|
||||
func NewKeyLengthIterator(it ethdb.Iterator, keyLen int) ethdb.Iterator { |
||||
return &KeyLengthIterator{ |
||||
Iterator: it, |
||||
requiredKeyLength: keyLen, |
||||
} |
||||
} |
||||
|
||||
func (it *KeyLengthIterator) Next() bool { |
||||
// Return true as soon as a key with the required key length is discovered
|
||||
for it.Iterator.Next() { |
||||
if len(it.Iterator.Key()) == it.requiredKeyLength { |
||||
return true |
||||
} |
||||
} |
||||
|
||||
// Return false when we exhaust the keys in the underlying iterator.
|
||||
return false |
||||
} |
||||
@ -0,0 +1,60 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"testing" |
||||
) |
||||
|
||||
func TestKeyLengthIterator(t *testing.T) { |
||||
db := NewMemoryDatabase() |
||||
|
||||
keyLen := 8 |
||||
expectedKeys := make(map[string]struct{}) |
||||
for i := 0; i < 100; i++ { |
||||
key := make([]byte, keyLen) |
||||
binary.BigEndian.PutUint64(key, uint64(i)) |
||||
if err := db.Put(key, []byte{0x1}); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
expectedKeys[string(key)] = struct{}{} |
||||
|
||||
longerKey := make([]byte, keyLen*2) |
||||
binary.BigEndian.PutUint64(longerKey, uint64(i)) |
||||
if err := db.Put(longerKey, []byte{0x1}); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
|
||||
it := NewKeyLengthIterator(db.NewIterator(nil, nil), keyLen) |
||||
for it.Next() { |
||||
key := it.Key() |
||||
_, exists := expectedKeys[string(key)] |
||||
if !exists { |
||||
t.Fatalf("Found unexpected key %d", binary.BigEndian.Uint64(key)) |
||||
} |
||||
delete(expectedKeys, string(key)) |
||||
if len(key) != keyLen { |
||||
t.Fatalf("Found unexpected key in key length iterator with length %d", len(key)) |
||||
} |
||||
} |
||||
|
||||
if len(expectedKeys) != 0 { |
||||
t.Fatalf("Expected all keys of length %d to be removed from expected keys during iteration", keyLen) |
||||
} |
||||
} |
||||
@ -0,0 +1,313 @@ |
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
// table is a wrapper around a database that prefixes each key access with a pre-
|
||||
// configured string.
|
||||
type table struct { |
||||
db ethdb.Database |
||||
prefix string |
||||
} |
||||
|
||||
// NewTable returns a database object that prefixes all keys with a given string.
|
||||
func NewTable(db ethdb.Database, prefix string) ethdb.Database { |
||||
return &table{ |
||||
db: db, |
||||
prefix: prefix, |
||||
} |
||||
} |
||||
|
||||
// Close is a noop to implement the Database interface.
|
||||
func (t *table) Close() error { |
||||
return nil |
||||
} |
||||
|
||||
// Has retrieves if a prefixed version of a key is present in the database.
|
||||
func (t *table) Has(key []byte) (bool, error) { |
||||
return t.db.Has(append([]byte(t.prefix), key...)) |
||||
} |
||||
|
||||
// Get retrieves the given prefixed key if it's present in the database.
|
||||
func (t *table) Get(key []byte) ([]byte, error) { |
||||
return t.db.Get(append([]byte(t.prefix), key...)) |
||||
} |
||||
|
||||
// HasAncient is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) HasAncient(kind string, number uint64) (bool, error) { |
||||
return t.db.HasAncient(kind, number) |
||||
} |
||||
|
||||
// Ancient is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) Ancient(kind string, number uint64) ([]byte, error) { |
||||
return t.db.Ancient(kind, number) |
||||
} |
||||
|
||||
// AncientRange is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { |
||||
return t.db.AncientRange(kind, start, count, maxBytes) |
||||
} |
||||
|
||||
// Ancients is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) Ancients() (uint64, error) { |
||||
return t.db.Ancients() |
||||
} |
||||
|
||||
// Tail is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) Tail() (uint64, error) { |
||||
return t.db.Tail() |
||||
} |
||||
|
||||
// AncientSize is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) AncientSize(kind string) (uint64, error) { |
||||
return t.db.AncientSize(kind) |
||||
} |
||||
|
||||
// ModifyAncients runs an ancient write operation on the underlying database.
|
||||
func (t *table) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (int64, error) { |
||||
return t.db.ModifyAncients(fn) |
||||
} |
||||
|
||||
func (t *table) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { |
||||
return t.db.ReadAncients(fn) |
||||
} |
||||
|
||||
// TruncateHead is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) TruncateHead(items uint64) error { |
||||
return t.db.TruncateHead(items) |
||||
} |
||||
|
||||
// TruncateTail is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) TruncateTail(items uint64) error { |
||||
return t.db.TruncateTail(items) |
||||
} |
||||
|
||||
// Sync is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) Sync() error { |
||||
return t.db.Sync() |
||||
} |
||||
|
||||
// MigrateTable processes the entries in a given table in sequence
|
||||
// converting them to a new format if they're of an old format.
|
||||
func (t *table) MigrateTable(kind string, convert convertLegacyFn) error { |
||||
return t.db.MigrateTable(kind, convert) |
||||
} |
||||
|
||||
// AncientDatadir returns the ancient datadir of the underlying database.
|
||||
func (t *table) AncientDatadir() (string, error) { |
||||
return t.db.AncientDatadir() |
||||
} |
||||
|
||||
// Put inserts the given value into the database at a prefixed version of the
|
||||
// provided key.
|
||||
func (t *table) Put(key []byte, value []byte) error { |
||||
return t.db.Put(append([]byte(t.prefix), key...), value) |
||||
} |
||||
|
||||
// Delete removes the given prefixed key from the database.
|
||||
func (t *table) Delete(key []byte) error { |
||||
return t.db.Delete(append([]byte(t.prefix), key...)) |
||||
} |
||||
|
||||
// NewIterator creates a binary-alphabetical iterator over a subset
|
||||
// of database content with a particular key prefix, starting at a particular
|
||||
// initial key (or after, if it does not exist).
|
||||
func (t *table) NewIterator(prefix []byte, start []byte) ethdb.Iterator { |
||||
innerPrefix := append([]byte(t.prefix), prefix...) |
||||
iter := t.db.NewIterator(innerPrefix, start) |
||||
return &tableIterator{ |
||||
iter: iter, |
||||
prefix: t.prefix, |
||||
} |
||||
} |
||||
|
||||
// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
|
||||
// of database content with a particular key prefix.
|
||||
func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { |
||||
return t.NewIterator(prefix, nil) |
||||
} |
||||
|
||||
// Stat returns a particular internal stat of the database.
|
||||
func (t *table) Stat(property string) (string, error) { |
||||
return t.db.Stat(property) |
||||
} |
||||
|
||||
// Compact flattens the underlying data store for the given key range. In essence,
|
||||
// deleted and overwritten versions are discarded, and the data is rearranged to
|
||||
// reduce the cost of operations needed to access them.
|
||||
//
|
||||
// A nil start is treated as a key before all keys in the data store; a nil limit
|
||||
// is treated as a key after all keys in the data store. If both is nil then it
|
||||
// will compact entire data store.
|
||||
func (t *table) Compact(start []byte, limit []byte) error { |
||||
// If no start was specified, use the table prefix as the first value
|
||||
if start == nil { |
||||
start = []byte(t.prefix) |
||||
} else { |
||||
start = append([]byte(t.prefix), start...) |
||||
} |
||||
// If no limit was specified, use the first element not matching the prefix
|
||||
// as the limit
|
||||
if limit == nil { |
||||
limit = []byte(t.prefix) |
||||
for i := len(limit) - 1; i >= 0; i-- { |
||||
// Bump the current character, stopping if it doesn't overflow
|
||||
limit[i]++ |
||||
if limit[i] > 0 { |
||||
break |
||||
} |
||||
// Character overflown, proceed to the next or nil if the last
|
||||
if i == 0 { |
||||
limit = nil |
||||
} |
||||
} |
||||
} else { |
||||
limit = append([]byte(t.prefix), limit...) |
||||
} |
||||
// Range correctly calculated based on table prefix, delegate down
|
||||
return t.db.Compact(start, limit) |
||||
} |
||||
|
||||
// NewBatch creates a write-only database that buffers changes to its host db
|
||||
// until a final write is called, each operation prefixing all keys with the
|
||||
// pre-configured string.
|
||||
func (t *table) NewBatch() ethdb.Batch { |
||||
return &tableBatch{t.db.NewBatch(), t.prefix} |
||||
} |
||||
|
||||
// NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
|
||||
func (t *table) NewBatchWithSize(size int) ethdb.Batch { |
||||
return &tableBatch{t.db.NewBatchWithSize(size), t.prefix} |
||||
} |
||||
|
||||
// NewSnapshot creates a database snapshot based on the current state.
|
||||
// The created snapshot will not be affected by all following mutations
|
||||
// happened on the database.
|
||||
func (t *table) NewSnapshot() (ethdb.Snapshot, error) { |
||||
return t.db.NewSnapshot() |
||||
} |
||||
|
||||
// tableBatch is a wrapper around a database batch that prefixes each key access
|
||||
// with a pre-configured string.
|
||||
type tableBatch struct { |
||||
batch ethdb.Batch |
||||
prefix string |
||||
} |
||||
|
||||
// Put inserts the given value into the batch for later committing.
|
||||
func (b *tableBatch) Put(key, value []byte) error { |
||||
return b.batch.Put(append([]byte(b.prefix), key...), value) |
||||
} |
||||
|
||||
// Delete inserts the a key removal into the batch for later committing.
|
||||
func (b *tableBatch) Delete(key []byte) error { |
||||
return b.batch.Delete(append([]byte(b.prefix), key...)) |
||||
} |
||||
|
||||
// ValueSize retrieves the amount of data queued up for writing.
|
||||
func (b *tableBatch) ValueSize() int { |
||||
return b.batch.ValueSize() |
||||
} |
||||
|
||||
// Write flushes any accumulated data to disk.
|
||||
func (b *tableBatch) Write() error { |
||||
return b.batch.Write() |
||||
} |
||||
|
||||
// Reset resets the batch for reuse.
|
||||
func (b *tableBatch) Reset() { |
||||
b.batch.Reset() |
||||
} |
||||
|
||||
// tableReplayer is a wrapper around a batch replayer which truncates
|
||||
// the added prefix.
|
||||
type tableReplayer struct { |
||||
w ethdb.KeyValueWriter |
||||
prefix string |
||||
} |
||||
|
||||
// Put implements the interface KeyValueWriter.
|
||||
func (r *tableReplayer) Put(key []byte, value []byte) error { |
||||
trimmed := key[len(r.prefix):] |
||||
return r.w.Put(trimmed, value) |
||||
} |
||||
|
||||
// Delete implements the interface KeyValueWriter.
|
||||
func (r *tableReplayer) Delete(key []byte) error { |
||||
trimmed := key[len(r.prefix):] |
||||
return r.w.Delete(trimmed) |
||||
} |
||||
|
||||
// Replay replays the batch contents.
|
||||
func (b *tableBatch) Replay(w ethdb.KeyValueWriter) error { |
||||
return b.batch.Replay(&tableReplayer{w: w, prefix: b.prefix}) |
||||
} |
||||
|
||||
// tableIterator is a wrapper around a database iterator that prefixes each key access
|
||||
// with a pre-configured string.
|
||||
type tableIterator struct { |
||||
iter ethdb.Iterator |
||||
prefix string |
||||
} |
||||
|
||||
// Next moves the iterator to the next key/value pair. It returns whether the
|
||||
// iterator is exhausted.
|
||||
func (iter *tableIterator) Next() bool { |
||||
return iter.iter.Next() |
||||
} |
||||
|
||||
// Error returns any accumulated error. Exhausting all the key/value pairs
|
||||
// is not considered to be an error.
|
||||
func (iter *tableIterator) Error() error { |
||||
return iter.iter.Error() |
||||
} |
||||
|
||||
// Key returns the key of the current key/value pair, or nil if done. The caller
|
||||
// should not modify the contents of the returned slice, and its contents may
|
||||
// change on the next call to Next.
|
||||
func (iter *tableIterator) Key() []byte { |
||||
key := iter.iter.Key() |
||||
if key == nil { |
||||
return nil |
||||
} |
||||
return key[len(iter.prefix):] |
||||
} |
||||
|
||||
// Value returns the value of the current key/value pair, or nil if done. The
|
||||
// caller should not modify the contents of the returned slice, and its contents
|
||||
// may change on the next call to Next.
|
||||
func (iter *tableIterator) Value() []byte { |
||||
return iter.iter.Value() |
||||
} |
||||
|
||||
// Release releases associated resources. Release should always succeed and can
|
||||
// be called multiple times without causing error.
|
||||
func (iter *tableIterator) Release() { |
||||
iter.iter.Release() |
||||
} |
||||
@ -0,0 +1,128 @@ |
||||
// Copyright 2020 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb |
||||
|
||||
import ( |
||||
"bytes" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
) |
||||
|
||||
func TestTableDatabase(t *testing.T) { testTableDatabase(t, "prefix") } |
||||
func TestEmptyPrefixTableDatabase(t *testing.T) { testTableDatabase(t, "") } |
||||
|
||||
type testReplayer struct { |
||||
puts [][]byte |
||||
dels [][]byte |
||||
} |
||||
|
||||
func (r *testReplayer) Put(key []byte, value []byte) error { |
||||
r.puts = append(r.puts, key) |
||||
return nil |
||||
} |
||||
|
||||
func (r *testReplayer) Delete(key []byte) error { |
||||
r.dels = append(r.dels, key) |
||||
return nil |
||||
} |
||||
|
||||
func testTableDatabase(t *testing.T, prefix string) { |
||||
db := NewTable(NewMemoryDatabase(), prefix) |
||||
|
||||
var entries = []struct { |
||||
key []byte |
||||
value []byte |
||||
}{ |
||||
{[]byte{0x01, 0x02}, []byte{0x0a, 0x0b}}, |
||||
{[]byte{0x03, 0x04}, []byte{0x0c, 0x0d}}, |
||||
{[]byte{0x05, 0x06}, []byte{0x0e, 0x0f}}, |
||||
|
||||
{[]byte{0xff, 0xff, 0x01}, []byte{0x1a, 0x1b}}, |
||||
{[]byte{0xff, 0xff, 0x02}, []byte{0x1c, 0x1d}}, |
||||
{[]byte{0xff, 0xff, 0x03}, []byte{0x1e, 0x1f}}, |
||||
} |
||||
|
||||
// Test Put/Get operation
|
||||
for _, entry := range entries { |
||||
db.Put(entry.key, entry.value) |
||||
} |
||||
for _, entry := range entries { |
||||
got, err := db.Get(entry.key) |
||||
if err != nil { |
||||
t.Fatalf("Failed to get value: %v", err) |
||||
} |
||||
if !bytes.Equal(got, entry.value) { |
||||
t.Fatalf("Value mismatch: want=%v, got=%v", entry.value, got) |
||||
} |
||||
} |
||||
|
||||
// Test batch operation
|
||||
db = NewTable(NewMemoryDatabase(), prefix) |
||||
batch := db.NewBatch() |
||||
for _, entry := range entries { |
||||
batch.Put(entry.key, entry.value) |
||||
} |
||||
batch.Write() |
||||
for _, entry := range entries { |
||||
got, err := db.Get(entry.key) |
||||
if err != nil { |
||||
t.Fatalf("Failed to get value: %v", err) |
||||
} |
||||
if !bytes.Equal(got, entry.value) { |
||||
t.Fatalf("Value mismatch: want=%v, got=%v", entry.value, got) |
||||
} |
||||
} |
||||
|
||||
// Test batch replayer
|
||||
r := &testReplayer{} |
||||
batch.Replay(r) |
||||
for index, entry := range entries { |
||||
got := r.puts[index] |
||||
if !bytes.Equal(got, entry.key) { |
||||
t.Fatalf("Key mismatch: want=%v, got=%v", entry.key, got) |
||||
} |
||||
} |
||||
|
||||
check := func(iter ethdb.Iterator, expCount, index int) { |
||||
count := 0 |
||||
for iter.Next() { |
||||
key, value := iter.Key(), iter.Value() |
||||
if !bytes.Equal(key, entries[index].key) { |
||||
t.Fatalf("Key mismatch: want=%v, got=%v", entries[index].key, key) |
||||
} |
||||
if !bytes.Equal(value, entries[index].value) { |
||||
t.Fatalf("Value mismatch: want=%v, got=%v", entries[index].value, value) |
||||
} |
||||
index += 1 |
||||
count++ |
||||
} |
||||
if count != expCount { |
||||
t.Fatalf("Wrong number of elems, exp %d got %d", expCount, count) |
||||
} |
||||
iter.Release() |
||||
} |
||||
// Test iterators
|
||||
check(db.NewIterator(nil, nil), 6, 0) |
||||
// Test iterators with prefix
|
||||
check(db.NewIterator([]byte{0xff, 0xff}, nil), 3, 3) |
||||
// Test iterators with start point
|
||||
check(db.NewIterator(nil, []byte{0xff, 0xff, 0x02}), 2, 4) |
||||
// Test iterators with prefix and start point
|
||||
check(db.NewIterator([]byte{0xee}, nil), 0, 0) |
||||
check(db.NewIterator(nil, []byte{0x00}), 6, 0) |
||||
} |
||||
Binary file not shown.
@ -0,0 +1,136 @@ |
||||
// Copyright 2020 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/common" |
||||
) |
||||
|
||||
type accessList struct { |
||||
addresses map[common.Address]int |
||||
slots []map[common.Hash]struct{} |
||||
} |
||||
|
||||
// ContainsAddress returns true if the address is in the access list.
|
||||
func (al *accessList) ContainsAddress(address common.Address) bool { |
||||
_, ok := al.addresses[address] |
||||
return ok |
||||
} |
||||
|
||||
// Contains checks if a slot within an account is present in the access list, returning
|
||||
// separate flags for the presence of the account and the slot respectively.
|
||||
func (al *accessList) Contains(address common.Address, slot common.Hash) (addressPresent bool, slotPresent bool) { |
||||
idx, ok := al.addresses[address] |
||||
if !ok { |
||||
// no such address (and hence zero slots)
|
||||
return false, false |
||||
} |
||||
if idx == -1 { |
||||
// address yes, but no slots
|
||||
return true, false |
||||
} |
||||
_, slotPresent = al.slots[idx][slot] |
||||
return true, slotPresent |
||||
} |
||||
|
||||
// newAccessList creates a new accessList.
|
||||
func newAccessList() *accessList { |
||||
return &accessList{ |
||||
addresses: make(map[common.Address]int), |
||||
} |
||||
} |
||||
|
||||
// Copy creates an independent copy of an accessList.
|
||||
func (a *accessList) Copy() *accessList { |
||||
cp := newAccessList() |
||||
for k, v := range a.addresses { |
||||
cp.addresses[k] = v |
||||
} |
||||
cp.slots = make([]map[common.Hash]struct{}, len(a.slots)) |
||||
for i, slotMap := range a.slots { |
||||
newSlotmap := make(map[common.Hash]struct{}, len(slotMap)) |
||||
for k := range slotMap { |
||||
newSlotmap[k] = struct{}{} |
||||
} |
||||
cp.slots[i] = newSlotmap |
||||
} |
||||
return cp |
||||
} |
||||
|
||||
// AddAddress adds an address to the access list, and returns 'true' if the operation
|
||||
// caused a change (addr was not previously in the list).
|
||||
func (al *accessList) AddAddress(address common.Address) bool { |
||||
if _, present := al.addresses[address]; present { |
||||
return false |
||||
} |
||||
al.addresses[address] = -1 |
||||
return true |
||||
} |
||||
|
||||
// AddSlot adds the specified (addr, slot) combo to the access list.
|
||||
// Return values are:
|
||||
// - address added
|
||||
// - slot added
|
||||
// For any 'true' value returned, a corresponding journal entry must be made.
|
||||
func (al *accessList) AddSlot(address common.Address, slot common.Hash) (addrChange bool, slotChange bool) { |
||||
idx, addrPresent := al.addresses[address] |
||||
if !addrPresent || idx == -1 { |
||||
// Address not present, or addr present but no slots there
|
||||
al.addresses[address] = len(al.slots) |
||||
slotmap := map[common.Hash]struct{}{slot: {}} |
||||
al.slots = append(al.slots, slotmap) |
||||
return !addrPresent, true |
||||
} |
||||
// There is already an (address,slot) mapping
|
||||
slotmap := al.slots[idx] |
||||
if _, ok := slotmap[slot]; !ok { |
||||
slotmap[slot] = struct{}{} |
||||
// Journal add slot change
|
||||
return false, true |
||||
} |
||||
// No changes required
|
||||
return false, false |
||||
} |
||||
|
||||
// DeleteSlot removes an (address, slot)-tuple from the access list.
|
||||
// This operation needs to be performed in the same order as the addition happened.
|
||||
// This method is meant to be used by the journal, which maintains ordering of
|
||||
// operations.
|
||||
func (al *accessList) DeleteSlot(address common.Address, slot common.Hash) { |
||||
idx, addrOk := al.addresses[address] |
||||
// There are two ways this can fail
|
||||
if !addrOk { |
||||
panic("reverting slot change, address not present in list") |
||||
} |
||||
slotmap := al.slots[idx] |
||||
delete(slotmap, slot) |
||||
// If that was the last (first) slot, remove it
|
||||
// Since additions and rollbacks are always performed in order,
|
||||
// we can delete the item without worrying about screwing up later indices
|
||||
if len(slotmap) == 0 { |
||||
al.slots = al.slots[:idx] |
||||
al.addresses[address] = -1 |
||||
} |
||||
} |
||||
|
||||
// DeleteAddress removes an address from the access list. This operation
|
||||
// needs to be performed in the same order as the addition happened.
|
||||
// This method is meant to be used by the journal, which maintains ordering of
|
||||
// operations.
|
||||
func (al *accessList) DeleteAddress(address common.Address) { |
||||
delete(al.addresses, address) |
||||
} |
||||
@ -0,0 +1,155 @@ |
||||
// Copyright 2015 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/core/types" |
||||
"github.com/ethereum/go-ethereum/rlp" |
||||
"github.com/ethereum/go-ethereum/trie" |
||||
) |
||||
|
||||
// NodeIterator is an iterator to traverse the entire state trie post-order,
|
||||
// including all of the contract code and contract state tries.
|
||||
type NodeIterator struct { |
||||
state *DB // State being iterated
|
||||
|
||||
stateIt trie.NodeIterator // Primary iterator for the global state trie
|
||||
dataIt trie.NodeIterator // Secondary iterator for the data trie of a contract
|
||||
|
||||
accountHash common.Hash // Hash of the node containing the account
|
||||
codeHash common.Hash // Hash of the contract source code
|
||||
code []byte // Source code associated with a contract
|
||||
|
||||
Hash common.Hash // Hash of the current entry being iterated (nil if not standalone)
|
||||
Parent common.Hash // Hash of the first full ancestor node (nil if current is the root)
|
||||
|
||||
Error error // Failure set in case of an internal error in the iterator
|
||||
} |
||||
|
||||
// NewNodeIterator creates an post-order state node iterator.
|
||||
func NewNodeIterator(state *DB) *NodeIterator { |
||||
return &NodeIterator{ |
||||
state: state, |
||||
} |
||||
} |
||||
|
||||
// Next moves the iterator to the next node, returning whether there are any
|
||||
// further nodes. In case of an internal error this method returns false and
|
||||
// sets the Error field to the encountered failure.
|
||||
func (it *NodeIterator) Next() bool { |
||||
// If the iterator failed previously, don't do anything
|
||||
if it.Error != nil { |
||||
return false |
||||
} |
||||
// Otherwise step forward with the iterator and report any errors
|
||||
if err := it.step(); err != nil { |
||||
it.Error = err |
||||
return false |
||||
} |
||||
return it.retrieve() |
||||
} |
||||
|
||||
// step moves the iterator to the next entry of the state trie.
|
||||
func (it *NodeIterator) step() error { |
||||
// Abort if we reached the end of the iteration
|
||||
if it.state == nil { |
||||
return nil |
||||
} |
||||
// Initialize the iterator if we've just started
|
||||
if it.stateIt == nil { |
||||
it.stateIt = it.state.trie.NodeIterator(nil) |
||||
} |
||||
// If we had data nodes previously, we surely have at least state nodes
|
||||
if it.dataIt != nil { |
||||
if cont := it.dataIt.Next(true); !cont { |
||||
if it.dataIt.Error() != nil { |
||||
return it.dataIt.Error() |
||||
} |
||||
it.dataIt = nil |
||||
} |
||||
return nil |
||||
} |
||||
// If we had source code previously, discard that
|
||||
if it.code != nil { |
||||
it.code = nil |
||||
return nil |
||||
} |
||||
// Step to the next state trie node, terminating if we're out of nodes
|
||||
if cont := it.stateIt.Next(true); !cont { |
||||
if it.stateIt.Error() != nil { |
||||
return it.stateIt.Error() |
||||
} |
||||
it.state, it.stateIt = nil, nil |
||||
return nil |
||||
} |
||||
// If the state trie node is an internal entry, leave as is
|
||||
if !it.stateIt.Leaf() { |
||||
return nil |
||||
} |
||||
// Otherwise we've reached an account node, initiate data iteration
|
||||
var account Account |
||||
if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { |
||||
return err |
||||
} |
||||
dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, common.BytesToHash(it.stateIt.LeafKey()), account.Root) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
it.dataIt = dataTrie.NodeIterator(nil) |
||||
if !it.dataIt.Next(true) { |
||||
it.dataIt = nil |
||||
} |
||||
if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { |
||||
it.codeHash = common.BytesToHash(account.CodeHash) |
||||
addrHash := common.BytesToHash(it.stateIt.LeafKey()) |
||||
it.code, err = it.state.db.ContractCode(addrHash, common.BytesToHash(account.CodeHash)) |
||||
if err != nil { |
||||
return fmt.Errorf("code %x: %v", account.CodeHash, err) |
||||
} |
||||
} |
||||
it.accountHash = it.stateIt.Parent() |
||||
return nil |
||||
} |
||||
|
||||
// retrieve pulls and caches the current state entry the iterator is traversing.
|
||||
// The method returns whether there are any more data left for inspection.
|
||||
func (it *NodeIterator) retrieve() bool { |
||||
// Clear out any previously set values
|
||||
it.Hash = common.Hash{} |
||||
|
||||
// If the iteration's done, return no available data
|
||||
if it.state == nil { |
||||
return false |
||||
} |
||||
// Otherwise retrieve the current entry
|
||||
switch { |
||||
case it.dataIt != nil: |
||||
it.Hash, it.Parent = it.dataIt.Hash(), it.dataIt.Parent() |
||||
if it.Parent == (common.Hash{}) { |
||||
it.Parent = it.accountHash |
||||
} |
||||
case it.code != nil: |
||||
it.Hash, it.Parent = it.codeHash, it.accountHash |
||||
case it.stateIt != nil: |
||||
it.Hash, it.Parent = it.stateIt.Hash(), it.stateIt.Parent() |
||||
} |
||||
return true |
||||
} |
||||
@ -0,0 +1,142 @@ |
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import ( |
||||
"math/big" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/crypto" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/harmony-one/harmony/core/rawdb" |
||||
) |
||||
|
||||
// testAccount is the data associated with an account used by the state tests.
|
||||
type testAccount struct { |
||||
address common.Address |
||||
balance *big.Int |
||||
nonce uint64 |
||||
code []byte |
||||
} |
||||
|
||||
// makeTestState create a sample test state to test node-wise reconstruction.
|
||||
func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { |
||||
// Create an empty state
|
||||
db := rawdb.NewMemoryDatabase() |
||||
sdb := NewDatabase(db) |
||||
state, _ := New(common.Hash{}, sdb, nil) |
||||
|
||||
// Fill it with some arbitrary data
|
||||
var accounts []*testAccount |
||||
for i := byte(0); i < 96; i++ { |
||||
obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i})) |
||||
acc := &testAccount{address: common.BytesToAddress([]byte{i})} |
||||
|
||||
obj.AddBalance(big.NewInt(int64(11 * i))) |
||||
acc.balance = big.NewInt(int64(11 * i)) |
||||
|
||||
obj.SetNonce(uint64(42 * i)) |
||||
acc.nonce = uint64(42 * i) |
||||
|
||||
if i%3 == 0 { |
||||
obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i}) |
||||
acc.code = []byte{i, i, i, i, i} |
||||
} |
||||
if i%5 == 0 { |
||||
for j := byte(0); j < 5; j++ { |
||||
hash := crypto.Keccak256Hash([]byte{i, i, i, i, i, j, j}) |
||||
obj.SetState(sdb, hash, hash) |
||||
} |
||||
} |
||||
state.updateStateObject(obj) |
||||
accounts = append(accounts, acc) |
||||
} |
||||
root, _ := state.Commit(false) |
||||
|
||||
// Return the generated state
|
||||
return db, sdb, root, accounts |
||||
} |
||||
|
||||
// Tests that the node iterator indeed walks over the entire database contents.
|
||||
func TestNodeIteratorCoverage(t *testing.T) { |
||||
// Create some arbitrary test state to iterate
|
||||
db, sdb, root, _ := makeTestState() |
||||
sdb.TrieDB().Commit(root, false) |
||||
|
||||
state, err := New(root, sdb, nil) |
||||
if err != nil { |
||||
t.Fatalf("failed to create state trie at %x: %v", root, err) |
||||
} |
||||
// Gather all the node hashes found by the iterator
|
||||
hashes := make(map[common.Hash]struct{}) |
||||
for it := NewNodeIterator(state); it.Next(); { |
||||
if it.Hash != (common.Hash{}) { |
||||
hashes[it.Hash] = struct{}{} |
||||
} |
||||
} |
||||
// Check in-disk nodes
|
||||
var ( |
||||
seenNodes = make(map[common.Hash]struct{}) |
||||
seenCodes = make(map[common.Hash]struct{}) |
||||
) |
||||
it := db.NewIterator(nil, nil) |
||||
for it.Next() { |
||||
ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value()) |
||||
if !ok { |
||||
continue |
||||
} |
||||
seenNodes[hash] = struct{}{} |
||||
} |
||||
it.Release() |
||||
|
||||
// Check in-disk codes
|
||||
it = db.NewIterator(nil, nil) |
||||
for it.Next() { |
||||
ok, hash := rawdb.IsCodeKey(it.Key()) |
||||
if !ok { |
||||
continue |
||||
} |
||||
if _, ok := hashes[common.BytesToHash(hash)]; !ok { |
||||
t.Errorf("state entry not reported %x", it.Key()) |
||||
} |
||||
seenCodes[common.BytesToHash(hash)] = struct{}{} |
||||
} |
||||
it.Release() |
||||
|
||||
// Cross check the iterated hashes and the database/nodepool content
|
||||
for hash := range hashes { |
||||
_, ok := seenNodes[hash] |
||||
if !ok { |
||||
_, ok = seenCodes[hash] |
||||
} |
||||
if !ok { |
||||
t.Errorf("failed to retrieve reported node %x", hash) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// isTrieNode is a helper function which reports if the provided
|
||||
// database entry belongs to a trie node or not.
|
||||
func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) { |
||||
if scheme == rawdb.HashScheme { |
||||
if len(key) == common.HashLength { |
||||
return true, common.BytesToHash(key) |
||||
} |
||||
} |
||||
return false, common.Hash{} |
||||
} |
||||
@ -0,0 +1,30 @@ |
||||
// Copyright 2021 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import "github.com/ethereum/go-ethereum/metrics" |
||||
|
||||
var ( |
||||
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) |
||||
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) |
||||
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) |
||||
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) |
||||
accountTrieUpdatedMeter = metrics.NewRegisteredMeter("state/update/accountnodes", nil) |
||||
storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil) |
||||
accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil) |
||||
storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil) |
||||
) |
||||
@ -0,0 +1,46 @@ |
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import ( |
||||
"bytes" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
) |
||||
|
||||
func BenchmarkCutOriginal(b *testing.B) { |
||||
value := common.HexToHash("0x01") |
||||
for i := 0; i < b.N; i++ { |
||||
bytes.TrimLeft(value[:], "\x00") |
||||
} |
||||
} |
||||
|
||||
func BenchmarkCutsetterFn(b *testing.B) { |
||||
value := common.HexToHash("0x01") |
||||
cutSetFn := func(r rune) bool { return r == 0 } |
||||
for i := 0; i < b.N; i++ { |
||||
bytes.TrimLeftFunc(value[:], cutSetFn) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkCutCustomTrim(b *testing.B) { |
||||
value := common.HexToHash("0x01") |
||||
for i := 0; i < b.N; i++ { |
||||
common.TrimLeftZeroes(value[:]) |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,56 @@ |
||||
// Copyright 2015 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import ( |
||||
"bytes" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/ethdb" |
||||
"github.com/ethereum/go-ethereum/rlp" |
||||
"github.com/ethereum/go-ethereum/trie" |
||||
) |
||||
|
||||
// NewStateSync create a new state trie download scheduler.
|
||||
func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync { |
||||
// Register the storage slot callback if the external callback is specified.
|
||||
var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error |
||||
if onLeaf != nil { |
||||
onSlot = func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { |
||||
return onLeaf(keys, leaf) |
||||
} |
||||
} |
||||
// Register the account callback to connect the state trie and the storage
|
||||
// trie belongs to the contract.
|
||||
var syncer *trie.Sync |
||||
onAccount := func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { |
||||
if onLeaf != nil { |
||||
if err := onLeaf(keys, leaf); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
var obj Account |
||||
if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { |
||||
return err |
||||
} |
||||
syncer.AddSubTrie(obj.Root, path, parent, parentPath, onSlot) |
||||
syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), path, parent, parentPath) |
||||
return nil |
||||
} |
||||
syncer = trie.NewSync(root, database, onAccount, scheme) |
||||
return syncer |
||||
} |
||||
@ -0,0 +1,55 @@ |
||||
// Copyright 2022 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/common" |
||||
) |
||||
|
||||
// transientStorage is a representation of EIP-1153 "Transient Storage".
|
||||
type transientStorage map[common.Address]Storage |
||||
|
||||
// newTransientStorage creates a new instance of a transientStorage.
|
||||
func newTransientStorage() transientStorage { |
||||
return make(transientStorage) |
||||
} |
||||
|
||||
// Set sets the transient-storage `value` for `key` at the given `addr`.
|
||||
func (t transientStorage) Set(addr common.Address, key, value common.Hash) { |
||||
if _, ok := t[addr]; !ok { |
||||
t[addr] = make(Storage) |
||||
} |
||||
t[addr][key] = value |
||||
} |
||||
|
||||
// Get gets the transient storage for `key` at the given `addr`.
|
||||
func (t transientStorage) Get(addr common.Address, key common.Hash) common.Hash { |
||||
val, ok := t[addr] |
||||
if !ok { |
||||
return common.Hash{} |
||||
} |
||||
return val[key] |
||||
} |
||||
|
||||
// Copy does a deep copy of the transientStorage
|
||||
func (t transientStorage) Copy() transientStorage { |
||||
storage := make(transientStorage) |
||||
for key, value := range t { |
||||
storage[key] = value.Copy() |
||||
} |
||||
return storage |
||||
} |
||||
@ -0,0 +1,354 @@ |
||||
// Copyright 2020 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import ( |
||||
"sync" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/metrics" |
||||
"github.com/harmony-one/harmony/internal/utils" |
||||
) |
||||
|
||||
var ( |
||||
// triePrefetchMetricsPrefix is the prefix under which to publish the metrics.
|
||||
triePrefetchMetricsPrefix = "trie/prefetch/" |
||||
) |
||||
|
||||
// triePrefetcher is an active prefetcher, which receives accounts or storage
|
||||
// items and does trie-loading of them. The goal is to get as much useful content
|
||||
// into the caches as possible.
|
||||
//
|
||||
// Note, the prefetcher's API is not thread safe.
|
||||
type triePrefetcher struct { |
||||
db Database // Database to fetch trie nodes through
|
||||
root common.Hash // Root hash of the account trie for metrics
|
||||
fetches map[string]Trie // Partially or fully fetcher tries
|
||||
fetchers map[string]*subfetcher // Subfetchers for each trie
|
||||
|
||||
deliveryMissMeter metrics.Meter |
||||
accountLoadMeter metrics.Meter |
||||
accountDupMeter metrics.Meter |
||||
accountSkipMeter metrics.Meter |
||||
accountWasteMeter metrics.Meter |
||||
storageLoadMeter metrics.Meter |
||||
storageDupMeter metrics.Meter |
||||
storageSkipMeter metrics.Meter |
||||
storageWasteMeter metrics.Meter |
||||
} |
||||
|
||||
func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher { |
||||
prefix := triePrefetchMetricsPrefix + namespace |
||||
p := &triePrefetcher{ |
||||
db: db, |
||||
root: root, |
||||
fetchers: make(map[string]*subfetcher), // Active prefetchers use the fetchers map
|
||||
|
||||
deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil), |
||||
accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil), |
||||
accountDupMeter: metrics.GetOrRegisterMeter(prefix+"/account/dup", nil), |
||||
accountSkipMeter: metrics.GetOrRegisterMeter(prefix+"/account/skip", nil), |
||||
accountWasteMeter: metrics.GetOrRegisterMeter(prefix+"/account/waste", nil), |
||||
storageLoadMeter: metrics.GetOrRegisterMeter(prefix+"/storage/load", nil), |
||||
storageDupMeter: metrics.GetOrRegisterMeter(prefix+"/storage/dup", nil), |
||||
storageSkipMeter: metrics.GetOrRegisterMeter(prefix+"/storage/skip", nil), |
||||
storageWasteMeter: metrics.GetOrRegisterMeter(prefix+"/storage/waste", nil), |
||||
} |
||||
return p |
||||
} |
||||
|
||||
// close iterates over all the subfetchers, aborts any that were left spinning
|
||||
// and reports the stats to the metrics subsystem.
|
||||
func (p *triePrefetcher) close() { |
||||
for _, fetcher := range p.fetchers { |
||||
fetcher.abort() // safe to do multiple times
|
||||
|
||||
if metrics.Enabled { |
||||
if fetcher.root == p.root { |
||||
p.accountLoadMeter.Mark(int64(len(fetcher.seen))) |
||||
p.accountDupMeter.Mark(int64(fetcher.dups)) |
||||
p.accountSkipMeter.Mark(int64(len(fetcher.tasks))) |
||||
|
||||
for _, key := range fetcher.used { |
||||
delete(fetcher.seen, string(key)) |
||||
} |
||||
p.accountWasteMeter.Mark(int64(len(fetcher.seen))) |
||||
} else { |
||||
p.storageLoadMeter.Mark(int64(len(fetcher.seen))) |
||||
p.storageDupMeter.Mark(int64(fetcher.dups)) |
||||
p.storageSkipMeter.Mark(int64(len(fetcher.tasks))) |
||||
|
||||
for _, key := range fetcher.used { |
||||
delete(fetcher.seen, string(key)) |
||||
} |
||||
p.storageWasteMeter.Mark(int64(len(fetcher.seen))) |
||||
} |
||||
} |
||||
} |
||||
// Clear out all fetchers (will crash on a second call, deliberate)
|
||||
p.fetchers = nil |
||||
} |
||||
|
||||
// copy creates a deep-but-inactive copy of the trie prefetcher. Any trie data
|
||||
// already loaded will be copied over, but no goroutines will be started. This
|
||||
// is mostly used in the miner which creates a copy of it's actively mutated
|
||||
// state to be sealed while it may further mutate the state.
|
||||
func (p *triePrefetcher) copy() *triePrefetcher { |
||||
copy := &triePrefetcher{ |
||||
db: p.db, |
||||
root: p.root, |
||||
fetches: make(map[string]Trie), // Active prefetchers use the fetches map
|
||||
|
||||
deliveryMissMeter: p.deliveryMissMeter, |
||||
accountLoadMeter: p.accountLoadMeter, |
||||
accountDupMeter: p.accountDupMeter, |
||||
accountSkipMeter: p.accountSkipMeter, |
||||
accountWasteMeter: p.accountWasteMeter, |
||||
storageLoadMeter: p.storageLoadMeter, |
||||
storageDupMeter: p.storageDupMeter, |
||||
storageSkipMeter: p.storageSkipMeter, |
||||
storageWasteMeter: p.storageWasteMeter, |
||||
} |
||||
// If the prefetcher is already a copy, duplicate the data
|
||||
if p.fetches != nil { |
||||
for root, fetch := range p.fetches { |
||||
if fetch == nil { |
||||
continue |
||||
} |
||||
copy.fetches[root] = p.db.CopyTrie(fetch) |
||||
} |
||||
return copy |
||||
} |
||||
// Otherwise we're copying an active fetcher, retrieve the current states
|
||||
for id, fetcher := range p.fetchers { |
||||
copy.fetches[id] = fetcher.peek() |
||||
} |
||||
return copy |
||||
} |
||||
|
||||
// prefetch schedules a batch of trie items to prefetch.
|
||||
func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]byte) { |
||||
// If the prefetcher is an inactive one, bail out
|
||||
if p.fetches != nil { |
||||
return |
||||
} |
||||
// Active fetcher, schedule the retrievals
|
||||
id := p.trieID(owner, root) |
||||
fetcher := p.fetchers[id] |
||||
if fetcher == nil { |
||||
fetcher = newSubfetcher(p.db, p.root, owner, root) |
||||
p.fetchers[id] = fetcher |
||||
} |
||||
fetcher.schedule(keys) |
||||
} |
||||
|
||||
// trie returns the trie matching the root hash, or nil if the prefetcher doesn't
|
||||
// have it.
|
||||
func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie { |
||||
// If the prefetcher is inactive, return from existing deep copies
|
||||
id := p.trieID(owner, root) |
||||
if p.fetches != nil { |
||||
trie := p.fetches[id] |
||||
if trie == nil { |
||||
p.deliveryMissMeter.Mark(1) |
||||
return nil |
||||
} |
||||
return p.db.CopyTrie(trie) |
||||
} |
||||
// Otherwise the prefetcher is active, bail if no trie was prefetched for this root
|
||||
fetcher := p.fetchers[id] |
||||
if fetcher == nil { |
||||
p.deliveryMissMeter.Mark(1) |
||||
return nil |
||||
} |
||||
// Interrupt the prefetcher if it's by any chance still running and return
|
||||
// a copy of any pre-loaded trie.
|
||||
fetcher.abort() // safe to do multiple times
|
||||
|
||||
trie := fetcher.peek() |
||||
if trie == nil { |
||||
p.deliveryMissMeter.Mark(1) |
||||
return nil |
||||
} |
||||
return trie |
||||
} |
||||
|
||||
// used marks a batch of state items used to allow creating statistics as to
|
||||
// how useful or wasteful the prefetcher is.
|
||||
func (p *triePrefetcher) used(owner common.Hash, root common.Hash, used [][]byte) { |
||||
if fetcher := p.fetchers[p.trieID(owner, root)]; fetcher != nil { |
||||
fetcher.used = used |
||||
} |
||||
} |
||||
|
||||
// trieID returns an unique trie identifier consists the trie owner and root hash.
|
||||
func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string { |
||||
return string(append(owner.Bytes(), root.Bytes()...)) |
||||
} |
||||
|
||||
// subfetcher is a trie fetcher goroutine responsible for pulling entries for a
|
||||
// single trie. It is spawned when a new root is encountered and lives until the
|
||||
// main prefetcher is paused and either all requested items are processed or if
|
||||
// the trie being worked on is retrieved from the prefetcher.
|
||||
type subfetcher struct { |
||||
db Database // Database to load trie nodes through
|
||||
state common.Hash // Root hash of the state to prefetch
|
||||
owner common.Hash // Owner of the trie, usually account hash
|
||||
root common.Hash // Root hash of the trie to prefetch
|
||||
trie Trie // Trie being populated with nodes
|
||||
|
||||
tasks [][]byte // Items queued up for retrieval
|
||||
lock sync.Mutex // Lock protecting the task queue
|
||||
|
||||
wake chan struct{} // Wake channel if a new task is scheduled
|
||||
stop chan struct{} // Channel to interrupt processing
|
||||
term chan struct{} // Channel to signal interruption
|
||||
copy chan chan Trie // Channel to request a copy of the current trie
|
||||
|
||||
seen map[string]struct{} // Tracks the entries already loaded
|
||||
dups int // Number of duplicate preload tasks
|
||||
used [][]byte // Tracks the entries used in the end
|
||||
} |
||||
|
||||
// newSubfetcher creates a goroutine to prefetch state items belonging to a
|
||||
// particular root hash.
|
||||
func newSubfetcher(db Database, state common.Hash, owner common.Hash, root common.Hash) *subfetcher { |
||||
sf := &subfetcher{ |
||||
db: db, |
||||
state: state, |
||||
owner: owner, |
||||
root: root, |
||||
wake: make(chan struct{}, 1), |
||||
stop: make(chan struct{}), |
||||
term: make(chan struct{}), |
||||
copy: make(chan chan Trie), |
||||
seen: make(map[string]struct{}), |
||||
} |
||||
go sf.loop() |
||||
return sf |
||||
} |
||||
|
||||
// schedule adds a batch of trie keys to the queue to prefetch.
|
||||
func (sf *subfetcher) schedule(keys [][]byte) { |
||||
// Append the tasks to the current queue
|
||||
sf.lock.Lock() |
||||
sf.tasks = append(sf.tasks, keys...) |
||||
sf.lock.Unlock() |
||||
|
||||
// Notify the prefetcher, it's fine if it's already terminated
|
||||
select { |
||||
case sf.wake <- struct{}{}: |
||||
default: |
||||
} |
||||
} |
||||
|
||||
// peek tries to retrieve a deep copy of the fetcher's trie in whatever form it
|
||||
// is currently.
|
||||
func (sf *subfetcher) peek() Trie { |
||||
ch := make(chan Trie) |
||||
select { |
||||
case sf.copy <- ch: |
||||
// Subfetcher still alive, return copy from it
|
||||
return <-ch |
||||
|
||||
case <-sf.term: |
||||
// Subfetcher already terminated, return a copy directly
|
||||
if sf.trie == nil { |
||||
return nil |
||||
} |
||||
return sf.db.CopyTrie(sf.trie) |
||||
} |
||||
} |
||||
|
||||
// abort interrupts the subfetcher immediately. It is safe to call abort multiple
|
||||
// times but it is not thread safe.
|
||||
func (sf *subfetcher) abort() { |
||||
select { |
||||
case <-sf.stop: |
||||
default: |
||||
close(sf.stop) |
||||
} |
||||
<-sf.term |
||||
} |
||||
|
||||
// loop waits for new tasks to be scheduled and keeps loading them until it runs
|
||||
// out of tasks or its underlying trie is retrieved for committing.
|
||||
func (sf *subfetcher) loop() { |
||||
// No matter how the loop stops, signal anyone waiting that it's terminated
|
||||
defer close(sf.term) |
||||
|
||||
// Start by opening the trie and stop processing if it fails
|
||||
if sf.owner == (common.Hash{}) { |
||||
trie, err := sf.db.OpenTrie(sf.root) |
||||
if err != nil { |
||||
utils.Logger().Warn().Err(err).Interface("root", sf.root).Msg("Trie prefetcher failed opening trie") |
||||
return |
||||
} |
||||
sf.trie = trie |
||||
} else { |
||||
trie, err := sf.db.OpenStorageTrie(sf.state, sf.owner, sf.root) |
||||
if err != nil { |
||||
utils.Logger().Warn().Err(err).Interface("root", sf.root).Msg("Trie prefetcher failed opening trie") |
||||
return |
||||
} |
||||
sf.trie = trie |
||||
} |
||||
// Trie opened successfully, keep prefetching items
|
||||
for { |
||||
select { |
||||
case <-sf.wake: |
||||
// Subfetcher was woken up, retrieve any tasks to avoid spinning the lock
|
||||
sf.lock.Lock() |
||||
tasks := sf.tasks |
||||
sf.tasks = nil |
||||
sf.lock.Unlock() |
||||
|
||||
// Prefetch any tasks until the loop is interrupted
|
||||
for i, task := range tasks { |
||||
select { |
||||
case <-sf.stop: |
||||
// If termination is requested, add any leftover back and return
|
||||
sf.lock.Lock() |
||||
sf.tasks = append(sf.tasks, tasks[i:]...) |
||||
sf.lock.Unlock() |
||||
return |
||||
|
||||
case ch := <-sf.copy: |
||||
// Somebody wants a copy of the current trie, grant them
|
||||
ch <- sf.db.CopyTrie(sf.trie) |
||||
|
||||
default: |
||||
// No termination request yet, prefetch the next entry
|
||||
if _, ok := sf.seen[string(task)]; ok { |
||||
sf.dups++ |
||||
} else { |
||||
sf.trie.TryGet(task) |
||||
sf.seen[string(task)] = struct{}{} |
||||
} |
||||
} |
||||
} |
||||
|
||||
case ch := <-sf.copy: |
||||
// Somebody wants a copy of the current trie, grant them
|
||||
ch <- sf.db.CopyTrie(sf.trie) |
||||
|
||||
case <-sf.stop: |
||||
// Termination is requested, abort and leave remaining tasks
|
||||
return |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,110 @@ |
||||
// Copyright 2021 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package state |
||||
|
||||
import ( |
||||
"math/big" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/harmony-one/harmony/core/rawdb" |
||||
) |
||||
|
||||
func filledStateDB() *DB { |
||||
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) |
||||
|
||||
// Create an account and check if the retrieved balance is correct
|
||||
addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") |
||||
skey := common.HexToHash("aaa") |
||||
sval := common.HexToHash("bbb") |
||||
|
||||
state.SetBalance(addr, big.NewInt(42)) // Change the account trie
|
||||
state.SetCode(addr, []byte("hello")) // Change an external metadata
|
||||
state.SetState(addr, skey, sval) // Change the storage trie
|
||||
for i := 0; i < 100; i++ { |
||||
sk := common.BigToHash(big.NewInt(int64(i))) |
||||
state.SetState(addr, sk, sk) // Change the storage trie
|
||||
} |
||||
return state |
||||
} |
||||
|
||||
func TestCopyAndClose(t *testing.T) { |
||||
db := filledStateDB() |
||||
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") |
||||
skey := common.HexToHash("aaa") |
||||
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) |
||||
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) |
||||
time.Sleep(1 * time.Second) |
||||
a := prefetcher.trie(common.Hash{}, db.originalRoot) |
||||
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) |
||||
b := prefetcher.trie(common.Hash{}, db.originalRoot) |
||||
cpy := prefetcher.copy() |
||||
cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) |
||||
cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) |
||||
c := cpy.trie(common.Hash{}, db.originalRoot) |
||||
prefetcher.close() |
||||
cpy2 := cpy.copy() |
||||
cpy2.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) |
||||
d := cpy2.trie(common.Hash{}, db.originalRoot) |
||||
cpy.close() |
||||
cpy2.close() |
||||
if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() { |
||||
t.Fatalf("Invalid trie, hashes should be equal: %v %v %v %v", a.Hash(), b.Hash(), c.Hash(), d.Hash()) |
||||
} |
||||
} |
||||
|
||||
func TestUseAfterClose(t *testing.T) { |
||||
db := filledStateDB() |
||||
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") |
||||
skey := common.HexToHash("aaa") |
||||
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) |
||||
a := prefetcher.trie(common.Hash{}, db.originalRoot) |
||||
prefetcher.close() |
||||
b := prefetcher.trie(common.Hash{}, db.originalRoot) |
||||
if a == nil { |
||||
t.Fatal("Prefetching before close should not return nil") |
||||
} |
||||
if b != nil { |
||||
t.Fatal("Trie after close should return nil") |
||||
} |
||||
} |
||||
|
||||
func TestCopyClose(t *testing.T) { |
||||
db := filledStateDB() |
||||
prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") |
||||
skey := common.HexToHash("aaa") |
||||
prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) |
||||
cpy := prefetcher.copy() |
||||
a := prefetcher.trie(common.Hash{}, db.originalRoot) |
||||
b := cpy.trie(common.Hash{}, db.originalRoot) |
||||
prefetcher.close() |
||||
c := prefetcher.trie(common.Hash{}, db.originalRoot) |
||||
d := cpy.trie(common.Hash{}, db.originalRoot) |
||||
if a == nil { |
||||
t.Fatal("Prefetching before close should not return nil") |
||||
} |
||||
if b == nil { |
||||
t.Fatal("Copy trie should return nil") |
||||
} |
||||
if c != nil { |
||||
t.Fatal("Trie after close should return nil") |
||||
} |
||||
if d == nil { |
||||
t.Fatal("Copy trie should not return nil") |
||||
} |
||||
} |
||||
Loading…
Reference in new issue