From 33e3a9ecf01dd1bf048b072086ede1dc5d290396 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Thu, 25 Jan 2024 12:16:21 +0100 Subject: [PATCH] fix storage Signed-off-by: Karim Taam --- .../controller/BesuControllerBuilder.java | 4 +- .../besu/ethereum/chain/GenesisState.java | 86 ++++++++++--------- ...nsaiSnapshotWorldStateKeyValueStorage.java | 1 + .../bonsai/worldview/BonsaiWorldState.java | 9 ++ ...asedSnapshotWorldStateKeyValueStorage.java | 5 +- .../common/trielog/NoOpTrieLogManager.java | 8 +- .../common/worldview/DiffBasedWorldState.java | 32 +++---- .../VerkleNoOpCachedWorldStorageManager.java | 2 - ...rkleSnapshotWorldStateKeyValueStorage.java | 1 + .../verkle/worldview/VerkleWorldState.java | 11 ++- 10 files changed, 87 insertions(+), 72 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 6e6bf9fb6e..bf9ed73d89 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -594,8 +594,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides final ProtocolSchedule protocolSchedule = createProtocolSchedule(); final GenesisState genesisState = - GenesisState.fromConfig( - dataStorageConfiguration.getDataStorageFormat(), genesisConfig, protocolSchedule); + GenesisState.fromConfig( + dataStorageConfiguration.getDataStorageFormat(), genesisConfig, protocolSchedule); final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index 17075f288c..42268e9369 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -37,8 +37,8 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValue import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiNoOpCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.VerkleNoOpCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview.VerkleWorldState; @@ -51,6 +51,7 @@ import org.hyperledger.besu.evm.log.LogsBloomFilter; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; import java.math.BigInteger; import java.util.HashMap; @@ -66,7 +67,6 @@ import com.google.common.base.MoreObjects; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; public final class GenesisState { @@ -87,9 +87,9 @@ public final class GenesisState { * @return A new {@link GenesisState}. */ public static GenesisState fromJson( - final DataStorageFormat dataStorageFormat, - final String json, - final ProtocolSchedule protocolSchedule) { + final DataStorageFormat dataStorageFormat, + final String json, + final ProtocolSchedule protocolSchedule) { return fromConfig(dataStorageFormat, GenesisConfigFile.fromConfig(json), protocolSchedule); } @@ -105,13 +105,15 @@ public final class GenesisState { } /** - * Construct a {@link GenesisState} from a JSON object with FOREST as a default data storage format. + * Construct a {@link GenesisState} from a JSON object with FOREST as a default data storage + * format. + * * @param config A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with */ public static GenesisState fromConfig( - final GenesisConfigFile config, final ProtocolSchedule protocolSchedule) { - return fromConfig(DataStorageFormat.FOREST, config, protocolSchedule); + final GenesisConfigFile config, final ProtocolSchedule protocolSchedule) { + return fromConfig(DataStorageFormat.FOREST, config, protocolSchedule); } /** * Construct a {@link GenesisState} from a JSON object. @@ -122,17 +124,17 @@ public final class GenesisState { * @return A new {@link GenesisState}. */ public static GenesisState fromConfig( - final DataStorageFormat dataStorageFormat, - final GenesisConfigFile config, - final ProtocolSchedule protocolSchedule) { + final DataStorageFormat dataStorageFormat, + final GenesisConfigFile config, + final ProtocolSchedule protocolSchedule) { final List genesisAccounts = parseAllocations(config).toList(); final Block block = - new Block( - buildHeader( - config, - calculateGenesisStateHash(dataStorageFormat, genesisAccounts), - protocolSchedule), - buildBody(config)); + new Block( + buildHeader( + config, + calculateGenesisStateHash(dataStorageFormat, genesisAccounts), + protocolSchedule), + buildBody(config)); return new GenesisState(block, genesisAccounts); } @@ -176,7 +178,7 @@ public final class GenesisState { } private static Hash calculateGenesisStateHash( - final DataStorageFormat dataStorageFormat, final List genesisAccounts) { + final DataStorageFormat dataStorageFormat, final List genesisAccounts) { final MutableWorldState worldState = loadWorldState(dataStorageFormat); writeAccountsTo(worldState, genesisAccounts, null); return worldState.rootHash(); @@ -186,40 +188,40 @@ public final class GenesisState { switch (dataStorageFormat) { case BONSAI -> { final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader = - new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = - new BonsaiWorldStateKeyValueStorage( - new KeyValueStorageProvider( - segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), - new InMemoryKeyValueStorage(), - new NoOpMetricsSystem()), - new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + new KeyValueStorageProvider( + segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), + new InMemoryKeyValueStorage(), + new NoOpMetricsSystem()), + new NoOpMetricsSystem()); return new BonsaiWorldState( - bonsaiWorldStateKeyValueStorage, - cachedMerkleTrieLoader, - new BonsaiNoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), - new NoOpTrieLogManager(), - EvmConfiguration.DEFAULT); + bonsaiWorldStateKeyValueStorage, + cachedMerkleTrieLoader, + new BonsaiNoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), + new NoOpTrieLogManager(), + EvmConfiguration.DEFAULT); } case VERKLE -> { final VerkleWorldStateKeyValueStorage verkleWorldStateKeyValueStorage = - new VerkleWorldStateKeyValueStorage( - new KeyValueStorageProvider( - segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), - new InMemoryKeyValueStorage(), - new NoOpMetricsSystem()), - new NoOpMetricsSystem()); + new VerkleWorldStateKeyValueStorage( + new KeyValueStorageProvider( + segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), + new InMemoryKeyValueStorage(), + new NoOpMetricsSystem()), + new NoOpMetricsSystem()); return new VerkleWorldState( - verkleWorldStateKeyValueStorage, - new VerkleNoOpCachedWorldStorageManager(verkleWorldStateKeyValueStorage), - new NoOpTrieLogManager(), - EvmConfiguration.DEFAULT); + verkleWorldStateKeyValueStorage, + new VerkleNoOpCachedWorldStorageManager(verkleWorldStateKeyValueStorage), + new NoOpTrieLogManager(), + EvmConfiguration.DEFAULT); } default -> { final ForestWorldStateKeyValueStorage stateStorage = - new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStatePreimageKeyValueStorage preimageStorage = - new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); + new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); return new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 1a47246014..af5ab27cde 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -229,6 +229,7 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey } } + @Override public BonsaiWorldStateKeyValueStorage getParentWorldStateStorage() { return parentWorldStateStorage; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java index 1b35363361..cedd889845 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java @@ -409,6 +409,15 @@ public class BonsaiWorldState extends DiffBasedWorldState { return Hash.hash(value); } + @Override + public Hash frontierRootHash() { + return calculateRootHash( + Optional.of( + new BonsaiWorldStateKeyValueStorage.Updater( + noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), + accumulator.copy()); + } + @Override protected Hash getEmptyTrieHash() { return Hash.EMPTY_TRIE_HASH; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java index 2100243f7c..9c1a691cf3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java @@ -15,4 +15,7 @@ */ package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; -public interface DiffBasedSnapshotWorldStateKeyValueStorage {} +public interface DiffBasedSnapshotWorldStateKeyValueStorage { + + DiffBasedWorldStateKeyValueStorage getParentWorldStateStorage(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java index 593ea89e5b..11836f3344 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java @@ -31,10 +31,10 @@ public class NoOpTrieLogManager extends TrieLogManager { @SuppressWarnings({"UnsynchronizedOverridesSynchronized", "squid:S3551"}) @Override public void saveTrieLog( - final DiffBasedWorldStateUpdateAccumulator localUpdater, - final Hash forWorldStateRootHash, - final BlockHeader forBlockHeader, - final DiffBasedWorldState forWorldState) { + final DiffBasedWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java index 06419f002e..8805f98a8c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java @@ -23,11 +23,9 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; @@ -113,7 +111,7 @@ public abstract class DiffBasedWorldState } private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { - return !(worldStateKeyValueStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); + return !(worldStateKeyValueStorage instanceof DiffBasedSnapshotWorldStateKeyValueStorage); } /** @@ -172,13 +170,13 @@ public abstract class DiffBasedWorldState .getWorldStateTransaction() .put( TRIE_BRANCH_STORAGE, - BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY, + DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); worldStateBlockHash = blockHeader.getHash(); } else { stateUpdater .getWorldStateTransaction() - .remove(TRIE_BRANCH_STORAGE, BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY); + .remove(TRIE_BRANCH_STORAGE, DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY); worldStateBlockHash = null; } @@ -186,7 +184,7 @@ public abstract class DiffBasedWorldState .getWorldStateTransaction() .put( TRIE_BRANCH_STORAGE, - BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY, + DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); worldStateRootHash = newWorldStateRootHash; success = true; @@ -226,7 +224,7 @@ public abstract class DiffBasedWorldState return Hash.wrap(worldStateRootHash); } - static final KeyValueStorageTransaction noOpTx = + protected static final KeyValueStorageTransaction noOpTx = new KeyValueStorageTransaction() { @Override @@ -250,7 +248,7 @@ public abstract class DiffBasedWorldState } }; - static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = + protected static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = new SegmentedKeyValueStorageTransaction() { @Override @@ -275,15 +273,6 @@ public abstract class DiffBasedWorldState } }; - @Override - public Hash frontierRootHash() { - return calculateRootHash( - Optional.of( - new BonsaiWorldStateKeyValueStorage.Updater( - noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), - accumulator.copy()); - } - public Hash blockHash() { return worldStateBlockHash; } @@ -314,8 +303,8 @@ public abstract class DiffBasedWorldState private void closeFrozenStorage() { try { - final BonsaiWorldStateLayerStorage worldStateLayerStorage = - (BonsaiWorldStateLayerStorage) worldStateKeyValueStorage; + final DiffBasedSnapshotWorldStateKeyValueStorage worldStateLayerStorage = + (DiffBasedSnapshotWorldStateKeyValueStorage) worldStateKeyValueStorage; if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { worldStateLayerStorage.getParentWorldStateStorage().close(); } @@ -324,6 +313,9 @@ public abstract class DiffBasedWorldState } } + @Override + public abstract Hash frontierRootHash(); + @Override public abstract MutableWorldState freeze(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java index 9df56a9e54..fe9f8e5472 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java @@ -16,8 +16,6 @@ package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java index bb797e7904..ada211afa9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java @@ -195,6 +195,7 @@ public class VerkleSnapshotWorldStateKeyValueStorage extends VerkleWorldStateKey } } + @Override public VerkleWorldStateKeyValueStorage getParentWorldStateStorage() { return parentWorldStateStorage; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index b8b5b804db..0d5269ecd9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -137,7 +137,7 @@ public class VerkleWorldState extends DiffBasedWorldState { })); LOG.info("end commit "); - //LOG.info(stateTrie.toDotTree()); + // LOG.info(stateTrie.toDotTree()); final Bytes32 rootHash = stateTrie.getRootHash(); LOG.info("end commit "); @@ -354,6 +354,15 @@ public class VerkleWorldState extends DiffBasedWorldState { return Hash.hash(value); } + @Override + public Hash frontierRootHash() { + return calculateRootHash( + Optional.of( + new VerkleWorldStateKeyValueStorage.Updater( + noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), + accumulator.copy()); + } + @Override protected Hash getEmptyTrieHash() { return Hash.wrap(Bytes32.ZERO);