From 3ce7f0ff7a53e098776df93f6b4c3df76c996dc9 Mon Sep 17 00:00:00 2001 From: matkt Date: Mon, 18 Jul 2022 20:59:05 +0200 Subject: [PATCH] candidate fix for missing parent worldstate (#4094) * candidate fix for missing parent worldstate * add rollback if failure * fix persist trie log issue * add trielog manager * fix inmemory issue for bonsai Signed-off-by: Karim TAAM Co-authored-by: garyschulte Co-authored-by: Justin Florentine --- .../controller/BesuControllerBuilder.java | 9 +- .../controller/BesuControllerBuilderTest.java | 11 +- .../bonsai/BonsaiInMemoryWorldState.java | 26 ++- .../bonsai/BonsaiPersistedWorldState.java | 61 +----- .../bonsai/BonsaiWorldStateArchive.java | 109 +++-------- .../besu/ethereum/bonsai/TrieLogManager.java | 182 ++++++++++++++++++ .../core/InMemoryKeyValueStorageProvider.java | 10 +- .../bonsai/BonsaiWorldStateArchiveTest.java | 59 +++++- .../besu/ethereum/bonsai/LogRollingTests.java | 12 +- .../besu/ethereum/bonsai/RollingImport.java | 6 +- 10 files changed, 332 insertions(+), 153 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogManager.java diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 612fb3fdfb..f791b269ee 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -32,6 +32,8 @@ import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods; import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.TrieLogManager; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.BlockchainStorage; import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; @@ -602,7 +604,12 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI: return new BonsaiWorldStateArchive( - storageProvider, blockchain, dataStorageConfiguration.getBonsaiMaxLayersToLoad()); + new TrieLogManager( + blockchain, + (BonsaiWorldStateKeyValueStorage) worldStateStorage, + dataStorageConfiguration.getBonsaiMaxLayersToLoad()), + storageProvider, + blockchain); case FOREST: default: final WorldStatePreimageStorage preimageStorage = diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index a6c1ce0947..331256df21 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.config.Keccak256ConfigOptions; import org.hyperledger.besu.crypto.NodeKey; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.GasLimitCalculator; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; @@ -46,6 +47,7 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; @@ -83,6 +85,7 @@ public class BesuControllerBuilderTest { @Mock StorageProvider storageProvider; @Mock GasLimitCalculator gasLimitCalculator; @Mock WorldStateStorage worldStateStorage; + @Mock BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorage; @Mock WorldStatePreimageStorage worldStatePreimageStorage; BigInteger networkId = BigInteger.ONE; @@ -128,7 +131,11 @@ public class BesuControllerBuilderTest { when(worldStatePreimageStorage.updater()) .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class)); - + BonsaiWorldStateKeyValueStorage.Updater bonsaiUpdater = + mock(BonsaiWorldStateKeyValueStorage.Updater.class); + when(bonsaiUpdater.getTrieLogStorageTransaction()) + .thenReturn(mock(KeyValueStorageTransaction.class)); + when(bonsaiWorldStateStorage.updater()).thenReturn(bonsaiUpdater); besuControllerBuilder = visitWithMockConfigs(new MainnetBesuControllerBuilder()); } @@ -152,6 +159,8 @@ public class BesuControllerBuilderTest { @Test public void shouldDisablePruningIfBonsaiIsEnabled() { + when(storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI)) + .thenReturn(bonsaiWorldStateStorage); besuControllerBuilder .isPruningEnabled(true) .dataStorageConfiguration( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java index 43e36e0081..f24238867f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java @@ -21,6 +21,8 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; public class BonsaiInMemoryWorldState extends BonsaiPersistedWorldState { + private boolean isPersisted = false; + public BonsaiInMemoryWorldState( final BonsaiWorldStateArchive archive, final BonsaiWorldStateKeyValueStorage worldStateStorage) { @@ -29,9 +31,16 @@ public class BonsaiInMemoryWorldState extends BonsaiPersistedWorldState { @Override public Hash rootHash() { + if (isPersisted) { + return worldStateRootHash; + } + return rootHash(updater.copy()); + } + + public Hash rootHash(final BonsaiWorldStateUpdater localUpdater) { final BonsaiWorldStateKeyValueStorage.Updater updater = worldStateStorage.updater(); try { - final Hash calculatedRootHash = calculateRootHash(updater); + final Hash calculatedRootHash = calculateRootHash(updater, localUpdater); return Hash.wrap(calculatedRootHash); } finally { updater.rollback(); @@ -41,13 +50,12 @@ public class BonsaiInMemoryWorldState extends BonsaiPersistedWorldState { @Override public void persist(final BlockHeader blockHeader) { final BonsaiWorldStateUpdater localUpdater = updater.copy(); - try { - final Hash newWorldStateRootHash = rootHash(); - prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash); - worldStateBlockHash = blockHeader.getHash(); - worldStateRootHash = newWorldStateRootHash; - } finally { - localUpdater.reset(); - } + final Hash newWorldStateRootHash = rootHash(localUpdater); + archive + .getTrieLogManager() + .saveTrieLog(archive, localUpdater, newWorldStateRootHash, blockHeader); + worldStateRootHash = newWorldStateRootHash; + worldStateBlockHash = blockHeader.getBlockHash(); + isPersisted = true; } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java index 9fd7b479b0..cafe2f48fb 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -25,7 +25,6 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.account.Account; @@ -100,10 +99,6 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld return worldStateStorage; } - protected Hash calculateRootHash(final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { - return calculateRootHash(stateUpdater, updater.copy()); - } - protected Hash calculateRootHash( final BonsaiWorldStateKeyValueStorage.Updater stateUpdater, final BonsaiWorldStateUpdater worldStateUpdater) { @@ -252,14 +247,16 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld // then persist the TrieLog for that transition. // If specified but not a direct descendant simply store the new block hash. if (blockHeader != null) { - // do not overwrite a trielog layer that already exists in the database. - // if it's only in memory we need to save it - // for example, like that in case of reorg we don't replace a trielog layer - if (worldStateStorage.getTrieLog(blockHeader.getHash()).isEmpty()) { - final TrieLogLayer trieLog = - prepareTrieLog(blockHeader, localUpdater, newWorldStateRootHash); - persistTrieLog(blockHeader, newWorldStateRootHash, trieLog, stateUpdater); + if (!newWorldStateRootHash.equals(blockHeader.getStateRoot())) { + throw new RuntimeException( + "World State Root does not match expected value, header " + + blockHeader.getStateRoot().toHexString() + + " calculated " + + newWorldStateRootHash.toHexString()); } + archive + .getTrieLogManager() + .saveTrieLog(archive, localUpdater, newWorldStateRootHash, blockHeader); stateUpdater .getTrieBranchStorageTransaction() .put(WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); @@ -283,46 +280,6 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld updater.reset(); } } - if (blockHeader != null) { - archive.scrubLayeredCache(blockHeader.getNumber()); - } - } - - protected TrieLogLayer prepareTrieLog( - final BlockHeader blockHeader, - final BonsaiWorldStateUpdater localUpdater, - final Hash currentWorldStateRootHash) { - - if (!currentWorldStateRootHash.equals(blockHeader.getStateRoot())) { - throw new RuntimeException( - "World State Root does not match expected value, header " - + blockHeader.getStateRoot().toHexString() - + " calculated " - + currentWorldStateRootHash.toHexString()); - } - - debugLambda(LOG, "Adding layered world state for {}", blockHeader::toLogString); - final TrieLogLayer trieLog = localUpdater.generateTrieLog(blockHeader.getBlockHash()); - trieLog.freeze(); - archive.addLayeredWorldState(this, blockHeader, currentWorldStateRootHash, trieLog); - return trieLog; - } - - private void persistTrieLog( - final BlockHeader blockHeader, - final Hash worldStateRootHash, - final TrieLogLayer trieLog, - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { - debugLambda( - LOG, - "Persisting trie log for block hash {} and world state root {}", - blockHeader::toLogString, - worldStateRootHash::toHexString); - final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); - trieLog.writeTo(rlpLog); - stateUpdater - .getTrieLogStorageTransaction() - .put(blockHeader.getHash().toArrayUnsafe(), rlpLog.encoded().toArrayUnsafe()); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java index ed69ab02f3..853ab83492 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -17,7 +17,6 @@ package org.hyperledger.besu.ethereum.bonsai; import static org.hyperledger.besu.datatypes.Hash.fromPlugin; -import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -31,13 +30,10 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.worldstate.WorldState; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,35 +42,20 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateArchive.class); - private static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks - private final Blockchain blockchain; + private final TrieLogManager trieLogManager; private final BonsaiPersistedWorldState persistedState; - private final Map layeredWorldStatesByHash; private final BonsaiWorldStateKeyValueStorage worldStateStorage; - private final long maxLayersToLoad; - - public BonsaiWorldStateArchive(final StorageProvider provider, final Blockchain blockchain) { - this(provider, blockchain, RETAINED_LAYERS, new HashMap<>()); - } - - public BonsaiWorldStateArchive( - final StorageProvider provider, final Blockchain blockchain, final long maxLayersToLoad) { - this(provider, blockchain, maxLayersToLoad, new HashMap<>()); - } public BonsaiWorldStateArchive( + final TrieLogManager trieLogManager, final StorageProvider provider, - final Blockchain blockchain, - final long maxLayersToLoad, - final Map layeredWorldStatesByHash) { + final Blockchain blockchain) { + this.trieLogManager = trieLogManager; this.blockchain = blockchain; - this.worldStateStorage = new BonsaiWorldStateKeyValueStorage(provider); this.persistedState = new BonsaiPersistedWorldState(this, worldStateStorage); - this.layeredWorldStatesByHash = layeredWorldStatesByHash; - this.maxLayersToLoad = maxLayersToLoad; blockchain.observeBlockAdded(this::blockAddedHandler); } @@ -82,22 +63,17 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { LOG.debug("New block add event {}", event); if (event.isNewCanonicalHead()) { final BlockHeader eventBlockHeader = event.getBlock().getHeader(); - layeredWorldStatesByHash.computeIfPresent( - eventBlockHeader.getParentHash(), - (parentHash, bonsaiLayeredWorldState) -> { - if (layeredWorldStatesByHash.containsKey(eventBlockHeader.getHash())) { - bonsaiLayeredWorldState.setNextWorldView( - Optional.of(layeredWorldStatesByHash.get(eventBlockHeader.getHash()))); - } - return bonsaiLayeredWorldState; - }); + trieLogManager.updateLayeredWorldState( + eventBlockHeader.getParentHash(), eventBlockHeader.getHash()); } } @Override public Optional get(final Hash rootHash, final Hash blockHash) { - if (layeredWorldStatesByHash.containsKey(blockHash)) { - return Optional.of(layeredWorldStatesByHash.get(blockHash)); + final Optional layeredWorldState = + trieLogManager.getBonsaiLayeredWorldState(blockHash); + if (layeredWorldState.isPresent()) { + return Optional.of(layeredWorldState.get()); } else if (rootHash.equals(persistedState.blockHash())) { return Optional.of(persistedState); } else { @@ -105,38 +81,9 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { } } - public void addLayeredWorldState( - final BonsaiWorldView persistedWorldState, - final BlockHeader blockHeader, - final Hash worldStateRootHash, - final TrieLogLayer trieLog) { - final BonsaiLayeredWorldState bonsaiLayeredWorldState = - new BonsaiLayeredWorldState( - blockchain, - this, - Optional.of(persistedWorldState), - blockHeader.getNumber(), - worldStateRootHash, - trieLog); - debugLambda( - LOG, - "adding layered world state for block {}, state root hash {}", - blockHeader::toLogString, - worldStateRootHash::toHexString); - layeredWorldStatesByHash.put(blockHeader.getHash(), bonsaiLayeredWorldState); - } - - public Optional getTrieLogLayer(final Hash blockHash) { - if (layeredWorldStatesByHash.containsKey(blockHash)) { - return Optional.of(layeredWorldStatesByHash.get(blockHash).getTrieLog()); - } else { - return worldStateStorage.getTrieLog(blockHash).map(TrieLogLayer::fromBytes); - } - } - @Override public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { - return layeredWorldStatesByHash.containsKey(blockHash) + return trieLogManager.getBonsaiLayeredWorldState(blockHash).isPresent() || persistedState.blockHash().equals(blockHash) || worldStateStorage.isWorldStateAvailable(rootHash, blockHash); } @@ -155,16 +102,21 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { public Optional getMutable( final Hash rootHash, final Hash blockHash, final boolean isPersistingState) { if (!isPersistingState) { - if (layeredWorldStatesByHash.containsKey(blockHash)) { - return Optional.of(layeredWorldStatesByHash.get(blockHash)); + final Optional layeredWorldState = + trieLogManager.getBonsaiLayeredWorldState(blockHash); + if (layeredWorldState.isPresent()) { + return layeredWorldState; } else { final BlockHeader header = blockchain.getBlockHeader(blockHash).get(); final BlockHeader currentHeader = blockchain.getChainHeadHeader(); - if ((currentHeader.getNumber() - header.getNumber()) >= maxLayersToLoad) { - LOG.warn("Exceeded the limit of back layers that can be loaded ({})", maxLayersToLoad); + if ((currentHeader.getNumber() - header.getNumber()) + >= trieLogManager.getMaxLayersToLoad()) { + LOG.warn( + "Exceeded the limit of back layers that can be loaded ({})", + trieLogManager.getMaxLayersToLoad()); return Optional.empty(); } - final Optional trieLogLayer = getTrieLogLayer(blockHash); + final Optional trieLogLayer = trieLogManager.getTrieLogLayer(blockHash); if (trieLogLayer.isPresent()) { return Optional.of( new BonsaiLayeredWorldState( @@ -195,7 +147,7 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { final List rollBacks = new ArrayList<>(); final List rollForwards = new ArrayList<>(); if (maybePersistedHeader.isEmpty()) { - getTrieLogLayer(persistedState.blockHash()).ifPresent(rollBacks::add); + trieLogManager.getTrieLogLayer(persistedState.blockHash()).ifPresent(rollBacks::add); } else { BlockHeader targetHeader = blockchain.getBlockHeader(blockHash).get(); BlockHeader persistedHeader = maybePersistedHeader.get(); @@ -203,7 +155,7 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { Hash persistedBlockHash = persistedHeader.getBlockHash(); while (persistedHeader.getNumber() > targetHeader.getNumber()) { LOG.debug("Rollback {}", persistedBlockHash); - rollBacks.add(getTrieLogLayer(persistedBlockHash).get()); + rollBacks.add(trieLogManager.getTrieLogLayer(persistedBlockHash).get()); persistedHeader = blockchain.getBlockHeader(persistedHeader.getParentHash()).get(); persistedBlockHash = persistedHeader.getBlockHash(); } @@ -211,7 +163,7 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { Hash targetBlockHash = targetHeader.getBlockHash(); while (persistedHeader.getNumber() < targetHeader.getNumber()) { LOG.debug("Rollforward {}", targetBlockHash); - rollForwards.add(getTrieLogLayer(targetBlockHash).get()); + rollForwards.add(trieLogManager.getTrieLogLayer(targetBlockHash).get()); targetHeader = blockchain.getBlockHeader(targetHeader.getParentHash()).get(); targetBlockHash = targetHeader.getBlockHash(); } @@ -220,10 +172,10 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { while (!persistedBlockHash.equals(targetBlockHash)) { LOG.debug("Paired Rollback {}", persistedBlockHash); LOG.debug("Paired Rollforward {}", targetBlockHash); - rollForwards.add(getTrieLogLayer(targetBlockHash).get()); + rollForwards.add(trieLogManager.getTrieLogLayer(targetBlockHash).get()); targetHeader = blockchain.getBlockHeader(targetHeader.getParentHash()).get(); - rollBacks.add(getTrieLogLayer(persistedBlockHash).get()); + rollBacks.add(trieLogManager.getTrieLogLayer(persistedBlockHash).get()); persistedHeader = blockchain.getBlockHeader(persistedHeader.getParentHash()).get(); targetBlockHash = targetHeader.getBlockHash(); @@ -269,6 +221,10 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { return persistedState; } + public TrieLogManager getTrieLogManager() { + return trieLogManager; + } + @Override public void setArchiveStateUnSafe(final BlockHeader blockHeader) { persistedState.setArchiveStateUnSafe(blockHeader); @@ -287,9 +243,4 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { // FIXME we can do proofs for layered tries and the persisted trie return Optional.empty(); } - - void scrubLayeredCache(final long newMaxHeight) { - final long waterline = newMaxHeight - RETAINED_LAYERS; - layeredWorldStatesByHash.entrySet().removeIf(entry -> entry.getValue().getHeight() < waterline); - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogManager.java new file mode 100644 index 0000000000..d8752e5e2f --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogManager.java @@ -0,0 +1,182 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.bonsai; + +import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TrieLogManager { + + private static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks + + private static final Logger LOG = LoggerFactory.getLogger(TrieLogManager.class); + + private final Blockchain blockchain; + private final BonsaiWorldStateKeyValueStorage worldStateStorage; + + private final Map layeredWorldStatesByHash; + private final long maxLayersToLoad; + + public TrieLogManager( + final Blockchain blockchain, + final BonsaiWorldStateKeyValueStorage worldStateStorage, + final long maxLayersToLoad, + final Map layeredWorldStatesByHash) { + this.blockchain = blockchain; + this.worldStateStorage = worldStateStorage; + this.layeredWorldStatesByHash = layeredWorldStatesByHash; + this.maxLayersToLoad = maxLayersToLoad; + } + + public TrieLogManager( + final Blockchain blockchain, + final BonsaiWorldStateKeyValueStorage worldStateStorage, + final long maxLayersToLoad) { + this(blockchain, worldStateStorage, maxLayersToLoad, new HashMap<>()); + } + + public TrieLogManager( + final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateStorage) { + this(blockchain, worldStateStorage, RETAINED_LAYERS, new HashMap<>()); + } + + public synchronized void saveTrieLog( + final BonsaiWorldStateArchive worldStateArchive, + final BonsaiWorldStateUpdater localUpdater, + final Hash worldStateRootHash, + final BlockHeader blockHeader) { + // do not overwrite a trielog layer that already exists in the database. + // if it's only in memory we need to save it + // for example, like that in case of reorg we don't replace a trielog layer + if (worldStateStorage.getTrieLog(blockHeader.getHash()).isEmpty()) { + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = worldStateStorage.updater(); + boolean success = false; + try { + final TrieLogLayer trieLog = + prepareTrieLog(blockHeader, worldStateRootHash, localUpdater, worldStateArchive); + persistTrieLog(blockHeader, worldStateRootHash, trieLog, stateUpdater); + success = true; + } finally { + if (success) { + stateUpdater.commit(); + } else { + stateUpdater.rollback(); + } + } + } + } + + public synchronized void addLayeredWorldState( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final TrieLogLayer trieLog, + final BonsaiWorldStateArchive worldStateArchive) { + + final BonsaiLayeredWorldState bonsaiLayeredWorldState = + new BonsaiLayeredWorldState( + blockchain, + worldStateArchive, + Optional.of((BonsaiPersistedWorldState) worldStateArchive.getMutable()), + blockHeader.getNumber(), + worldStateRootHash, + trieLog); + debugLambda( + LOG, + "adding layered world state for block {}, state root hash {}", + blockHeader::toLogString, + worldStateRootHash::toHexString); + layeredWorldStatesByHash.put(blockHeader.getHash(), bonsaiLayeredWorldState); + scrubLayeredCache(blockHeader.getNumber()); + } + + public synchronized void updateLayeredWorldState( + final Hash blockParentHash, final Hash blockHash) { + layeredWorldStatesByHash.computeIfPresent( + blockParentHash, + (parentHash, bonsaiLayeredWorldState) -> { + if (layeredWorldStatesByHash.containsKey(blockHash)) { + bonsaiLayeredWorldState.setNextWorldView( + Optional.of(layeredWorldStatesByHash.get(blockHash))); + } + return bonsaiLayeredWorldState; + }); + } + + public synchronized void scrubLayeredCache(final long newMaxHeight) { + final long waterline = newMaxHeight - RETAINED_LAYERS; + layeredWorldStatesByHash.entrySet().removeIf(entry -> entry.getValue().getHeight() < waterline); + } + + public long getMaxLayersToLoad() { + return maxLayersToLoad; + } + + public Optional getTrieLogLayer(final Hash blockHash) { + if (layeredWorldStatesByHash.containsKey(blockHash)) { + return Optional.of(layeredWorldStatesByHash.get(blockHash).getTrieLog()); + } else { + return worldStateStorage.getTrieLog(blockHash).map(TrieLogLayer::fromBytes); + } + } + + public Optional getBonsaiLayeredWorldState(final Hash blockHash) { + if (layeredWorldStatesByHash.containsKey(blockHash)) { + return Optional.of(layeredWorldStatesByHash.get(blockHash)); + } + return Optional.empty(); + } + + private TrieLogLayer prepareTrieLog( + final BlockHeader blockHeader, + final Hash currentWorldStateRootHash, + final BonsaiWorldStateUpdater localUpdater, + final BonsaiWorldStateArchive worldStateArchive) { + debugLambda(LOG, "Adding layered world state for {}", blockHeader::toLogString); + final TrieLogLayer trieLog = localUpdater.generateTrieLog(blockHeader.getBlockHash()); + trieLog.freeze(); + addLayeredWorldState(blockHeader, currentWorldStateRootHash, trieLog, worldStateArchive); + return trieLog; + } + + private void persistTrieLog( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final TrieLogLayer trieLog, + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { + debugLambda( + LOG, + "Persisting trie log for block hash {} and world state root {}", + blockHeader::toLogString, + worldStateRootHash::toHexString); + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + trieLog.writeTo(rlpLog); + stateUpdater + .getTrieLogStorageTransaction() + .put(blockHeader.getHash().toArrayUnsafe(), rlpLog.encoded().toArrayUnsafe()); + } +} diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 8ce4a09e3c..18de012ed3 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -15,6 +15,8 @@ package org.hyperledger.besu.ethereum.core; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.TrieLogManager; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -63,7 +65,13 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider { public static BonsaiWorldStateArchive createBonsaiInMemoryWorldStateArchive( final Blockchain blockchain) { - return new BonsaiWorldStateArchive(new InMemoryKeyValueStorageProvider(), blockchain); + final InMemoryKeyValueStorageProvider inMemoryKeyValueStorageProvider = + new InMemoryKeyValueStorageProvider(); + return new BonsaiWorldStateArchive( + new TrieLogManager( + blockchain, new BonsaiWorldStateKeyValueStorage(inMemoryKeyValueStorageProvider)), + inMemoryKeyValueStorageProvider, + blockchain); } public static MutableWorldState createInMemoryWorldState() { diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java index 925b0e548c..7947df0155 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java @@ -80,7 +80,11 @@ public class BonsaiWorldStateArchiveTest { .thenReturn(Optional.of(chainHead.getStateRoot().toArrayUnsafe())); when(keyValueStorage.get(WORLD_BLOCK_HASH_KEY)) .thenReturn(Optional.of(chainHead.getHash().toArrayUnsafe())); - bonsaiWorldStateArchive = new BonsaiWorldStateArchive(storageProvider, blockchain, 1); + bonsaiWorldStateArchive = + new BonsaiWorldStateArchive( + new TrieLogManager(blockchain, new BonsaiWorldStateKeyValueStorage(storageProvider), 1), + storageProvider, + blockchain); assertThat(bonsaiWorldStateArchive.getMutable(null, chainHead.getHash(), true)) .containsInstanceOf(BonsaiPersistedWorldState.class); @@ -88,7 +92,12 @@ public class BonsaiWorldStateArchiveTest { @Test public void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { - bonsaiWorldStateArchive = new BonsaiWorldStateArchive(storageProvider, blockchain, 512); + bonsaiWorldStateArchive = + new BonsaiWorldStateArchive( + new TrieLogManager( + blockchain, new BonsaiWorldStateKeyValueStorage(storageProvider), 512), + storageProvider, + blockchain); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); @@ -98,7 +107,12 @@ public class BonsaiWorldStateArchiveTest { @Test public void testGetMutableWhenLoadLessThanLimitLayersBack() { - bonsaiWorldStateArchive = new BonsaiWorldStateArchive(storageProvider, blockchain, 512); + bonsaiWorldStateArchive = + new BonsaiWorldStateArchive( + new TrieLogManager( + blockchain, new BonsaiWorldStateKeyValueStorage(storageProvider), 512), + storageProvider, + blockchain); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(511).buildHeader(); @@ -123,7 +137,14 @@ public class BonsaiWorldStateArchiveTest { final Map layeredWorldStatesByHash = mock(HashMap.class); bonsaiWorldStateArchive = - new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash); + new BonsaiWorldStateArchive( + new TrieLogManager( + blockchain, + new BonsaiWorldStateKeyValueStorage(storageProvider), + 12, + layeredWorldStatesByHash), + storageProvider, + blockchain); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); @@ -143,7 +164,15 @@ public class BonsaiWorldStateArchiveTest { final Map layeredWorldStatesByHash = mock(HashMap.class); bonsaiWorldStateArchive = - spy(new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash)); + spy( + new BonsaiWorldStateArchive( + new TrieLogManager( + blockchain, + new BonsaiWorldStateKeyValueStorage(storageProvider), + 12, + layeredWorldStatesByHash), + storageProvider, + blockchain)); var updater = spy(bonsaiWorldStateArchive.getUpdater()); when(bonsaiWorldStateArchive.getUpdater()).thenReturn(updater); @@ -179,7 +208,15 @@ public class BonsaiWorldStateArchiveTest { .thenReturn(mock(BonsaiLayeredWorldState.class, Answers.RETURNS_MOCKS)); bonsaiWorldStateArchive = - spy(new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash)); + spy( + new BonsaiWorldStateArchive( + new TrieLogManager( + blockchain, + new BonsaiWorldStateKeyValueStorage(storageProvider), + 12, + layeredWorldStatesByHash), + storageProvider, + blockchain)); var updater = spy(bonsaiWorldStateArchive.getUpdater()); when(bonsaiWorldStateArchive.getUpdater()).thenReturn(updater); @@ -222,7 +259,15 @@ public class BonsaiWorldStateArchiveTest { .thenReturn(mock(BonsaiLayeredWorldState.class, Answers.RETURNS_MOCKS)); bonsaiWorldStateArchive = - spy(new BonsaiWorldStateArchive(storageProvider, blockchain, 12, layeredWorldStatesByHash)); + spy( + new BonsaiWorldStateArchive( + new TrieLogManager( + blockchain, + new BonsaiWorldStateKeyValueStorage(storageProvider), + 12, + layeredWorldStatesByHash), + storageProvider, + blockchain)); var updater = spy(bonsaiWorldStateArchive.getUpdater()); when(bonsaiWorldStateArchive.getUpdater()).thenReturn(updater); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java index bee1131855..f6f7ea5dec 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java @@ -109,7 +109,11 @@ public class LogRollingTests { @Before public void createStorage() { final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); - archive = new BonsaiWorldStateArchive(provider, blockchain); + archive = + new BonsaiWorldStateArchive( + new TrieLogManager(blockchain, new BonsaiWorldStateKeyValueStorage(provider)), + provider, + blockchain); accountStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); @@ -128,7 +132,11 @@ public class LogRollingTests { provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); final InMemoryKeyValueStorageProvider secondProvider = new InMemoryKeyValueStorageProvider(); - secondArchive = new BonsaiWorldStateArchive(secondProvider, blockchain); + secondArchive = + new BonsaiWorldStateArchive( + new TrieLogManager(blockchain, new BonsaiWorldStateKeyValueStorage(secondProvider)), + secondProvider, + blockchain); secondAccountStorage = (InMemoryKeyValueStorage) secondProvider.getStorageBySegmentIdentifier( diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java index a1cf2948d6..3c45cc16e7 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java @@ -38,7 +38,11 @@ public class RollingImport { new RollingFileReader((i, c) -> Path.of(String.format(arg[0] + "-%04d.rdat", i)), false); final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); - final BonsaiWorldStateArchive archive = new BonsaiWorldStateArchive(provider, null); + final BonsaiWorldStateArchive archive = + new BonsaiWorldStateArchive( + new TrieLogManager(null, new BonsaiWorldStateKeyValueStorage(provider)), + provider, + null); final InMemoryKeyValueStorage accountStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);