TrieLogFactory plugin support (#5440)

* TrieLogs in plugin data
* adds dagger-wired plugincontext and TrieLogService
* add getTrieLogByRange, TrieLogRangeTuple composition
---------

Signed-off-by: garyschulte <garyschulte@gmail.com>
pull/5465/head
garyschulte 2 years ago committed by GitHub
parent 3b4f5e13d6
commit c85841d308
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 10
      besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
  2. 3
      besu/src/main/java/org/hyperledger/besu/components/BesuCommandModule.java
  3. 12
      besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java
  4. 29
      besu/src/main/java/org/hyperledger/besu/components/BesuPluginContextModule.java
  5. 3
      besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
  6. 3
      besu/src/main/java/org/hyperledger/besu/services/BesuPluginContextImpl.java
  7. 2
      besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java
  8. 55
      datatypes/src/main/java/org/hyperledger/besu/datatypes/AccountValue.java
  9. 111
      datatypes/src/main/java/org/hyperledger/besu/datatypes/StorageSlotKey.java
  10. 1
      ethereum/core/build.gradle
  11. 17
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java
  12. 38
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java
  13. 21
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java
  14. 4
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java
  15. 91
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java
  16. 2
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java
  17. 6
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java
  18. 35
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java
  19. 20
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogAddedEvent.java
  20. 120
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogFactoryImpl.java
  21. 78
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayer.java
  22. 6
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java
  23. 6
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java
  24. 50
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java
  25. 1
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldView.java
  26. 62
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/StorageSlotKey.java
  27. 16
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/StateTrieAccountValue.java
  28. 3
      ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java
  29. 2
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java
  30. 3
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java
  31. 17
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java
  32. 2
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java
  33. 2
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java
  34. 8
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java
  35. 2
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java
  36. 52
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogFactoryTests.java
  37. 7
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayerTests.java
  38. 7
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java
  39. 2
      plugin-api/build.gradle
  40. 9
      plugin-api/src/main/java/org/hyperledger/besu/plugin/BesuContext.java
  41. 58
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TrieLogService.java
  42. 176
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLog.java
  43. 50
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogAccumulator.java
  44. 49
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogEvent.java
  45. 46
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogFactory.java
  46. 61
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogProvider.java

@ -1329,11 +1329,10 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
private BesuComponent besuComponent;
private final Supplier<ObservableMetricsSystem> metricsSystem =
Suppliers.memoize(
() -> {
return besuComponent == null
? MetricsSystemFactory.create(metricsConfiguration())
: besuComponent.getObservableMetricsSystem();
});
() ->
besuComponent == null || besuComponent.getObservableMetricsSystem() == null
? MetricsSystemFactory.create(metricsConfiguration())
: besuComponent.getObservableMetricsSystem());
private Vertx vertx;
private EnodeDnsConfiguration enodeDnsConfiguration;
private KeyValueStorageProvider keyValueStorageProvider;
@ -1412,6 +1411,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
final PrivacyPluginServiceImpl privacyPluginService,
final PkiBlockCreationConfigurationProvider pkiBlockCreationConfigProvider,
final RpcEndpointServiceImpl rpcEndpointServiceImpl) {
this.besuComponent = besuComponent;
this.logger = besuComponent.getBesuCommandLogger();
this.rlpBlockImporter = rlpBlockImporter;
this.rlpBlockExporterFactory = rlpBlockExporterFactory;

@ -23,7 +23,6 @@ import org.hyperledger.besu.chainimport.RlpBlockImporter;
import org.hyperledger.besu.cli.BesuCommand;
import org.hyperledger.besu.controller.BesuController;
import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
import org.hyperledger.besu.services.BesuPluginContextImpl;
import javax.inject.Named;
import javax.inject.Singleton;
@ -50,7 +49,7 @@ public class BesuCommandModule {
RlpBlockExporter::new,
new RunnerBuilder(),
new BesuController.Builder(),
new BesuPluginContextImpl(),
besuComponent.getBesuPluginContext(),
System.getenv());
besuCommand.toCommandLine();
return besuCommand;

@ -21,6 +21,7 @@ import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoaderModule;
import org.hyperledger.besu.metrics.MetricsSystemModule;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.services.BesuPluginContextImpl;
import javax.inject.Named;
import javax.inject.Singleton;
@ -34,7 +35,8 @@ import org.slf4j.Logger;
modules = {
BesuCommandModule.class,
MetricsSystemModule.class,
CachedMerkleTrieLoaderModule.class
CachedMerkleTrieLoaderModule.class,
BesuPluginContextModule.class
})
public interface BesuComponent {
@ -66,4 +68,12 @@ public interface BesuComponent {
*/
@Named("besuCommandLogger")
Logger getBesuCommandLogger();
/**
* Besu plugin context for doing plugin service discovery.
*
* @return BesuComponent
*/
@Named("besuPluginContext")
BesuPluginContextImpl getBesuPluginContext();
}

@ -13,16 +13,29 @@
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.ethereum.bonsai.trielog;
package org.hyperledger.besu.components;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.services.BesuPluginContextImpl;
/** Interface for serializing and deserializing {@link TrieLogLayer} objects. */
public interface TrieLogFactory<T extends TrieLogLayer> {
T create(BonsaiWorldStateUpdateAccumulator accumulator, final Hash blockHash);
import javax.inject.Named;
import javax.inject.Singleton;
T deserialize(final byte[] bytes);
import dagger.Module;
import dagger.Provides;
byte[] serialize(final T layer);
/** A dagger module that know how to create the BesuPluginContextImpl singleton. */
@Module
public class BesuPluginContextModule {
/**
* Creates a BesuPluginContextImpl, used for plugin service discovery.
*
* @return the BesuPluginContext
*/
@Provides
@Named("besuPluginContext")
@Singleton
public BesuPluginContextImpl provideBesuPluginContext() {
return new BesuPluginContextImpl();
}
}

@ -1022,7 +1022,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
blockchain,
Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
cachedMerkleTrieLoader,
metricsSystem);
metricsSystem,
besuComponent.getBesuPluginContext());
case FOREST:
default:

@ -83,6 +83,7 @@ public class BesuPluginContextImpl implements BesuContext, PluginVersionsProvide
* @param serviceType the service type
* @param service the service
*/
@Override
public <T extends BesuService> void addService(final Class<T> serviceType, final T service) {
checkArgument(serviceType.isInterface(), "Services must be Java interfaces.");
checkArgument(
@ -118,7 +119,7 @@ public class BesuPluginContextImpl implements BesuContext, PluginVersionsProvide
for (final BesuPlugin plugin : serviceLoader) {
try {
plugin.register(this);
LOG.debug("Registered plugin of type {}.", plugin.getClass().getName());
LOG.info("Registered plugin of type {}.", plugin.getClass().getName());
addPluginVersion(plugin);
} catch (final Exception e) {
LOG.error(

@ -237,6 +237,8 @@ public abstract class CommandTestAbstract {
when(mockControllerBuilder.lowerBoundPeers(anyInt())).thenReturn(mockControllerBuilder);
when(mockControllerBuilder.maxRemotelyInitiatedPeers(anyInt()))
.thenReturn(mockControllerBuilder);
when(mockControllerBuilder.besuComponent(any(BesuComponent.class)))
.thenReturn(mockControllerBuilder);
// doReturn used because of generic BesuController
doReturn(mockController).when(mockControllerBuilder).build();
lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);

@ -0,0 +1,55 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.datatypes;
import org.hyperledger.besu.ethereum.rlp.RLPOutput;
/** The values of an account in the world state trie. */
public interface AccountValue {
/**
* The nonce of the account.
*
* @return the nonce of the account.
*/
long getNonce();
/**
* The available balance of that account.
*
* @return the balance, in Wei, of the account.
*/
Wei getBalance();
/**
* The hash of the root of the storage trie associated with this account.
*
* @return the hash of the root node of the storage trie.
*/
Hash getStorageRoot();
/**
* The hash of the EVM bytecode associated with this account.
*
* @return the hash of the account code (which may be {@link Hash#EMPTY}).
*/
Hash getCodeHash();
/**
* Writes the account value to the provided {@link RLPOutput}.
*
* @param out the {@link RLPOutput} to write to.
*/
void writeTo(final RLPOutput out);
}

@ -0,0 +1,111 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.datatypes;
import java.util.Objects;
import java.util.Optional;
import org.apache.tuweni.units.bigints.UInt256;
import org.jetbrains.annotations.NotNull;
/**
* StorageSlotKey represents a key used for storage slots in Ethereum. It contains the hash of the
* slot key and an optional representation of the key itself.
*
* <p>The class provides methods for accessing the hash and key, as well as methods for equality,
* hashcode, and comparison.
*
* <p>StorageSlotKey is used to uniquely identify storage slots within the Ethereum state.
*/
public class StorageSlotKey implements Comparable<StorageSlotKey> {
private final Hash slotHash;
private final Optional<UInt256> slotKey;
/**
* Creates a StorageSlotKey.
*
* @param slotHash Hashed storage slot key.
* @param slotKey Optional UInt256 storage slot key.
*/
public StorageSlotKey(final Hash slotHash, final Optional<UInt256> slotKey) {
this.slotHash = slotHash;
this.slotKey = slotKey;
}
/**
* Creates a StorageSlotKey, hashing the slotKey.
*
* @param slotKey the UInt256 storage slot key.
*/
public StorageSlotKey(final UInt256 slotKey) {
this(Hash.hash(slotKey), Optional.of(slotKey));
}
/**
* Gets the hash representation of the storage slot key.
*
* @return the hash of the storage slot key.
*/
public Hash getSlotHash() {
return slotHash;
}
/**
* Gets the optional UInt256 representation of the storage slot key.
*
* @return an Optional containing the UInt256 storage slot key if present, otherwise an empty
* Optional.
*/
public Optional<UInt256> getSlotKey() {
return slotKey;
}
/**
* Indicates whether some other object is "equal to" this one. Two StorageSlotKey objects are
* considered equal if their slot hash values are equal.
*
* @param o the reference object with which to compare.
* @return true if this object is the same as the obj argument; false otherwise.
*/
@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
StorageSlotKey that = (StorageSlotKey) o;
return Objects.equals(slotHash, that.slotHash);
}
@Override
public int hashCode() {
return Objects.hash(slotHash.hashCode());
}
@Override
public String toString() {
return String.format(
"StorageSlotKey{slotHash=%s, slotKey=%s}",
slotHash, slotKey.map(UInt256::toString).orElse("null"));
}
@Override
public int compareTo(@NotNull final StorageSlotKey other) {
return this.slotHash.compareTo(other.slotHash);
}
}

@ -50,6 +50,7 @@ dependencies {
annotationProcessor 'com.google.dagger:dagger-compiler'
implementation 'io.opentelemetry:opentelemetry-api'
implementation 'io.vertx:vertx-core'
implementation 'net.java.dev.jna:jna'
implementation 'org.apache.commons:commons-lang3'
implementation 'org.apache.tuweni:tuweni-bytes'

@ -16,6 +16,7 @@
package org.hyperledger.besu.ethereum.bonsai;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.Wei;
@ -24,7 +25,7 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.rlp.RLPException;
import org.hyperledger.besu.ethereum.rlp.RLPInput;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.ethereum.rlp.RLPOutput;
import org.hyperledger.besu.evm.ModificationNotAllowedException;
import org.hyperledger.besu.evm.account.AccountStorageEntry;
import org.hyperledger.besu.evm.account.EvmAccount;
@ -40,7 +41,7 @@ import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt256;
public class BonsaiAccount implements MutableAccount, EvmAccount {
public class BonsaiAccount implements MutableAccount, EvmAccount, AccountValue {
private final BonsaiWorldView context;
private final boolean mutable;
@ -77,7 +78,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
public BonsaiAccount(
final BonsaiWorldView context,
final Address address,
final StateTrieAccountValue stateTrieAccount,
final AccountValue stateTrieAccount,
final boolean mutable) {
this(
context,
@ -224,6 +225,12 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
public Bytes serializeAccount() {
final BytesValueRLPOutput out = new BytesValueRLPOutput();
writeTo(out);
return out.encoded();
}
@Override
public void writeTo(final RLPOutput out) {
out.startList();
out.writeLongScalar(nonce);
@ -232,7 +239,6 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
out.writeBytes(codeHash);
out.endList();
return out.encoded();
}
@Override
@ -262,6 +268,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
}
}
@Override
public Hash getStorageRoot() {
return storageRoot;
}
@ -298,7 +305,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
* @throws IllegalStateException if the stored values differ
*/
public static void assertCloseEnoughForDiffing(
final BonsaiAccount source, final StateTrieAccountValue account, final String context) {
final BonsaiAccount source, final AccountValue account, final String context) {
if (source == null) {
throw new IllegalStateException(context + ": source is null but target isn't");
} else {

@ -16,15 +16,12 @@
package org.hyperledger.besu.ethereum.bonsai;
import org.hyperledger.besu.ethereum.rlp.RLPOutput;
import java.util.Objects;
import java.util.function.BiConsumer;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
public class BonsaiValue<T> {
public class BonsaiValue<T> implements TrieLog.LogTuple<T> {
private T prior;
private T updated;
private boolean cleared;
@ -41,10 +38,12 @@ public class BonsaiValue<T> {
this.cleared = cleared;
}
@Override
public T getPrior() {
return prior;
}
@Override
public T getUpdated() {
return updated;
}
@ -60,38 +59,11 @@ public class BonsaiValue<T> {
return this;
}
public void writeRlp(final RLPOutput output, final BiConsumer<RLPOutput, T> writer) {
output.startList();
writeInnerRlp(output, writer);
output.endList();
}
public void writeInnerRlp(final RLPOutput output, final BiConsumer<RLPOutput, T> writer) {
if (prior == null) {
output.writeNull();
} else {
writer.accept(output, prior);
}
if (updated == null) {
output.writeNull();
} else {
writer.accept(output, updated);
}
if (!cleared) {
output.writeNull();
} else {
output.writeInt(1);
}
}
public boolean isUnchanged() {
return Objects.equals(updated, prior);
}
public void setCleared() {
this.cleared = true;
}
@Override
public boolean isCleared() {
return cleared;
}

@ -23,7 +23,6 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
@ -40,6 +39,8 @@ import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.evm.worldstate.WorldState;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.BesuContext;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import java.util.ArrayList;
import java.util.HashSet;
@ -70,14 +71,16 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
final StorageProvider provider,
final Blockchain blockchain,
final CachedMerkleTrieLoader cachedMerkleTrieLoader,
final ObservableMetricsSystem metricsSystem) {
final ObservableMetricsSystem metricsSystem,
final BesuContext pluginContext) {
this(
(BonsaiWorldStateKeyValueStorage)
provider.createWorldStateStorage(DataStorageFormat.BONSAI),
blockchain,
Optional.empty(),
cachedMerkleTrieLoader,
metricsSystem);
metricsSystem,
pluginContext);
}
public BonsaiWorldStateProvider(
@ -85,7 +88,8 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
final Blockchain blockchain,
final Optional<Long> maxLayersToLoad,
final CachedMerkleTrieLoader cachedMerkleTrieLoader,
final ObservableMetricsSystem metricsSystem) {
final ObservableMetricsSystem metricsSystem,
final BesuContext pluginContext) {
// TODO: de-dup constructors
this.trieLogManager =
@ -94,7 +98,8 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
blockchain,
worldStateStorage,
metricsSystem,
maxLayersToLoad.orElse(RETAINED_LAYERS));
maxLayersToLoad.orElse(RETAINED_LAYERS),
pluginContext);
this.blockchain = blockchain;
this.worldStateStorage = worldStateStorage;
this.persistedState = new BonsaiWorldState(this, worldStateStorage);
@ -189,8 +194,8 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
final Optional<BlockHeader> maybePersistedHeader =
blockchain.getBlockHeader(mutableState.blockHash()).map(BlockHeader.class::cast);
final List<TrieLogLayer> rollBacks = new ArrayList<>();
final List<TrieLogLayer> rollForwards = new ArrayList<>();
final List<TrieLog> rollBacks = new ArrayList<>();
final List<TrieLog> rollForwards = new ArrayList<>();
if (maybePersistedHeader.isEmpty()) {
trieLogManager.getTrieLogLayer(mutableState.blockHash()).ifPresent(rollBacks::add);
} else {
@ -232,7 +237,7 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
final BonsaiWorldStateUpdateAccumulator bonsaiUpdater =
(BonsaiWorldStateUpdateAccumulator) mutableState.updater();
try {
for (final TrieLogLayer rollBack : rollBacks) {
for (final TrieLog rollBack : rollBacks) {
LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash());
bonsaiUpdater.rollBack(rollBack);
}

@ -17,9 +17,9 @@ package org.hyperledger.besu.ethereum.bonsai.cache;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.trie.MerkleTrie;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie;
@ -122,7 +122,7 @@ public class CachedMerkleTrieLoader implements BonsaiStorageSubscriber {
Hash.hash(storageRoot),
Function.identity(),
Function.identity());
storageTrie.get(slotKey.slotHash());
storageTrie.get(slotKey.getSlotHash());
} catch (MerkleTrieException e) {
// ignore exception for the cache
}

@ -21,10 +21,16 @@ import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStor
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage;
import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.BesuContext;
import org.hyperledger.besu.plugin.services.TrieLogService;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogProvider;
import java.util.ArrayList;
import java.util.Comparator;
@ -33,7 +39,11 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
import java.util.stream.Stream;
import com.google.common.annotations.VisibleForTesting;
import org.apache.tuweni.bytes.Bytes32;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -50,8 +60,9 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
final BonsaiWorldStateKeyValueStorage worldStateStorage,
final long maxLayersToLoad,
final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash,
final BesuContext pluginContext,
final ObservableMetricsSystem metricsSystem) {
super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash);
super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash, pluginContext);
worldStateStorage.subscribe(this);
this.archive = archive;
this.metricsSystem = metricsSystem;
@ -62,13 +73,15 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
final Blockchain blockchain,
final BonsaiWorldStateKeyValueStorage worldStateStorage,
final ObservableMetricsSystem metricsSystem,
final long maxLayersToLoad) {
final long maxLayersToLoad,
final BesuContext pluginContext) {
this(
archive,
blockchain,
worldStateStorage,
maxLayersToLoad,
new ConcurrentHashMap<>(),
pluginContext,
metricsSystem);
}
@ -198,4 +211,78 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
public void onCloseStorage() {
this.cachedWorldStatesByHash.clear();
}
@VisibleForTesting
@Override
protected TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext) {
// if we have a TrieLogService from pluginContext, use it.
var trieLogServicez =
Optional.ofNullable(pluginContext)
.flatMap(context -> context.getService(TrieLogService.class));
if (trieLogServicez.isPresent()) {
var trieLogService = trieLogServicez.get();
// push the TrieLogProvider into the TrieLogService
trieLogService.configureTrieLogProvider(getTrieLogProvider());
// configure plugin observers:
trieLogService.getObservers().forEach(trieLogObservers::subscribe);
// return the TrieLogFactory implementation from the TrieLogService
return trieLogService.getTrieLogFactory();
} else {
// Otherwise default to TrieLogFactoryImpl
return new TrieLogFactoryImpl();
}
}
@VisibleForTesting
TrieLogProvider getTrieLogProvider() {
return new TrieLogProvider() {
@Override
public <T extends TrieLog.LogTuple<?>> Optional<TrieLog> getTrieLogLayer(
final Hash blockHash) {
return CachedWorldStorageManager.this.getTrieLogLayer(blockHash);
}
@Override
public <T extends TrieLog.LogTuple<?>> Optional<TrieLog> getTrieLogLayer(
final long blockNumber) {
return CachedWorldStorageManager.this
.blockchain
.getBlockHeader(blockNumber)
.map(BlockHeader::getHash)
.flatMap(CachedWorldStorageManager.this::getTrieLogLayer);
}
@Override
public <T extends TrieLog.LogTuple<?>> List<TrieLogRangeTuple> getTrieLogsByRange(
final long fromBlockNumber, final long toBlockNumber) {
return rangeAsStream(fromBlockNumber, toBlockNumber)
.map(blockchain::getBlockHeader)
.map(
headerOpt ->
headerOpt.flatMap(
header ->
CachedWorldStorageManager.this
.getTrieLogLayer(header.getBlockHash())
.map(
layer ->
new TrieLogRangeTuple(
header.getBlockHash(), header.getNumber(), layer))))
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toList());
}
Stream<Long> rangeAsStream(final long fromBlockNumber, final long toBlockNumber) {
if (Math.abs(toBlockNumber - fromBlockNumber) > LOG_RANGE_LIMIT) {
throw new IllegalArgumentException("Requested Range too large");
}
long left = Math.min(fromBlockNumber, toBlockNumber);
long right = Math.max(fromBlockNumber, toBlockNumber);
return LongStream.range(left, right).boxed();
}
};
}
}

@ -16,8 +16,8 @@
package org.hyperledger.besu.ethereum.bonsai.storage;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.services.exception.StorageException;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;

@ -15,7 +15,7 @@
package org.hyperledger.besu.ethereum.bonsai.storage;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.trie.MerkleTrie;
@ -254,7 +254,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
getStorageValueCounter.inc();
Optional<Bytes> response =
storageStorage
.get(Bytes.concatenate(accountHash, storageSlotKey.slotHash()).toArrayUnsafe())
.get(Bytes.concatenate(accountHash, storageSlotKey.getSlotHash()).toArrayUnsafe())
.map(Bytes::wrap);
if (response.isEmpty()) {
final Optional<Hash> storageRoot = storageRootSupplier.get();
@ -267,7 +267,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
Function.identity(),
Function.identity()),
storageRoot.get())
.get(storageSlotKey.slotHash())
.get(storageSlotKey.getSlotHash())
.map(bytes -> Bytes32.leftPad(RLP.decodeValue(bytes)));
if (response.isEmpty()) getStorageValueMissingMerkleTrieCounter.inc();
else getStorageValueMerkleTrieCounter.inc();

@ -19,11 +19,14 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedBonsaiWorldView;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiUpdater;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent.TrieLogAddedObserver;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.plugin.BesuContext;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent.TrieLogObserver;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import org.hyperledger.besu.util.Subscribers;
import java.util.Map;
@ -37,27 +40,31 @@ import org.slf4j.LoggerFactory;
public abstract class AbstractTrieLogManager implements TrieLogManager {
private static final Logger LOG = LoggerFactory.getLogger(AbstractTrieLogManager.class);
public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks
public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs
protected final Blockchain blockchain;
protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage;
protected final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash;
protected final long maxLayersToLoad;
private final Subscribers<TrieLogAddedObserver> trieLogAddedObservers = Subscribers.create();
protected final Subscribers<TrieLogObserver> trieLogObservers = Subscribers.create();
// TODO plumb factory from plugin service:
TrieLogFactory<TrieLogLayer> trieLogFactory = new TrieLogFactoryImpl();
protected final TrieLogFactory trieLogFactory;
protected AbstractTrieLogManager(
final Blockchain blockchain,
final BonsaiWorldStateKeyValueStorage worldStateStorage,
final long maxLayersToLoad,
final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash) {
final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash,
final BesuContext pluginContext) {
this.blockchain = blockchain;
this.rootWorldStateStorage = worldStateStorage;
this.cachedWorldStatesByHash = cachedWorldStatesByHash;
this.maxLayersToLoad = maxLayersToLoad;
this.trieLogFactory = setupTrieLogFactory(pluginContext);
}
protected abstract TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext);
@Override
public synchronized void saveTrieLog(
final BonsaiWorldStateUpdateAccumulator localUpdater,
@ -71,11 +78,11 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
final BonsaiUpdater stateUpdater = forWorldState.getWorldStateStorage().updater();
boolean success = false;
try {
final TrieLogLayer trieLog = prepareTrieLog(forBlockHeader, localUpdater);
final TrieLog trieLog = prepareTrieLog(forBlockHeader, localUpdater);
persistTrieLog(forBlockHeader, forWorldStateRootHash, trieLog, stateUpdater);
// notify trie log added observers, synchronously
trieLogAddedObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog)));
trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog)));
success = true;
} finally {
@ -89,13 +96,13 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
}
@VisibleForTesting
TrieLogLayer prepareTrieLog(
TrieLog prepareTrieLog(
final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) {
LOG.atDebug()
.setMessage("Adding layered world state for {}")
.addArgument(blockHeader::toLogString)
.log();
final TrieLogLayer trieLog = localUpdater.generateTrieLog(blockHeader.getBlockHash());
final TrieLog trieLog = localUpdater.generateTrieLog(blockHeader);
trieLog.freeze();
return trieLog;
}
@ -117,7 +124,7 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
private void persistTrieLog(
final BlockHeader blockHeader,
final Hash worldStateRootHash,
final TrieLogLayer trieLog,
final TrieLog trieLog,
final BonsaiUpdater stateUpdater) {
LOG.atDebug()
.setMessage("Persisting trie log for block hash {} and world state root {}")
@ -141,17 +148,17 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
}
@Override
public Optional<TrieLogLayer> getTrieLogLayer(final Hash blockHash) {
public Optional<TrieLog> getTrieLogLayer(final Hash blockHash) {
return rootWorldStateStorage.getTrieLog(blockHash).map(trieLogFactory::deserialize);
}
@Override
public synchronized long subscribe(final TrieLogAddedObserver sub) {
return trieLogAddedObservers.subscribe(sub);
public synchronized long subscribe(final TrieLogObserver sub) {
return trieLogObservers.subscribe(sub);
}
@Override
public synchronized void unsubscribe(final long id) {
trieLogAddedObservers.unsubscribe(id);
trieLogObservers.unsubscribe(id);
}
}

@ -15,21 +15,13 @@
*/
package org.hyperledger.besu.ethereum.bonsai.trielog;
public class TrieLogAddedEvent {
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent;
private final TrieLogLayer layer;
public record TrieLogAddedEvent(TrieLog layer) implements TrieLogEvent {
public TrieLogAddedEvent(final TrieLogLayer layer) {
this.layer = layer;
}
public TrieLogLayer getLayer() {
return layer;
}
@FunctionalInterface
interface TrieLogAddedObserver {
void onTrieLogAdded(TrieLogAddedEvent event);
@Override
public Type getType() {
return Type.ADDED;
}
}

@ -15,80 +15,61 @@
*/
package org.hyperledger.besu.ethereum.bonsai.trielog;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.rlp.RLPInput;
import org.hyperledger.besu.ethereum.rlp.RLPOutput;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.plugin.data.BlockHeader;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogAccumulator;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.Function;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256;
public class TrieLogFactoryImpl implements TrieLogFactory<TrieLogLayer> {
public class TrieLogFactoryImpl implements TrieLogFactory {
@Override
public TrieLogLayer create(
final BonsaiWorldStateUpdateAccumulator accumulator, final Hash blockHash) {
public TrieLogLayer create(final TrieLogAccumulator accumulator, final BlockHeader blockHeader) {
TrieLogLayer layer = new TrieLogLayer();
layer.setBlockHash(blockHash);
for (final Map.Entry<Address, BonsaiValue<BonsaiAccount>> updatedAccount :
accumulator.getAccountsToUpdate().entrySet()) {
final BonsaiValue<BonsaiAccount> bonsaiValue = updatedAccount.getValue();
final BonsaiAccount oldValue = bonsaiValue.getPrior();
final StateTrieAccountValue oldAccount =
oldValue == null
? null
: new StateTrieAccountValue(
oldValue.getNonce(),
oldValue.getBalance(),
oldValue.getStorageRoot(),
oldValue.getCodeHash());
final BonsaiAccount newValue = bonsaiValue.getUpdated();
final StateTrieAccountValue newAccount =
newValue == null
? null
: new StateTrieAccountValue(
newValue.getNonce(),
newValue.getBalance(),
newValue.getStorageRoot(),
newValue.getCodeHash());
if (oldValue == null && newValue == null) {
layer.setBlockHash(blockHeader.getBlockHash());
layer.setBlockNumber(blockHeader.getNumber());
for (final var updatedAccount : accumulator.getAccountsToUpdate().entrySet()) {
final var bonsaiValue = updatedAccount.getValue();
final var oldAccountValue = bonsaiValue.getPrior();
final var newAccountValue = bonsaiValue.getUpdated();
if (oldAccountValue == null && newAccountValue == null) {
// by default do not persist empty reads of accounts to the trie log
continue;
}
layer.addAccountChange(updatedAccount.getKey(), oldAccount, newAccount);
layer.addAccountChange(updatedAccount.getKey(), oldAccountValue, newAccountValue);
}
for (final Map.Entry<Address, BonsaiValue<Bytes>> updatedCode :
accumulator.getCodeToUpdate().entrySet()) {
for (final var updatedCode : accumulator.getCodeToUpdate().entrySet()) {
layer.addCodeChange(
updatedCode.getKey(),
updatedCode.getValue().getPrior(),
updatedCode.getValue().getUpdated(),
blockHash);
blockHeader.getBlockHash());
}
for (final Map.Entry<
Address,
BonsaiWorldStateUpdateAccumulator.StorageConsumingMap<
StorageSlotKey, BonsaiValue<UInt256>>>
updatesStorage : accumulator.getStorageToUpdate().entrySet()) {
for (final var updatesStorage : accumulator.getStorageToUpdate().entrySet()) {
final Address address = updatesStorage.getKey();
for (final Map.Entry<StorageSlotKey, BonsaiValue<UInt256>> slotUpdate :
updatesStorage.getValue().entrySet()) {
for (final var slotUpdate : updatesStorage.getValue().entrySet()) {
var val = slotUpdate.getValue();
if (val.getPrior() == null && val.getUpdated() == null) {
@ -103,52 +84,53 @@ public class TrieLogFactoryImpl implements TrieLogFactory<TrieLogLayer> {
}
@Override
public byte[] serialize(final TrieLogLayer layer) {
public byte[] serialize(final TrieLog layer) {
final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput();
writeTo(layer, rlpLog);
return rlpLog.encoded().toArrayUnsafe();
}
public static void writeTo(final TrieLogLayer layer, final RLPOutput output) {
public static void writeTo(final TrieLog layer, final RLPOutput output) {
layer.freeze();
final Set<Address> addresses = new TreeSet<>();
addresses.addAll(layer.getAccounts().keySet());
addresses.addAll(layer.getCode().keySet());
addresses.addAll(layer.getStorage().keySet());
addresses.addAll(layer.getAccountChanges().keySet());
addresses.addAll(layer.getCodeChanges().keySet());
addresses.addAll(layer.getStorageChanges().keySet());
output.startList(); // container
output.writeBytes(layer.blockHash);
output.writeBytes(layer.getBlockHash());
for (final Address address : addresses) {
output.startList(); // this change
output.writeBytes(address);
final BonsaiValue<StateTrieAccountValue> accountChange = layer.accounts.get(address);
final TrieLog.LogTuple<AccountValue> accountChange = layer.getAccountChanges().get(address);
if (accountChange == null || accountChange.isUnchanged()) {
output.writeNull();
} else {
accountChange.writeRlp(output, (o, sta) -> sta.writeTo(o));
writeRlp(accountChange, output, (o, sta) -> sta.writeTo(o));
}
final BonsaiValue<Bytes> codeChange = layer.code.get(address);
final TrieLog.LogTuple<Bytes> codeChange = layer.getCodeChanges().get(address);
if (codeChange == null || codeChange.isUnchanged()) {
output.writeNull();
} else {
codeChange.writeRlp(output, RLPOutput::writeBytes);
writeRlp(codeChange, output, RLPOutput::writeBytes);
}
final Map<StorageSlotKey, BonsaiValue<UInt256>> storageChanges = layer.storage.get(address);
final Map<StorageSlotKey, TrieLog.LogTuple<UInt256>> storageChanges =
layer.getStorageChanges().get(address);
if (storageChanges == null) {
output.writeNull();
} else {
output.startList();
for (final Map.Entry<StorageSlotKey, BonsaiValue<UInt256>> storageChangeEntry :
for (final Map.Entry<StorageSlotKey, TrieLog.LogTuple<UInt256>> storageChangeEntry :
storageChanges.entrySet()) {
output.startList();
// do not write slotKey, it is not used in mainnet bonsai trielogs
output.writeBytes(storageChangeEntry.getKey().slotHash());
storageChangeEntry.getValue().writeInnerRlp(output, RLPOutput::writeUInt256Scalar);
output.writeBytes(storageChangeEntry.getKey().getSlotHash());
writeInnerRlp(storageChangeEntry.getValue(), output, RLPOutput::writeUInt256Scalar);
output.endList();
}
output.endList();
@ -242,4 +224,34 @@ public class TrieLogFactoryImpl implements TrieLogFactory<TrieLogLayer> {
.filter(i -> i == 1)
.isPresent();
}
public static <T> void writeRlp(
final TrieLog.LogTuple<T> value,
final RLPOutput output,
final BiConsumer<RLPOutput, T> writer) {
output.startList();
writeInnerRlp(value, output, writer);
output.endList();
}
public static <T> void writeInnerRlp(
final TrieLog.LogTuple<T> value,
final RLPOutput output,
final BiConsumer<RLPOutput, T> writer) {
if (value.getPrior() == null) {
output.writeNull();
} else {
writer.accept(output, value.getPrior());
}
if (value.getUpdated() == null) {
output.writeNull();
} else {
writer.accept(output, value.getUpdated());
}
if (!value.isCleared()) {
output.writeNull();
} else {
output.writeInt(1);
}
}
}

@ -18,18 +18,18 @@ package org.hyperledger.besu.ethereum.bonsai.trielog;
import static com.google.common.base.Preconditions.checkState;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.TreeMap;
import java.util.stream.Stream;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
@ -40,14 +40,16 @@ import org.apache.tuweni.units.bigints.UInt256;
* This class encapsulates the changes that are done to transition one block to the next. This
* includes serialization and deserialization tasks for storing this log to off-memory storage.
*
* <p>In this particular formulation only the "Leaves" are tracked" Future layers may track patrica
* <p>In this particular formulation only the "Leaves" are tracked Future layers may track patricia
* trie changes as well.
*/
public class TrieLogLayer {
@SuppressWarnings("unchecked")
public class TrieLogLayer implements TrieLog {
protected Hash blockHash;
protected Optional<Long> blockNumber = Optional.empty();
Map<Address, BonsaiValue<StateTrieAccountValue>> getAccounts() {
Map<Address, BonsaiValue<AccountValue>> getAccounts() {
return accounts;
}
@ -59,7 +61,7 @@ public class TrieLogLayer {
return storage;
}
protected final Map<Address, BonsaiValue<StateTrieAccountValue>> accounts;
protected final Map<Address, BonsaiValue<AccountValue>> accounts;
protected final Map<Address, BonsaiValue<Bytes>> code;
protected final Map<Address, Map<StorageSlotKey, BonsaiValue<UInt256>>> storage;
protected boolean frozen = false;
@ -72,10 +74,12 @@ public class TrieLogLayer {
}
/** Locks the layer so no new changes can be added; */
void freeze() {
frozen = true; // The code never bothered me anyway
@Override
public void freeze() {
frozen = true; // The code never bothered me anyway 🥶
}
@Override
public Hash getBlockHash() {
return blockHash;
}
@ -86,10 +90,19 @@ public class TrieLogLayer {
return this;
}
@Override
public Optional<Long> getBlockNumber() {
return blockNumber;
}
public TrieLogLayer setBlockNumber(final long blockNumber) {
checkState(!frozen, "Layer is Frozen");
this.blockNumber = Optional.of(blockNumber);
return this;
}
public TrieLogLayer addAccountChange(
final Address address,
final StateTrieAccountValue oldValue,
final StateTrieAccountValue newValue) {
final Address address, final AccountValue oldValue, final AccountValue newValue) {
checkState(!frozen, "Layer is Frozen");
accounts.put(address, new BonsaiValue<>(oldValue, newValue));
return this;
@ -114,64 +127,71 @@ public class TrieLogLayer {
return this;
}
public Stream<Map.Entry<Address, BonsaiValue<StateTrieAccountValue>>> streamAccountChanges() {
return accounts.entrySet().stream();
@Override
public Map<Address, BonsaiValue<AccountValue>> getAccountChanges() {
return accounts;
}
public Stream<Map.Entry<Address, BonsaiValue<Bytes>>> streamCodeChanges() {
return code.entrySet().stream();
@Override
public Map<Address, BonsaiValue<Bytes>> getCodeChanges() {
return code;
}
public Stream<Map.Entry<Address, Map<StorageSlotKey, BonsaiValue<UInt256>>>>
streamStorageChanges() {
return storage.entrySet().stream();
@Override
public Map<Address, Map<StorageSlotKey, BonsaiValue<UInt256>>> getStorageChanges() {
return storage;
}
public boolean hasStorageChanges(final Address address) {
return storage.containsKey(address);
}
public Stream<Map.Entry<StorageSlotKey, BonsaiValue<UInt256>>> streamStorageChanges(
final Address address) {
return storage.getOrDefault(address, Map.of()).entrySet().stream();
@Override
public Map<StorageSlotKey, BonsaiValue<UInt256>> getStorageChanges(final Address address) {
return storage.getOrDefault(address, Map.of());
}
@Override
public Optional<Bytes> getPriorCode(final Address address) {
return Optional.ofNullable(code.get(address)).map(BonsaiValue::getPrior);
}
@Override
public Optional<Bytes> getCode(final Address address) {
return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated);
}
Optional<UInt256> getPriorStorageByStorageSlotKey(
@Override
public Optional<UInt256> getPriorStorageByStorageSlotKey(
final Address address, final StorageSlotKey storageSlotKey) {
return Optional.ofNullable(storage.get(address))
.map(i -> i.get(storageSlotKey))
.map(BonsaiValue::getPrior);
}
Optional<UInt256> getStorageByStorageSlotKey(
@Override
public Optional<UInt256> getStorageByStorageSlotKey(
final Address address, final StorageSlotKey storageSlotKey) {
return Optional.ofNullable(storage.get(address))
.map(i -> i.get(storageSlotKey))
.map(BonsaiValue::getUpdated);
}
public Optional<StateTrieAccountValue> getPriorAccount(final Address address) {
@Override
public Optional<AccountValue> getPriorAccount(final Address address) {
return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getPrior);
}
public Optional<StateTrieAccountValue> getAccount(final Address address) {
@Override
public Optional<AccountValue> getAccount(final Address address) {
return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated);
}
public String dump() {
final StringBuilder sb = new StringBuilder();
sb.append("TrieLogLayer{" + "blockHash=").append(blockHash).append(frozen).append('}');
sb.append("TrieLog{" + "blockHash=").append(blockHash).append(frozen).append('}');
sb.append("accounts\n");
for (final Map.Entry<Address, BonsaiValue<StateTrieAccountValue>> account :
accounts.entrySet()) {
for (final Map.Entry<Address, BonsaiValue<AccountValue>> account : accounts.entrySet()) {
sb.append(" : ").append(account.getKey()).append("\n");
if (Objects.equals(account.getValue().getPrior(), account.getValue().getUpdated())) {
sb.append(" = ").append(account.getValue().getUpdated()).append("\n");

@ -19,6 +19,8 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent;
import java.util.Optional;
import java.util.function.Function;
@ -47,9 +49,9 @@ public interface TrieLogManager {
void reset();
Optional<TrieLogLayer> getTrieLogLayer(final Hash blockHash);
Optional<? extends TrieLog> getTrieLogLayer(final Hash blockHash);
long subscribe(final TrieLogAddedEvent.TrieLogAddedObserver sub);
long subscribe(final TrieLogEvent.TrieLogObserver sub);
void unsubscribe(final long id);
}

@ -22,6 +22,7 @@ import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyVa
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider;
@ -261,7 +262,7 @@ public class BonsaiWorldState
// for manicured tries and composting, collect branches here (not implemented)
for (final Map.Entry<StorageSlotKey, BonsaiValue<UInt256>> storageUpdate :
storageAccountUpdate.getValue().entrySet()) {
final Hash slotHash = storageUpdate.getKey().slotHash();
final Hash slotHash = storageUpdate.getKey().getSlotHash();
final UInt256 updatedStorage = storageUpdate.getValue().getUpdated();
try {
if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
@ -503,7 +504,8 @@ public class BonsaiWorldState
@Override
public UInt256 getStorageValue(final Address address, final UInt256 storageKey) {
return Optional.ofNullable(getStorageValue(address, storageKey)).orElse(UInt256.ZERO);
return getStorageValueByStorageSlotKey(address, new StorageSlotKey(storageKey))
.orElse(UInt256.ZERO);
}
@Override

@ -16,23 +16,26 @@
package org.hyperledger.besu.ethereum.bonsai.worldview;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactory;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.evm.account.Account;
import org.hyperledger.besu.evm.account.EvmAccount;
import org.hyperledger.besu.evm.worldstate.AbstractWorldUpdater;
import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount;
import org.hyperledger.besu.evm.worldstate.WrappedEvmAccount;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogAccumulator;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import java.util.Collection;
import java.util.Collections;
@ -55,7 +58,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BonsaiWorldStateUpdateAccumulator
extends AbstractWorldUpdater<BonsaiWorldView, BonsaiAccount> implements BonsaiWorldView {
extends AbstractWorldUpdater<BonsaiWorldView, BonsaiAccount>
implements BonsaiWorldView, TrieLogAccumulator {
private static final Logger LOG =
LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class);
private final Consumer<BonsaiValue<BonsaiAccount>> accountPreloader;
@ -72,7 +76,7 @@ public class BonsaiWorldStateUpdateAccumulator
storageToUpdate = new ConcurrentHashMap<>();
// todo plumb me from plugin service:
TrieLogFactory<TrieLogLayer> trieLogFactory = new TrieLogFactoryImpl();
TrieLogFactory trieLogFactory = new TrieLogFactoryImpl();
private boolean isAccumulatorStateChanged;
@ -146,10 +150,12 @@ public class BonsaiWorldStateUpdateAccumulator
return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(newAccount)));
}
@Override
public Map<Address, BonsaiValue<BonsaiAccount>> getAccountsToUpdate() {
return accountsToUpdate;
}
@Override
public Map<Address, BonsaiValue<Bytes>> getCodeToUpdate() {
return codeToUpdate;
}
@ -158,6 +164,7 @@ public class BonsaiWorldStateUpdateAccumulator
return storageToClear;
}
@Override
public Map<Address, StorageConsumingMap<StorageSlotKey, BonsaiValue<UInt256>>>
getStorageToUpdate() {
return storageToUpdate;
@ -477,7 +484,8 @@ public class BonsaiWorldStateUpdateAccumulator
storageToUpdate.get(address);
if (bonsaiValueStorage != null) {
// hash the key to match the implied storage interface of hashed slotKey
bonsaiValueStorage.forEach((key, value) -> results.put(key.slotHash(), value.getUpdated()));
bonsaiValueStorage.forEach(
(key, value) -> results.put(key.getSlotHash(), value.getUpdated()));
}
return results;
}
@ -492,25 +500,22 @@ public class BonsaiWorldStateUpdateAccumulator
return wrappedWorldView().getWorldStateStorage();
}
public TrieLogLayer generateTrieLog(final Hash blockHash) {
return trieLogFactory.create(this, blockHash);
public TrieLog generateTrieLog(final BlockHeader blockHeader) {
return trieLogFactory.create(this, blockHeader);
}
public void rollForward(final TrieLogLayer layer) {
layer
.streamAccountChanges()
public void rollForward(final TrieLog layer) {
layer.getAccountChanges().entrySet().stream()
.forEach(
entry ->
rollAccountChange(
entry.getKey(), entry.getValue().getPrior(), entry.getValue().getUpdated()));
layer
.streamCodeChanges()
layer.getCodeChanges().entrySet().stream()
.forEach(
entry ->
rollCodeChange(
entry.getKey(), entry.getValue().getPrior(), entry.getValue().getUpdated()));
layer
.streamStorageChanges()
layer.getStorageChanges().entrySet().stream()
.forEach(
entry ->
entry
@ -524,21 +529,18 @@ public class BonsaiWorldStateUpdateAccumulator
value.getUpdated())));
}
public void rollBack(final TrieLogLayer layer) {
layer
.streamAccountChanges()
public void rollBack(final TrieLog layer) {
layer.getAccountChanges().entrySet().stream()
.forEach(
entry ->
rollAccountChange(
entry.getKey(), entry.getValue().getUpdated(), entry.getValue().getPrior()));
layer
.streamCodeChanges()
layer.getCodeChanges().entrySet().stream()
.forEach(
entry ->
rollCodeChange(
entry.getKey(), entry.getValue().getUpdated(), entry.getValue().getPrior()));
layer
.streamStorageChanges()
layer.getStorageChanges().entrySet().stream()
.forEach(
entry ->
entry
@ -554,8 +556,8 @@ public class BonsaiWorldStateUpdateAccumulator
private void rollAccountChange(
final Address address,
final StateTrieAccountValue expectedValue,
final StateTrieAccountValue replacementValue) {
final AccountValue expectedValue,
final AccountValue replacementValue) {
if (Objects.equals(expectedValue, replacementValue)) {
// non-change, a cached read.
return;

@ -18,6 +18,7 @@ package org.hyperledger.besu.ethereum.bonsai.worldview;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.evm.worldstate.WorldUpdater;

@ -1,62 +0,0 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.ethereum.bonsai.worldview;
import org.hyperledger.besu.datatypes.Hash;
import java.util.Objects;
import java.util.Optional;
import org.apache.tuweni.units.bigints.UInt256;
import org.jetbrains.annotations.NotNull;
public record StorageSlotKey(Hash slotHash, Optional<UInt256> slotKey)
implements Comparable<org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey> {
public StorageSlotKey(final UInt256 slotKey) {
this(Hash.hash(slotKey), Optional.of(slotKey));
}
@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
StorageSlotKey that = (StorageSlotKey) o;
return Objects.equals(slotHash, that.slotHash);
}
@Override
public int hashCode() {
return Objects.hash(slotHash.hashCode());
}
@Override
public String toString() {
return String.format(
"StorageSlotKey{slotHash=%s, slotKey=%s}",
slotHash, slotKey.map(UInt256::toString).orElse("null"));
}
@Override
public int compareTo(
@NotNull final org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey other) {
return this.slotHash.compareTo(other.slotHash);
}
}

@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.worldstate;
import static com.google.common.base.Preconditions.checkNotNull;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.rlp.RLPInput;
@ -26,12 +27,12 @@ import java.util.Objects;
import org.apache.tuweni.bytes.Bytes32;
/** Represents the raw values associated with an account in the world state trie. */
public class StateTrieAccountValue {
public class StateTrieAccountValue implements AccountValue {
private final long nonce;
private final Wei balance;
private final Hash storageRoot;
private final Hash codeHash;
protected final long nonce;
protected final Wei balance;
protected final Hash storageRoot;
protected final Hash codeHash;
public StateTrieAccountValue(
final long nonce, final Wei balance, final Hash storageRoot, final Hash codeHash) {
@ -49,6 +50,7 @@ public class StateTrieAccountValue {
*
* @return the account nonce.
*/
@Override
public long getNonce() {
return nonce;
}
@ -58,6 +60,7 @@ public class StateTrieAccountValue {
*
* @return the balance, in Wei, of the account.
*/
@Override
public Wei getBalance() {
return balance;
}
@ -67,6 +70,7 @@ public class StateTrieAccountValue {
*
* @return the hash of the root node of the storage trie.
*/
@Override
public Hash getStorageRoot() {
return storageRoot;
}
@ -76,6 +80,7 @@ public class StateTrieAccountValue {
*
* @return the hash of the account code (which may be {@link Hash#EMPTY}).
*/
@Override
public Hash getCodeHash() {
return codeHash;
}
@ -96,6 +101,7 @@ public class StateTrieAccountValue {
return Objects.hash(nonce, balance, storageRoot, codeHash);
}
@Override
public void writeTo(final RLPOutput out) {
out.startList();

@ -73,7 +73,8 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider {
inMemoryKeyValueStorageProvider,
blockchain,
cachedMerkleTrieLoader,
new NoOpMetricsSystem());
new NoOpMetricsSystem(),
null);
}
public static MutableWorldState createInMemoryWorldState() {

@ -89,7 +89,7 @@ public class BlockImportExceptionHandlingTest {
// do we need to also test with a DefaultWorldStateArchive?
spy(
new BonsaiWorldStateProvider(
storageProvider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem()));
storageProvider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null));
private final BonsaiWorldState persisted =
spy(

@ -139,7 +139,8 @@ public abstract class AbstractIsolationTests {
blockchain,
Optional.of(16L),
new CachedMerkleTrieLoader(new NoOpMetricsSystem()),
new NoOpMetricsSystem());
new NoOpMetricsSystem(),
null);
var ws = archive.getMutable();
genesisState.writeStateTo(ws);
protocolContext = new ProtocolContext(blockchain, archive, null);

@ -20,6 +20,7 @@ import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyVa
import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
@ -105,7 +106,8 @@ public class BonsaiWorldStateArchiveTest {
blockchain,
Optional.of(512L),
new CachedMerkleTrieLoader(new NoOpMetricsSystem()),
new NoOpMetricsSystem());
new NoOpMetricsSystem(),
null);
final BlockHeader blockHeader = blockBuilder.number(0).buildHeader();
final BlockHeader chainHead = blockBuilder.number(512).buildHeader();
when(blockchain.getChainHeadHeader()).thenReturn(chainHead);
@ -141,7 +143,9 @@ public class BonsaiWorldStateArchiveTest {
public void testGetMutableWithStorageInconsistencyRollbackTheState() {
when(keyValueStorage.startTransaction()).thenReturn(mock(KeyValueStorageTransaction.class));
when(trieLogManager.getTrieLogLayer(any())).thenReturn(Optional.of(mock(TrieLogLayer.class)));
doAnswer(__ -> Optional.of(mock(TrieLogLayer.class)))
.when(trieLogManager)
.getTrieLogLayer(any(Hash.class));
var worldStateStorage =
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem());
@ -201,7 +205,9 @@ public class BonsaiWorldStateArchiveTest {
final BlockHeader blockHeaderChainB =
blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader();
when(trieLogManager.getTrieLogLayer(any())).thenReturn(Optional.of(mock(TrieLogLayer.class)));
doAnswer(__ -> Optional.of(mock(TrieLogLayer.class)))
.when(trieLogManager)
.getTrieLogLayer(any(Hash.class));
var worldStateStorage =
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem());
@ -242,8 +248,9 @@ public class BonsaiWorldStateArchiveTest {
final BlockHeader blockHeaderChainB =
blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader();
when(trieLogManager.getTrieLogLayer(any(Hash.class)))
.thenReturn(Optional.of(mock(TrieLogLayer.class)));
doAnswer(__ -> Optional.of(mock(TrieLogLayer.class)))
.when(trieLogManager)
.getTrieLogLayer(any(Hash.class));
bonsaiWorldStateArchive =
spy(

@ -24,8 +24,8 @@ import static org.mockito.Mockito.verify;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
import org.hyperledger.besu.ethereum.core.TrieGenerator;
import org.hyperledger.besu.ethereum.rlp.RLP;

@ -19,9 +19,9 @@ import static org.assertj.core.api.Assertions.assertThat;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
import org.hyperledger.besu.ethereum.core.TrieGenerator;
import org.hyperledger.besu.ethereum.rlp.RLP;

@ -126,7 +126,7 @@ public class LogRollingTests {
new CachedMerkleTrieLoader(new NoOpMetricsSystem());
archive =
new BonsaiWorldStateProvider(
provider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem());
provider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null);
accountStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
@ -149,7 +149,11 @@ public class LogRollingTests {
new CachedMerkleTrieLoader(new NoOpMetricsSystem());
secondArchive =
new BonsaiWorldStateProvider(
secondProvider, blockchain, secondOptimizedMerkleTrieLoader, new NoOpMetricsSystem());
secondProvider,
blockchain,
secondOptimizedMerkleTrieLoader,
new NoOpMetricsSystem(),
null);
secondAccountStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(

@ -49,7 +49,7 @@ public class RollingImport {
new CachedMerkleTrieLoader(new NoOpMetricsSystem());
final BonsaiWorldStateProvider archive =
new BonsaiWorldStateProvider(
provider, null, cachedMerkleTrieLoader, new NoOpMetricsSystem());
provider, null, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null);
final InMemoryKeyValueStorage accountStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);

@ -19,13 +19,15 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture;
import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256;
@ -36,31 +38,37 @@ import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class TrieLogFactoryTests {
BlockchainSetupUtil setup = BlockchainSetupUtil.forTesting(DataStorageFormat.BONSAI);
final BlockchainSetupUtil setup = BlockchainSetupUtil.forTesting(DataStorageFormat.BONSAI);
final Address accountFixture = Address.fromHexString("0xdeadbeef");
final BlockHeader headerFixture =
new BlockHeaderTestFixture()
.parentHash(setup.getGenesisState().getBlock().getHash())
.coinbase(Address.ZERO)
.buildHeader();
final TrieLogLayer trieLogFixture =
new TrieLogLayer()
.setBlockHash(headerFixture.getBlockHash())
.addAccountChange(
accountFixture,
null,
new StateTrieAccountValue(0, Wei.fromEth(1), Hash.EMPTY, Hash.EMPTY))
.addCodeChange(
Address.ZERO,
null,
Bytes.fromHexString("0xfeeddeadbeef"),
headerFixture.getBlockHash())
.addStorageChange(Address.ZERO, new StorageSlotKey(UInt256.ZERO), null, UInt256.ONE);
@Test
public void testSerializeDeserializeAreEqual() {
BlockHeader header =
new BlockHeaderTestFixture()
.parentHash(setup.getGenesisState().getBlock().getHash())
.coinbase(Address.ZERO)
.buildHeader();
TrieLogLayer fixture =
new TrieLogLayer()
.setBlockHash(header.getBlockHash())
.addAccountChange(
Address.fromHexString("0xdeadbeef"),
null,
new StateTrieAccountValue(0, Wei.fromEth(1), Hash.EMPTY, Hash.EMPTY))
.addCodeChange(
Address.ZERO, null, Bytes.fromHexString("0xdeadbeef"), header.getBlockHash())
.addStorageChange(Address.ZERO, new StorageSlotKey(UInt256.ZERO), null, UInt256.ONE);
TrieLogFactory<TrieLogLayer> factory = new TrieLogFactoryImpl();
byte[] rlp = factory.serialize(fixture);
TrieLogFactory factory = new TrieLogFactoryImpl();
byte[] rlp = factory.serialize(trieLogFixture);
TrieLogLayer layer = factory.deserialize(rlp);
assertThat(layer).isEqualTo(fixture);
TrieLog layer = factory.deserialize(rlp);
assertThat(layer).isEqualTo(trieLogFixture);
}
}

@ -15,10 +15,11 @@
*/
package org.hyperledger.besu.ethereum.bonsai.trielog;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import java.util.Optional;
@ -66,11 +67,11 @@ public class TrieLogLayerTests {
Assertions.assertThat(trieLogLayer).isEqualTo(otherTrieLogLayer);
Optional<StateTrieAccountValue> priorAccount = trieLogLayer.getPriorAccount(address);
Optional<AccountValue> priorAccount = trieLogLayer.getPriorAccount(address);
Assertions.assertThat(priorAccount).isPresent();
Assertions.assertThat(priorAccount.get()).isEqualTo(oldValue);
Optional<StateTrieAccountValue> updatedAccount = trieLogLayer.getAccount(address);
Optional<AccountValue> updatedAccount = trieLogLayer.getAccount(address);
Assertions.assertThat(updatedAccount).isPresent();
Assertions.assertThat(updatedAccount.get()).isEqualTo(newValue);
}

@ -59,7 +59,12 @@ public class TrieLogManagerTests {
public void setup() {
trieLogManager =
new CachedWorldStorageManager(
archive, blockchain, bonsaiWorldStateKeyValueStorage, new NoOpMetricsSystem(), 512);
archive,
blockchain,
bonsaiWorldStateKeyValueStorage,
new NoOpMetricsSystem(),
512,
null);
}
@Test

@ -69,7 +69,7 @@ Calculated : ${currentHash}
tasks.register('checkAPIChanges', FileStateChecker) {
description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
files = sourceSets.main.allJava.files
knownHash = 'aqSCy4h1L4eGXKojEW09GiRoyl1YGN9hLf5B2wf0oxg='
knownHash = '3/qsZ9+jA10YbctpPtQ5Rn7cvYHwKRWBv6jXa+7WQMY='
}
check.dependsOn('checkAPIChanges')

@ -21,6 +21,15 @@ import java.util.Optional;
/** Allows plugins to access Besu services. */
public interface BesuContext {
/**
* Add service.
*
* @param <T> the type parameter
* @param serviceType the service type
* @param service the service
*/
<T extends BesuService> void addService(final Class<T> serviceType, final T service);
/**
* Get the requested service, if it is available. There are a number of reasons that a service may
* not be available:

@ -0,0 +1,58 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.plugin.services;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogProvider;
import java.util.List;
/**
* A service interface for registering observers for trie log events.
*
* <p>Implementations must be thread-safe.
*/
public interface TrieLogService extends BesuService {
/**
* Provides list of observers to configure for trie log events.
*
* @return the list of observers to configure
*/
List<TrieLogEvent.TrieLogObserver> getObservers();
/**
* Provide a TrieLogFactory implementation to use for serializing and deserializing TrieLogs.
*
* @return the TrieLogFactory implementation
*/
TrieLogFactory getTrieLogFactory();
/**
* Configure a TrieLogProvider implementation to use for retrieving stored TrieLogs.
*
* @param provider the TrieLogProvider implementation
*/
void configureTrieLogProvider(TrieLogProvider provider);
/**
* Retrieve the configured TrieLogProvider implementation.
*
* @return the TrieLogProvider implementation
*/
TrieLogProvider getTrieLogProvider();
}

@ -0,0 +1,176 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256;
/**
* An interface for interacting with TrieLog objects, which represent changes to accounts, code, and
* storage in a blockchain state trie.
*/
public interface TrieLog {
/**
* Gets the block hash associated with this TrieLog.
*
* @return the block hash
*/
Hash getBlockHash();
/**
* Gets the block number associated with this TrieLog, if available.
*
* @return an Optional containing the block number if available, otherwise an empty Optional
*/
Optional<Long> getBlockNumber();
/** Freezes the TrieLog to prevent further modifications. */
void freeze();
/**
* Gets a map of addresses to their account value changes.
*
* @param <U> the type of LogTuple representing the account value changes
* @return a map of addresses to their account value changes
*/
<U extends LogTuple<AccountValue>> Map<Address, U> getAccountChanges();
/**
* Gets a map of addresses to their code changes.
*
* @param <U> the type of LogTuple representing the code changes
* @return a map of addresses to their code changes
*/
<U extends LogTuple<Bytes>> Map<Address, U> getCodeChanges();
/**
* Gets a map of addresses to their storage changes.
*
* @param <U> the type of LogTuple representing the storage changes
* @return a map of addresses to their storage changes
*/
<U extends LogTuple<UInt256>> Map<Address, Map<StorageSlotKey, U>> getStorageChanges();
/**
* Gets the storage changes for a specific address.
*
* @param address the address to get the storage changes for
* @param <U> the type of LogTuple representing the storage changes
* @return a map of storage slot keys to their changes
*/
<U extends LogTuple<UInt256>> Map<StorageSlotKey, U> getStorageChanges(final Address address);
/**
* Gets the prior code for a specific address, if available.
*
* @param address the address to get the prior code for
* @return an Optional containing the prior code if available, otherwise an empty Optional
*/
Optional<Bytes> getPriorCode(final Address address);
/**
* Gets the code for a specific address, if available.
*
* @param address the address to get the code for
* @return an Optional containing the code if available, otherwise an empty Optional
*/
Optional<Bytes> getCode(final Address address);
/**
* Gets the prior storage value for a specific address and storage slot key, if available.
*
* @param address the address to get the prior storage value for
* @param storageSlotKey the storage slot key to get the prior storage value for
* @return an Optional containing the prior storage value if available, otherwise an empty
* Optional
*/
Optional<UInt256> getPriorStorageByStorageSlotKey(
final Address address, final StorageSlotKey storageSlotKey);
/**
* Gets the storage value for a specific address and storage slot key, if available.
*
* @param address the address to get the storage value for
* @param storageSlotKey the storage slot key to get the storage value for
* @return an Optional containing the storage value if available, otherwise an empty Optional
*/
Optional<UInt256> getStorageByStorageSlotKey(
final Address address, final StorageSlotKey storageSlotKey);
/**
* Gets the prior account value for a specific address, if available.
*
* @param address the address to get the prior account value for
* @return an Optional containing the prior account value if available, otherwise an empty
* Optional
*/
Optional<? extends AccountValue> getPriorAccount(final Address address);
/**
* Gets the account value for a specific address, if available.
*
* @param address the address to get the account value for
* @return an Optional containing the account value if available, otherwise an empty Optional
*/
Optional<? extends AccountValue> getAccount(final Address address);
/**
* An interface representing a tuple of prior and updated values for a specific type T in a log.
* The interface also provides methods to check if the values are unchanged or cleared.
*
* @param <T> the type of values stored in the log tuple
*/
public interface LogTuple<T> {
/**
* Gets the prior value of the tuple.
*
* @return the prior value of type T
*/
T getPrior();
/**
* Gets the updated value of the tuple.
*
* @return the updated value of type T
*/
T getUpdated();
/**
* Checks if the prior and updated values are equal.
*
* @return true if the prior and updated values are equal, false otherwise
*/
default boolean isUnchanged() {
return Objects.equals(getUpdated(), getPrior());
}
/**
* Checks if the updated value represents a cleared state.
*
* @return true if the updated value is cleared, false otherwise
*/
boolean isCleared();
}
}

@ -0,0 +1,50 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import java.util.Map;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256;
/** Accumulator interface tor provding trie updates for creating TrieLogs. */
public interface TrieLogAccumulator {
/**
* Returns the state trie accounts which have been updated.
*
* @return the accounts to update
*/
Map<Address, ? extends TrieLog.LogTuple<? extends AccountValue>> getAccountsToUpdate();
/**
* Returns code which has been updated.
*
* @return the code to update
*/
Map<Address, ? extends TrieLog.LogTuple<Bytes>> getCodeToUpdate();
/**
* Returns storage which has been updated.
*
* @return the storage to update
*/
Map<Address, ? extends Map<StorageSlotKey, ? extends TrieLog.LogTuple<UInt256>>>
getStorageToUpdate();
}

@ -0,0 +1,49 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
/** A TrieLog event. */
public interface TrieLogEvent {
/** The type of the event. */
enum Type {
/** TrieLog added event type */
ADDED
}
/**
* The type of the event.
*
* @return the type of the event
*/
TrieLogEvent.Type getType();
/**
* The TrieLog layer.
*
* @return the TrieLog layer
*/
TrieLog layer();
/** Observer interface for TrieLog events. */
interface TrieLogObserver {
/**
* Called when a TrieLog is added.
*
* @param event the TrieLog event
*/
void onTrieLogAdded(TrieLogEvent event);
}
}

@ -0,0 +1,46 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
import org.hyperledger.besu.plugin.data.BlockHeader;
/** Interface for serializing and deserializing {@link TrieLog} objects. */
public interface TrieLogFactory {
/**
* Creates a new TrieLog object.
*
* @param accumulator the accumulator
* @param blockHeader the block header
* @return a new TrieLog object
*/
TrieLog create(TrieLogAccumulator accumulator, BlockHeader blockHeader);
/**
* Deserializes a TrieLog object.
*
* @param bytes the serialized TrieLog
* @return the deserialized TrieLog
*/
TrieLog deserialize(final byte[] bytes);
/**
* Serializes a TrieLog object.
*
* @param layer the TrieLog
* @return the serialized TrieLog
*/
byte[] serialize(final TrieLog layer);
}

@ -0,0 +1,61 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
import org.hyperledger.besu.datatypes.Hash;
import java.util.List;
import java.util.Optional;
/** Trielog provider interface for a given block hash. */
public interface TrieLogProvider {
/**
* Returns the TrieLog layer for the given block hash.
*
* @param blockHash the block hash
* @return the TrieLog layer for the given block hash
* @param <T> the type of the TrieLog
*/
<T extends TrieLog.LogTuple<?>> Optional<TrieLog> getTrieLogLayer(final Hash blockHash);
/**
* Returns the TrieLog layer for the given block number.
*
* @param blockNumber the block hash
* @return the TrieLog layer for the given block hash
* @param <T> the type of the TrieLog
*/
<T extends TrieLog.LogTuple<?>> Optional<TrieLog> getTrieLogLayer(final long blockNumber);
/**
* Returns the TrieLog layers for the given block number range.
*
* @param fromBlockNumber the from block number
* @param toBlockNumber the to block number
* @return the TrieLog layers for the given block number range
* @param <T> the type of the TrieLog
*/
<T extends TrieLog.LogTuple<?>> List<TrieLogRangeTuple> getTrieLogsByRange(
long fromBlockNumber, long toBlockNumber);
/**
* Block and TrieLog layer composition, used for returning a range of TrieLog layers.
*
* @param blockHash the block hash
* @param blockNumber the block number
* @param trieLog the associated TrieLog layer
*/
record TrieLogRangeTuple(Hash blockHash, long blockNumber, TrieLog trieLog) {}
}
Loading…
Cancel
Save