TrieLogFactory plugin support (#5440)

* TrieLogs in plugin data
* adds dagger-wired plugincontext and TrieLogService
* add getTrieLogByRange, TrieLogRangeTuple composition
---------

Signed-off-by: garyschulte <garyschulte@gmail.com>
pull/5465/head
garyschulte 2 years ago committed by GitHub
parent 3b4f5e13d6
commit c85841d308
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 10
      besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
  2. 3
      besu/src/main/java/org/hyperledger/besu/components/BesuCommandModule.java
  3. 12
      besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java
  4. 29
      besu/src/main/java/org/hyperledger/besu/components/BesuPluginContextModule.java
  5. 3
      besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
  6. 3
      besu/src/main/java/org/hyperledger/besu/services/BesuPluginContextImpl.java
  7. 2
      besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java
  8. 55
      datatypes/src/main/java/org/hyperledger/besu/datatypes/AccountValue.java
  9. 111
      datatypes/src/main/java/org/hyperledger/besu/datatypes/StorageSlotKey.java
  10. 1
      ethereum/core/build.gradle
  11. 17
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java
  12. 38
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java
  13. 21
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java
  14. 4
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java
  15. 91
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java
  16. 2
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java
  17. 6
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java
  18. 35
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java
  19. 20
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogAddedEvent.java
  20. 120
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogFactoryImpl.java
  21. 78
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayer.java
  22. 6
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java
  23. 6
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java
  24. 50
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java
  25. 1
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldView.java
  26. 62
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/StorageSlotKey.java
  27. 16
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/StateTrieAccountValue.java
  28. 3
      ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java
  29. 2
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java
  30. 3
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java
  31. 17
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java
  32. 2
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java
  33. 2
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java
  34. 8
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java
  35. 2
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java
  36. 52
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogFactoryTests.java
  37. 7
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayerTests.java
  38. 7
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java
  39. 2
      plugin-api/build.gradle
  40. 9
      plugin-api/src/main/java/org/hyperledger/besu/plugin/BesuContext.java
  41. 58
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TrieLogService.java
  42. 176
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLog.java
  43. 50
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogAccumulator.java
  44. 49
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogEvent.java
  45. 46
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogFactory.java
  46. 61
      plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogProvider.java

@ -1329,11 +1329,10 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
private BesuComponent besuComponent; private BesuComponent besuComponent;
private final Supplier<ObservableMetricsSystem> metricsSystem = private final Supplier<ObservableMetricsSystem> metricsSystem =
Suppliers.memoize( Suppliers.memoize(
() -> { () ->
return besuComponent == null besuComponent == null || besuComponent.getObservableMetricsSystem() == null
? MetricsSystemFactory.create(metricsConfiguration()) ? MetricsSystemFactory.create(metricsConfiguration())
: besuComponent.getObservableMetricsSystem(); : besuComponent.getObservableMetricsSystem());
});
private Vertx vertx; private Vertx vertx;
private EnodeDnsConfiguration enodeDnsConfiguration; private EnodeDnsConfiguration enodeDnsConfiguration;
private KeyValueStorageProvider keyValueStorageProvider; private KeyValueStorageProvider keyValueStorageProvider;
@ -1412,6 +1411,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
final PrivacyPluginServiceImpl privacyPluginService, final PrivacyPluginServiceImpl privacyPluginService,
final PkiBlockCreationConfigurationProvider pkiBlockCreationConfigProvider, final PkiBlockCreationConfigurationProvider pkiBlockCreationConfigProvider,
final RpcEndpointServiceImpl rpcEndpointServiceImpl) { final RpcEndpointServiceImpl rpcEndpointServiceImpl) {
this.besuComponent = besuComponent;
this.logger = besuComponent.getBesuCommandLogger(); this.logger = besuComponent.getBesuCommandLogger();
this.rlpBlockImporter = rlpBlockImporter; this.rlpBlockImporter = rlpBlockImporter;
this.rlpBlockExporterFactory = rlpBlockExporterFactory; this.rlpBlockExporterFactory = rlpBlockExporterFactory;

@ -23,7 +23,6 @@ import org.hyperledger.besu.chainimport.RlpBlockImporter;
import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.cli.BesuCommand;
import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.controller.BesuController;
import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
import org.hyperledger.besu.services.BesuPluginContextImpl;
import javax.inject.Named; import javax.inject.Named;
import javax.inject.Singleton; import javax.inject.Singleton;
@ -50,7 +49,7 @@ public class BesuCommandModule {
RlpBlockExporter::new, RlpBlockExporter::new,
new RunnerBuilder(), new RunnerBuilder(),
new BesuController.Builder(), new BesuController.Builder(),
new BesuPluginContextImpl(), besuComponent.getBesuPluginContext(),
System.getenv()); System.getenv());
besuCommand.toCommandLine(); besuCommand.toCommandLine();
return besuCommand; return besuCommand;

@ -21,6 +21,7 @@ import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoaderModule; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoaderModule;
import org.hyperledger.besu.metrics.MetricsSystemModule; import org.hyperledger.besu.metrics.MetricsSystemModule;
import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.services.BesuPluginContextImpl;
import javax.inject.Named; import javax.inject.Named;
import javax.inject.Singleton; import javax.inject.Singleton;
@ -34,7 +35,8 @@ import org.slf4j.Logger;
modules = { modules = {
BesuCommandModule.class, BesuCommandModule.class,
MetricsSystemModule.class, MetricsSystemModule.class,
CachedMerkleTrieLoaderModule.class CachedMerkleTrieLoaderModule.class,
BesuPluginContextModule.class
}) })
public interface BesuComponent { public interface BesuComponent {
@ -66,4 +68,12 @@ public interface BesuComponent {
*/ */
@Named("besuCommandLogger") @Named("besuCommandLogger")
Logger getBesuCommandLogger(); Logger getBesuCommandLogger();
/**
* Besu plugin context for doing plugin service discovery.
*
* @return BesuComponent
*/
@Named("besuPluginContext")
BesuPluginContextImpl getBesuPluginContext();
} }

@ -13,16 +13,29 @@
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
* *
*/ */
package org.hyperledger.besu.ethereum.bonsai.trielog; package org.hyperledger.besu.components;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.services.BesuPluginContextImpl;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
/** Interface for serializing and deserializing {@link TrieLogLayer} objects. */ import javax.inject.Named;
public interface TrieLogFactory<T extends TrieLogLayer> { import javax.inject.Singleton;
T create(BonsaiWorldStateUpdateAccumulator accumulator, final Hash blockHash);
T deserialize(final byte[] bytes); import dagger.Module;
import dagger.Provides;
byte[] serialize(final T layer); /** A dagger module that know how to create the BesuPluginContextImpl singleton. */
@Module
public class BesuPluginContextModule {
/**
* Creates a BesuPluginContextImpl, used for plugin service discovery.
*
* @return the BesuPluginContext
*/
@Provides
@Named("besuPluginContext")
@Singleton
public BesuPluginContextImpl provideBesuPluginContext() {
return new BesuPluginContextImpl();
}
} }

@ -1022,7 +1022,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
blockchain, blockchain,
Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
cachedMerkleTrieLoader, cachedMerkleTrieLoader,
metricsSystem); metricsSystem,
besuComponent.getBesuPluginContext());
case FOREST: case FOREST:
default: default:

@ -83,6 +83,7 @@ public class BesuPluginContextImpl implements BesuContext, PluginVersionsProvide
* @param serviceType the service type * @param serviceType the service type
* @param service the service * @param service the service
*/ */
@Override
public <T extends BesuService> void addService(final Class<T> serviceType, final T service) { public <T extends BesuService> void addService(final Class<T> serviceType, final T service) {
checkArgument(serviceType.isInterface(), "Services must be Java interfaces."); checkArgument(serviceType.isInterface(), "Services must be Java interfaces.");
checkArgument( checkArgument(
@ -118,7 +119,7 @@ public class BesuPluginContextImpl implements BesuContext, PluginVersionsProvide
for (final BesuPlugin plugin : serviceLoader) { for (final BesuPlugin plugin : serviceLoader) {
try { try {
plugin.register(this); plugin.register(this);
LOG.debug("Registered plugin of type {}.", plugin.getClass().getName()); LOG.info("Registered plugin of type {}.", plugin.getClass().getName());
addPluginVersion(plugin); addPluginVersion(plugin);
} catch (final Exception e) { } catch (final Exception e) {
LOG.error( LOG.error(

@ -237,6 +237,8 @@ public abstract class CommandTestAbstract {
when(mockControllerBuilder.lowerBoundPeers(anyInt())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.lowerBoundPeers(anyInt())).thenReturn(mockControllerBuilder);
when(mockControllerBuilder.maxRemotelyInitiatedPeers(anyInt())) when(mockControllerBuilder.maxRemotelyInitiatedPeers(anyInt()))
.thenReturn(mockControllerBuilder); .thenReturn(mockControllerBuilder);
when(mockControllerBuilder.besuComponent(any(BesuComponent.class)))
.thenReturn(mockControllerBuilder);
// doReturn used because of generic BesuController // doReturn used because of generic BesuController
doReturn(mockController).when(mockControllerBuilder).build(); doReturn(mockController).when(mockControllerBuilder).build();
lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager); lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);

@ -0,0 +1,55 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.datatypes;
import org.hyperledger.besu.ethereum.rlp.RLPOutput;
/** The values of an account in the world state trie. */
public interface AccountValue {
/**
* The nonce of the account.
*
* @return the nonce of the account.
*/
long getNonce();
/**
* The available balance of that account.
*
* @return the balance, in Wei, of the account.
*/
Wei getBalance();
/**
* The hash of the root of the storage trie associated with this account.
*
* @return the hash of the root node of the storage trie.
*/
Hash getStorageRoot();
/**
* The hash of the EVM bytecode associated with this account.
*
* @return the hash of the account code (which may be {@link Hash#EMPTY}).
*/
Hash getCodeHash();
/**
* Writes the account value to the provided {@link RLPOutput}.
*
* @param out the {@link RLPOutput} to write to.
*/
void writeTo(final RLPOutput out);
}

@ -0,0 +1,111 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.datatypes;
import java.util.Objects;
import java.util.Optional;
import org.apache.tuweni.units.bigints.UInt256;
import org.jetbrains.annotations.NotNull;
/**
* StorageSlotKey represents a key used for storage slots in Ethereum. It contains the hash of the
* slot key and an optional representation of the key itself.
*
* <p>The class provides methods for accessing the hash and key, as well as methods for equality,
* hashcode, and comparison.
*
* <p>StorageSlotKey is used to uniquely identify storage slots within the Ethereum state.
*/
public class StorageSlotKey implements Comparable<StorageSlotKey> {
private final Hash slotHash;
private final Optional<UInt256> slotKey;
/**
* Creates a StorageSlotKey.
*
* @param slotHash Hashed storage slot key.
* @param slotKey Optional UInt256 storage slot key.
*/
public StorageSlotKey(final Hash slotHash, final Optional<UInt256> slotKey) {
this.slotHash = slotHash;
this.slotKey = slotKey;
}
/**
* Creates a StorageSlotKey, hashing the slotKey.
*
* @param slotKey the UInt256 storage slot key.
*/
public StorageSlotKey(final UInt256 slotKey) {
this(Hash.hash(slotKey), Optional.of(slotKey));
}
/**
* Gets the hash representation of the storage slot key.
*
* @return the hash of the storage slot key.
*/
public Hash getSlotHash() {
return slotHash;
}
/**
* Gets the optional UInt256 representation of the storage slot key.
*
* @return an Optional containing the UInt256 storage slot key if present, otherwise an empty
* Optional.
*/
public Optional<UInt256> getSlotKey() {
return slotKey;
}
/**
* Indicates whether some other object is "equal to" this one. Two StorageSlotKey objects are
* considered equal if their slot hash values are equal.
*
* @param o the reference object with which to compare.
* @return true if this object is the same as the obj argument; false otherwise.
*/
@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
StorageSlotKey that = (StorageSlotKey) o;
return Objects.equals(slotHash, that.slotHash);
}
@Override
public int hashCode() {
return Objects.hash(slotHash.hashCode());
}
@Override
public String toString() {
return String.format(
"StorageSlotKey{slotHash=%s, slotKey=%s}",
slotHash, slotKey.map(UInt256::toString).orElse("null"));
}
@Override
public int compareTo(@NotNull final StorageSlotKey other) {
return this.slotHash.compareTo(other.slotHash);
}
}

@ -50,6 +50,7 @@ dependencies {
annotationProcessor 'com.google.dagger:dagger-compiler' annotationProcessor 'com.google.dagger:dagger-compiler'
implementation 'io.opentelemetry:opentelemetry-api' implementation 'io.opentelemetry:opentelemetry-api'
implementation 'io.vertx:vertx-core' implementation 'io.vertx:vertx-core'
implementation 'net.java.dev.jna:jna' implementation 'net.java.dev.jna:jna'
implementation 'org.apache.commons:commons-lang3' implementation 'org.apache.commons:commons-lang3'
implementation 'org.apache.tuweni:tuweni-bytes' implementation 'org.apache.tuweni:tuweni-bytes'

@ -16,6 +16,7 @@
package org.hyperledger.besu.ethereum.bonsai; package org.hyperledger.besu.ethereum.bonsai;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.datatypes.Wei;
@ -24,7 +25,7 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPException;
import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPInput;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.rlp.RLPOutput;
import org.hyperledger.besu.evm.ModificationNotAllowedException; import org.hyperledger.besu.evm.ModificationNotAllowedException;
import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.account.AccountStorageEntry;
import org.hyperledger.besu.evm.account.EvmAccount; import org.hyperledger.besu.evm.account.EvmAccount;
@ -40,7 +41,7 @@ import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt256; import org.apache.tuweni.units.bigints.UInt256;
public class BonsaiAccount implements MutableAccount, EvmAccount { public class BonsaiAccount implements MutableAccount, EvmAccount, AccountValue {
private final BonsaiWorldView context; private final BonsaiWorldView context;
private final boolean mutable; private final boolean mutable;
@ -77,7 +78,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
public BonsaiAccount( public BonsaiAccount(
final BonsaiWorldView context, final BonsaiWorldView context,
final Address address, final Address address,
final StateTrieAccountValue stateTrieAccount, final AccountValue stateTrieAccount,
final boolean mutable) { final boolean mutable) {
this( this(
context, context,
@ -224,6 +225,12 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
public Bytes serializeAccount() { public Bytes serializeAccount() {
final BytesValueRLPOutput out = new BytesValueRLPOutput(); final BytesValueRLPOutput out = new BytesValueRLPOutput();
writeTo(out);
return out.encoded();
}
@Override
public void writeTo(final RLPOutput out) {
out.startList(); out.startList();
out.writeLongScalar(nonce); out.writeLongScalar(nonce);
@ -232,7 +239,6 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
out.writeBytes(codeHash); out.writeBytes(codeHash);
out.endList(); out.endList();
return out.encoded();
} }
@Override @Override
@ -262,6 +268,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
} }
} }
@Override
public Hash getStorageRoot() { public Hash getStorageRoot() {
return storageRoot; return storageRoot;
} }
@ -298,7 +305,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
* @throws IllegalStateException if the stored values differ * @throws IllegalStateException if the stored values differ
*/ */
public static void assertCloseEnoughForDiffing( public static void assertCloseEnoughForDiffing(
final BonsaiAccount source, final StateTrieAccountValue account, final String context) { final BonsaiAccount source, final AccountValue account, final String context) {
if (source == null) { if (source == null) {
throw new IllegalStateException(context + ": source is null but target isn't"); throw new IllegalStateException(context + ": source is null but target isn't");
} else { } else {

@ -16,15 +16,12 @@
package org.hyperledger.besu.ethereum.bonsai; package org.hyperledger.besu.ethereum.bonsai;
import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import java.util.Objects;
import java.util.function.BiConsumer;
import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder;
public class BonsaiValue<T> { public class BonsaiValue<T> implements TrieLog.LogTuple<T> {
private T prior; private T prior;
private T updated; private T updated;
private boolean cleared; private boolean cleared;
@ -41,10 +38,12 @@ public class BonsaiValue<T> {
this.cleared = cleared; this.cleared = cleared;
} }
@Override
public T getPrior() { public T getPrior() {
return prior; return prior;
} }
@Override
public T getUpdated() { public T getUpdated() {
return updated; return updated;
} }
@ -60,38 +59,11 @@ public class BonsaiValue<T> {
return this; return this;
} }
public void writeRlp(final RLPOutput output, final BiConsumer<RLPOutput, T> writer) {
output.startList();
writeInnerRlp(output, writer);
output.endList();
}
public void writeInnerRlp(final RLPOutput output, final BiConsumer<RLPOutput, T> writer) {
if (prior == null) {
output.writeNull();
} else {
writer.accept(output, prior);
}
if (updated == null) {
output.writeNull();
} else {
writer.accept(output, updated);
}
if (!cleared) {
output.writeNull();
} else {
output.writeInt(1);
}
}
public boolean isUnchanged() {
return Objects.equals(updated, prior);
}
public void setCleared() { public void setCleared() {
this.cleared = true; this.cleared = true;
} }
@Override
public boolean isCleared() { public boolean isCleared() {
return cleared; return cleared;
} }

@ -23,7 +23,6 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
@ -40,6 +39,8 @@ import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.evm.worldstate.WorldState;
import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.BesuContext;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
@ -70,14 +71,16 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
final StorageProvider provider, final StorageProvider provider,
final Blockchain blockchain, final Blockchain blockchain,
final CachedMerkleTrieLoader cachedMerkleTrieLoader, final CachedMerkleTrieLoader cachedMerkleTrieLoader,
final ObservableMetricsSystem metricsSystem) { final ObservableMetricsSystem metricsSystem,
final BesuContext pluginContext) {
this( this(
(BonsaiWorldStateKeyValueStorage) (BonsaiWorldStateKeyValueStorage)
provider.createWorldStateStorage(DataStorageFormat.BONSAI), provider.createWorldStateStorage(DataStorageFormat.BONSAI),
blockchain, blockchain,
Optional.empty(), Optional.empty(),
cachedMerkleTrieLoader, cachedMerkleTrieLoader,
metricsSystem); metricsSystem,
pluginContext);
} }
public BonsaiWorldStateProvider( public BonsaiWorldStateProvider(
@ -85,7 +88,8 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
final Blockchain blockchain, final Blockchain blockchain,
final Optional<Long> maxLayersToLoad, final Optional<Long> maxLayersToLoad,
final CachedMerkleTrieLoader cachedMerkleTrieLoader, final CachedMerkleTrieLoader cachedMerkleTrieLoader,
final ObservableMetricsSystem metricsSystem) { final ObservableMetricsSystem metricsSystem,
final BesuContext pluginContext) {
// TODO: de-dup constructors // TODO: de-dup constructors
this.trieLogManager = this.trieLogManager =
@ -94,7 +98,8 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
blockchain, blockchain,
worldStateStorage, worldStateStorage,
metricsSystem, metricsSystem,
maxLayersToLoad.orElse(RETAINED_LAYERS)); maxLayersToLoad.orElse(RETAINED_LAYERS),
pluginContext);
this.blockchain = blockchain; this.blockchain = blockchain;
this.worldStateStorage = worldStateStorage; this.worldStateStorage = worldStateStorage;
this.persistedState = new BonsaiWorldState(this, worldStateStorage); this.persistedState = new BonsaiWorldState(this, worldStateStorage);
@ -189,8 +194,8 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
final Optional<BlockHeader> maybePersistedHeader = final Optional<BlockHeader> maybePersistedHeader =
blockchain.getBlockHeader(mutableState.blockHash()).map(BlockHeader.class::cast); blockchain.getBlockHeader(mutableState.blockHash()).map(BlockHeader.class::cast);
final List<TrieLogLayer> rollBacks = new ArrayList<>(); final List<TrieLog> rollBacks = new ArrayList<>();
final List<TrieLogLayer> rollForwards = new ArrayList<>(); final List<TrieLog> rollForwards = new ArrayList<>();
if (maybePersistedHeader.isEmpty()) { if (maybePersistedHeader.isEmpty()) {
trieLogManager.getTrieLogLayer(mutableState.blockHash()).ifPresent(rollBacks::add); trieLogManager.getTrieLogLayer(mutableState.blockHash()).ifPresent(rollBacks::add);
} else { } else {
@ -232,7 +237,7 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
final BonsaiWorldStateUpdateAccumulator bonsaiUpdater = final BonsaiWorldStateUpdateAccumulator bonsaiUpdater =
(BonsaiWorldStateUpdateAccumulator) mutableState.updater(); (BonsaiWorldStateUpdateAccumulator) mutableState.updater();
try { try {
for (final TrieLogLayer rollBack : rollBacks) { for (final TrieLog rollBack : rollBacks) {
LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash());
bonsaiUpdater.rollBack(rollBack); bonsaiUpdater.rollBack(rollBack);
} }

@ -17,9 +17,9 @@ package org.hyperledger.besu.ethereum.bonsai.cache;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrie;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie;
@ -122,7 +122,7 @@ public class CachedMerkleTrieLoader implements BonsaiStorageSubscriber {
Hash.hash(storageRoot), Hash.hash(storageRoot),
Function.identity(), Function.identity(),
Function.identity()); Function.identity());
storageTrie.get(slotKey.slotHash()); storageTrie.get(slotKey.getSlotHash());
} catch (MerkleTrieException e) { } catch (MerkleTrieException e) {
// ignore exception for the cache // ignore exception for the cache
} }

@ -21,10 +21,16 @@ import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStor
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage;
import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager; import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.BesuContext;
import org.hyperledger.besu.plugin.services.TrieLogService;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogProvider;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
@ -33,7 +39,11 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
import java.util.stream.Stream;
import com.google.common.annotations.VisibleForTesting;
import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.bytes.Bytes32;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -50,8 +60,9 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
final BonsaiWorldStateKeyValueStorage worldStateStorage, final BonsaiWorldStateKeyValueStorage worldStateStorage,
final long maxLayersToLoad, final long maxLayersToLoad,
final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash, final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash,
final BesuContext pluginContext,
final ObservableMetricsSystem metricsSystem) { final ObservableMetricsSystem metricsSystem) {
super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash); super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash, pluginContext);
worldStateStorage.subscribe(this); worldStateStorage.subscribe(this);
this.archive = archive; this.archive = archive;
this.metricsSystem = metricsSystem; this.metricsSystem = metricsSystem;
@ -62,13 +73,15 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
final Blockchain blockchain, final Blockchain blockchain,
final BonsaiWorldStateKeyValueStorage worldStateStorage, final BonsaiWorldStateKeyValueStorage worldStateStorage,
final ObservableMetricsSystem metricsSystem, final ObservableMetricsSystem metricsSystem,
final long maxLayersToLoad) { final long maxLayersToLoad,
final BesuContext pluginContext) {
this( this(
archive, archive,
blockchain, blockchain,
worldStateStorage, worldStateStorage,
maxLayersToLoad, maxLayersToLoad,
new ConcurrentHashMap<>(), new ConcurrentHashMap<>(),
pluginContext,
metricsSystem); metricsSystem);
} }
@ -198,4 +211,78 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
public void onCloseStorage() { public void onCloseStorage() {
this.cachedWorldStatesByHash.clear(); this.cachedWorldStatesByHash.clear();
} }
@VisibleForTesting
@Override
protected TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext) {
// if we have a TrieLogService from pluginContext, use it.
var trieLogServicez =
Optional.ofNullable(pluginContext)
.flatMap(context -> context.getService(TrieLogService.class));
if (trieLogServicez.isPresent()) {
var trieLogService = trieLogServicez.get();
// push the TrieLogProvider into the TrieLogService
trieLogService.configureTrieLogProvider(getTrieLogProvider());
// configure plugin observers:
trieLogService.getObservers().forEach(trieLogObservers::subscribe);
// return the TrieLogFactory implementation from the TrieLogService
return trieLogService.getTrieLogFactory();
} else {
// Otherwise default to TrieLogFactoryImpl
return new TrieLogFactoryImpl();
}
}
@VisibleForTesting
TrieLogProvider getTrieLogProvider() {
return new TrieLogProvider() {
@Override
public <T extends TrieLog.LogTuple<?>> Optional<TrieLog> getTrieLogLayer(
final Hash blockHash) {
return CachedWorldStorageManager.this.getTrieLogLayer(blockHash);
}
@Override
public <T extends TrieLog.LogTuple<?>> Optional<TrieLog> getTrieLogLayer(
final long blockNumber) {
return CachedWorldStorageManager.this
.blockchain
.getBlockHeader(blockNumber)
.map(BlockHeader::getHash)
.flatMap(CachedWorldStorageManager.this::getTrieLogLayer);
}
@Override
public <T extends TrieLog.LogTuple<?>> List<TrieLogRangeTuple> getTrieLogsByRange(
final long fromBlockNumber, final long toBlockNumber) {
return rangeAsStream(fromBlockNumber, toBlockNumber)
.map(blockchain::getBlockHeader)
.map(
headerOpt ->
headerOpt.flatMap(
header ->
CachedWorldStorageManager.this
.getTrieLogLayer(header.getBlockHash())
.map(
layer ->
new TrieLogRangeTuple(
header.getBlockHash(), header.getNumber(), layer))))
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toList());
}
Stream<Long> rangeAsStream(final long fromBlockNumber, final long toBlockNumber) {
if (Math.abs(toBlockNumber - fromBlockNumber) > LOG_RANGE_LIMIT) {
throw new IllegalArgumentException("Requested Range too large");
}
long left = Math.min(fromBlockNumber, toBlockNumber);
long right = Math.max(fromBlockNumber, toBlockNumber);
return LongStream.range(left, right).boxed();
}
};
}
} }

@ -16,8 +16,8 @@
package org.hyperledger.besu.ethereum.bonsai.storage; package org.hyperledger.besu.ethereum.bonsai.storage;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.exception.StorageException;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;

@ -15,7 +15,7 @@
package org.hyperledger.besu.ethereum.bonsai.storage; package org.hyperledger.besu.ethereum.bonsai.storage;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey; import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrie;
@ -254,7 +254,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
getStorageValueCounter.inc(); getStorageValueCounter.inc();
Optional<Bytes> response = Optional<Bytes> response =
storageStorage storageStorage
.get(Bytes.concatenate(accountHash, storageSlotKey.slotHash()).toArrayUnsafe()) .get(Bytes.concatenate(accountHash, storageSlotKey.getSlotHash()).toArrayUnsafe())
.map(Bytes::wrap); .map(Bytes::wrap);
if (response.isEmpty()) { if (response.isEmpty()) {
final Optional<Hash> storageRoot = storageRootSupplier.get(); final Optional<Hash> storageRoot = storageRootSupplier.get();
@ -267,7 +267,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
Function.identity(), Function.identity(),
Function.identity()), Function.identity()),
storageRoot.get()) storageRoot.get())
.get(storageSlotKey.slotHash()) .get(storageSlotKey.getSlotHash())
.map(bytes -> Bytes32.leftPad(RLP.decodeValue(bytes))); .map(bytes -> Bytes32.leftPad(RLP.decodeValue(bytes)));
if (response.isEmpty()) getStorageValueMissingMerkleTrieCounter.inc(); if (response.isEmpty()) getStorageValueMissingMerkleTrieCounter.inc();
else getStorageValueMerkleTrieCounter.inc(); else getStorageValueMerkleTrieCounter.inc();

@ -19,11 +19,14 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedBonsaiWorldView; import org.hyperledger.besu.ethereum.bonsai.cache.CachedBonsaiWorldView;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiUpdater; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiUpdater;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent.TrieLogAddedObserver;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.plugin.BesuContext;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent.TrieLogObserver;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import org.hyperledger.besu.util.Subscribers; import org.hyperledger.besu.util.Subscribers;
import java.util.Map; import java.util.Map;
@ -37,27 +40,31 @@ import org.slf4j.LoggerFactory;
public abstract class AbstractTrieLogManager implements TrieLogManager { public abstract class AbstractTrieLogManager implements TrieLogManager {
private static final Logger LOG = LoggerFactory.getLogger(AbstractTrieLogManager.class); private static final Logger LOG = LoggerFactory.getLogger(AbstractTrieLogManager.class);
public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks
public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs
protected final Blockchain blockchain; protected final Blockchain blockchain;
protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage;
protected final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash; protected final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash;
protected final long maxLayersToLoad; protected final long maxLayersToLoad;
private final Subscribers<TrieLogAddedObserver> trieLogAddedObservers = Subscribers.create(); protected final Subscribers<TrieLogObserver> trieLogObservers = Subscribers.create();
// TODO plumb factory from plugin service: protected final TrieLogFactory trieLogFactory;
TrieLogFactory<TrieLogLayer> trieLogFactory = new TrieLogFactoryImpl();
protected AbstractTrieLogManager( protected AbstractTrieLogManager(
final Blockchain blockchain, final Blockchain blockchain,
final BonsaiWorldStateKeyValueStorage worldStateStorage, final BonsaiWorldStateKeyValueStorage worldStateStorage,
final long maxLayersToLoad, final long maxLayersToLoad,
final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash) { final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash,
final BesuContext pluginContext) {
this.blockchain = blockchain; this.blockchain = blockchain;
this.rootWorldStateStorage = worldStateStorage; this.rootWorldStateStorage = worldStateStorage;
this.cachedWorldStatesByHash = cachedWorldStatesByHash; this.cachedWorldStatesByHash = cachedWorldStatesByHash;
this.maxLayersToLoad = maxLayersToLoad; this.maxLayersToLoad = maxLayersToLoad;
this.trieLogFactory = setupTrieLogFactory(pluginContext);
} }
protected abstract TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext);
@Override @Override
public synchronized void saveTrieLog( public synchronized void saveTrieLog(
final BonsaiWorldStateUpdateAccumulator localUpdater, final BonsaiWorldStateUpdateAccumulator localUpdater,
@ -71,11 +78,11 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
final BonsaiUpdater stateUpdater = forWorldState.getWorldStateStorage().updater(); final BonsaiUpdater stateUpdater = forWorldState.getWorldStateStorage().updater();
boolean success = false; boolean success = false;
try { try {
final TrieLogLayer trieLog = prepareTrieLog(forBlockHeader, localUpdater); final TrieLog trieLog = prepareTrieLog(forBlockHeader, localUpdater);
persistTrieLog(forBlockHeader, forWorldStateRootHash, trieLog, stateUpdater); persistTrieLog(forBlockHeader, forWorldStateRootHash, trieLog, stateUpdater);
// notify trie log added observers, synchronously // notify trie log added observers, synchronously
trieLogAddedObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog)));
success = true; success = true;
} finally { } finally {
@ -89,13 +96,13 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
} }
@VisibleForTesting @VisibleForTesting
TrieLogLayer prepareTrieLog( TrieLog prepareTrieLog(
final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) { final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) {
LOG.atDebug() LOG.atDebug()
.setMessage("Adding layered world state for {}") .setMessage("Adding layered world state for {}")
.addArgument(blockHeader::toLogString) .addArgument(blockHeader::toLogString)
.log(); .log();
final TrieLogLayer trieLog = localUpdater.generateTrieLog(blockHeader.getBlockHash()); final TrieLog trieLog = localUpdater.generateTrieLog(blockHeader);
trieLog.freeze(); trieLog.freeze();
return trieLog; return trieLog;
} }
@ -117,7 +124,7 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
private void persistTrieLog( private void persistTrieLog(
final BlockHeader blockHeader, final BlockHeader blockHeader,
final Hash worldStateRootHash, final Hash worldStateRootHash,
final TrieLogLayer trieLog, final TrieLog trieLog,
final BonsaiUpdater stateUpdater) { final BonsaiUpdater stateUpdater) {
LOG.atDebug() LOG.atDebug()
.setMessage("Persisting trie log for block hash {} and world state root {}") .setMessage("Persisting trie log for block hash {} and world state root {}")
@ -141,17 +148,17 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
} }
@Override @Override
public Optional<TrieLogLayer> getTrieLogLayer(final Hash blockHash) { public Optional<TrieLog> getTrieLogLayer(final Hash blockHash) {
return rootWorldStateStorage.getTrieLog(blockHash).map(trieLogFactory::deserialize); return rootWorldStateStorage.getTrieLog(blockHash).map(trieLogFactory::deserialize);
} }
@Override @Override
public synchronized long subscribe(final TrieLogAddedObserver sub) { public synchronized long subscribe(final TrieLogObserver sub) {
return trieLogAddedObservers.subscribe(sub); return trieLogObservers.subscribe(sub);
} }
@Override @Override
public synchronized void unsubscribe(final long id) { public synchronized void unsubscribe(final long id) {
trieLogAddedObservers.unsubscribe(id); trieLogObservers.unsubscribe(id);
} }
} }

@ -15,21 +15,13 @@
*/ */
package org.hyperledger.besu.ethereum.bonsai.trielog; package org.hyperledger.besu.ethereum.bonsai.trielog;
public class TrieLogAddedEvent { import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent;
private final TrieLogLayer layer; public record TrieLogAddedEvent(TrieLog layer) implements TrieLogEvent {
public TrieLogAddedEvent(final TrieLogLayer layer) { @Override
this.layer = layer; public Type getType() {
} return Type.ADDED;
public TrieLogLayer getLayer() {
return layer;
}
@FunctionalInterface
interface TrieLogAddedObserver {
void onTrieLogAdded(TrieLogAddedEvent event);
} }
} }

@ -15,80 +15,61 @@
*/ */
package org.hyperledger.besu.ethereum.bonsai.trielog; package org.hyperledger.besu.ethereum.bonsai.trielog;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPInput;
import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.ethereum.rlp.RLPOutput;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.plugin.data.BlockHeader;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogAccumulator;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.Function; import java.util.function.Function;
import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256; import org.apache.tuweni.units.bigints.UInt256;
public class TrieLogFactoryImpl implements TrieLogFactory<TrieLogLayer> { public class TrieLogFactoryImpl implements TrieLogFactory {
@Override @Override
public TrieLogLayer create( public TrieLogLayer create(final TrieLogAccumulator accumulator, final BlockHeader blockHeader) {
final BonsaiWorldStateUpdateAccumulator accumulator, final Hash blockHash) {
TrieLogLayer layer = new TrieLogLayer(); TrieLogLayer layer = new TrieLogLayer();
layer.setBlockHash(blockHash); layer.setBlockHash(blockHeader.getBlockHash());
for (final Map.Entry<Address, BonsaiValue<BonsaiAccount>> updatedAccount : layer.setBlockNumber(blockHeader.getNumber());
accumulator.getAccountsToUpdate().entrySet()) { for (final var updatedAccount : accumulator.getAccountsToUpdate().entrySet()) {
final BonsaiValue<BonsaiAccount> bonsaiValue = updatedAccount.getValue(); final var bonsaiValue = updatedAccount.getValue();
final BonsaiAccount oldValue = bonsaiValue.getPrior(); final var oldAccountValue = bonsaiValue.getPrior();
final StateTrieAccountValue oldAccount = final var newAccountValue = bonsaiValue.getUpdated();
oldValue == null if (oldAccountValue == null && newAccountValue == null) {
? null
: new StateTrieAccountValue(
oldValue.getNonce(),
oldValue.getBalance(),
oldValue.getStorageRoot(),
oldValue.getCodeHash());
final BonsaiAccount newValue = bonsaiValue.getUpdated();
final StateTrieAccountValue newAccount =
newValue == null
? null
: new StateTrieAccountValue(
newValue.getNonce(),
newValue.getBalance(),
newValue.getStorageRoot(),
newValue.getCodeHash());
if (oldValue == null && newValue == null) {
// by default do not persist empty reads of accounts to the trie log // by default do not persist empty reads of accounts to the trie log
continue; continue;
} }
layer.addAccountChange(updatedAccount.getKey(), oldAccount, newAccount); layer.addAccountChange(updatedAccount.getKey(), oldAccountValue, newAccountValue);
} }
for (final Map.Entry<Address, BonsaiValue<Bytes>> updatedCode : for (final var updatedCode : accumulator.getCodeToUpdate().entrySet()) {
accumulator.getCodeToUpdate().entrySet()) {
layer.addCodeChange( layer.addCodeChange(
updatedCode.getKey(), updatedCode.getKey(),
updatedCode.getValue().getPrior(), updatedCode.getValue().getPrior(),
updatedCode.getValue().getUpdated(), updatedCode.getValue().getUpdated(),
blockHash); blockHeader.getBlockHash());
} }
for (final Map.Entry< for (final var updatesStorage : accumulator.getStorageToUpdate().entrySet()) {
Address,
BonsaiWorldStateUpdateAccumulator.StorageConsumingMap<
StorageSlotKey, BonsaiValue<UInt256>>>
updatesStorage : accumulator.getStorageToUpdate().entrySet()) {
final Address address = updatesStorage.getKey(); final Address address = updatesStorage.getKey();
for (final Map.Entry<StorageSlotKey, BonsaiValue<UInt256>> slotUpdate : for (final var slotUpdate : updatesStorage.getValue().entrySet()) {
updatesStorage.getValue().entrySet()) {
var val = slotUpdate.getValue(); var val = slotUpdate.getValue();
if (val.getPrior() == null && val.getUpdated() == null) { if (val.getPrior() == null && val.getUpdated() == null) {
@ -103,52 +84,53 @@ public class TrieLogFactoryImpl implements TrieLogFactory<TrieLogLayer> {
} }
@Override @Override
public byte[] serialize(final TrieLogLayer layer) { public byte[] serialize(final TrieLog layer) {
final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput();
writeTo(layer, rlpLog); writeTo(layer, rlpLog);
return rlpLog.encoded().toArrayUnsafe(); return rlpLog.encoded().toArrayUnsafe();
} }
public static void writeTo(final TrieLogLayer layer, final RLPOutput output) { public static void writeTo(final TrieLog layer, final RLPOutput output) {
layer.freeze(); layer.freeze();
final Set<Address> addresses = new TreeSet<>(); final Set<Address> addresses = new TreeSet<>();
addresses.addAll(layer.getAccounts().keySet()); addresses.addAll(layer.getAccountChanges().keySet());
addresses.addAll(layer.getCode().keySet()); addresses.addAll(layer.getCodeChanges().keySet());
addresses.addAll(layer.getStorage().keySet()); addresses.addAll(layer.getStorageChanges().keySet());
output.startList(); // container output.startList(); // container
output.writeBytes(layer.blockHash); output.writeBytes(layer.getBlockHash());
for (final Address address : addresses) { for (final Address address : addresses) {
output.startList(); // this change output.startList(); // this change
output.writeBytes(address); output.writeBytes(address);
final BonsaiValue<StateTrieAccountValue> accountChange = layer.accounts.get(address); final TrieLog.LogTuple<AccountValue> accountChange = layer.getAccountChanges().get(address);
if (accountChange == null || accountChange.isUnchanged()) { if (accountChange == null || accountChange.isUnchanged()) {
output.writeNull(); output.writeNull();
} else { } else {
accountChange.writeRlp(output, (o, sta) -> sta.writeTo(o)); writeRlp(accountChange, output, (o, sta) -> sta.writeTo(o));
} }
final BonsaiValue<Bytes> codeChange = layer.code.get(address); final TrieLog.LogTuple<Bytes> codeChange = layer.getCodeChanges().get(address);
if (codeChange == null || codeChange.isUnchanged()) { if (codeChange == null || codeChange.isUnchanged()) {
output.writeNull(); output.writeNull();
} else { } else {
codeChange.writeRlp(output, RLPOutput::writeBytes); writeRlp(codeChange, output, RLPOutput::writeBytes);
} }
final Map<StorageSlotKey, BonsaiValue<UInt256>> storageChanges = layer.storage.get(address); final Map<StorageSlotKey, TrieLog.LogTuple<UInt256>> storageChanges =
layer.getStorageChanges().get(address);
if (storageChanges == null) { if (storageChanges == null) {
output.writeNull(); output.writeNull();
} else { } else {
output.startList(); output.startList();
for (final Map.Entry<StorageSlotKey, BonsaiValue<UInt256>> storageChangeEntry : for (final Map.Entry<StorageSlotKey, TrieLog.LogTuple<UInt256>> storageChangeEntry :
storageChanges.entrySet()) { storageChanges.entrySet()) {
output.startList(); output.startList();
// do not write slotKey, it is not used in mainnet bonsai trielogs // do not write slotKey, it is not used in mainnet bonsai trielogs
output.writeBytes(storageChangeEntry.getKey().slotHash()); output.writeBytes(storageChangeEntry.getKey().getSlotHash());
storageChangeEntry.getValue().writeInnerRlp(output, RLPOutput::writeUInt256Scalar); writeInnerRlp(storageChangeEntry.getValue(), output, RLPOutput::writeUInt256Scalar);
output.endList(); output.endList();
} }
output.endList(); output.endList();
@ -242,4 +224,34 @@ public class TrieLogFactoryImpl implements TrieLogFactory<TrieLogLayer> {
.filter(i -> i == 1) .filter(i -> i == 1)
.isPresent(); .isPresent();
} }
public static <T> void writeRlp(
final TrieLog.LogTuple<T> value,
final RLPOutput output,
final BiConsumer<RLPOutput, T> writer) {
output.startList();
writeInnerRlp(value, output, writer);
output.endList();
}
public static <T> void writeInnerRlp(
final TrieLog.LogTuple<T> value,
final RLPOutput output,
final BiConsumer<RLPOutput, T> writer) {
if (value.getPrior() == null) {
output.writeNull();
} else {
writer.accept(output, value.getPrior());
}
if (value.getUpdated() == null) {
output.writeNull();
} else {
writer.accept(output, value.getUpdated());
}
if (!value.isCleared()) {
output.writeNull();
} else {
output.writeInt(1);
}
}
} }

@ -18,18 +18,18 @@ package org.hyperledger.besu.ethereum.bonsai.trielog;
import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Preconditions.checkState;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey; import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.stream.Stream;
import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder;
@ -40,14 +40,16 @@ import org.apache.tuweni.units.bigints.UInt256;
* This class encapsulates the changes that are done to transition one block to the next. This * This class encapsulates the changes that are done to transition one block to the next. This
* includes serialization and deserialization tasks for storing this log to off-memory storage. * includes serialization and deserialization tasks for storing this log to off-memory storage.
* *
* <p>In this particular formulation only the "Leaves" are tracked" Future layers may track patrica * <p>In this particular formulation only the "Leaves" are tracked Future layers may track patricia
* trie changes as well. * trie changes as well.
*/ */
public class TrieLogLayer { @SuppressWarnings("unchecked")
public class TrieLogLayer implements TrieLog {
protected Hash blockHash; protected Hash blockHash;
protected Optional<Long> blockNumber = Optional.empty();
Map<Address, BonsaiValue<StateTrieAccountValue>> getAccounts() { Map<Address, BonsaiValue<AccountValue>> getAccounts() {
return accounts; return accounts;
} }
@ -59,7 +61,7 @@ public class TrieLogLayer {
return storage; return storage;
} }
protected final Map<Address, BonsaiValue<StateTrieAccountValue>> accounts; protected final Map<Address, BonsaiValue<AccountValue>> accounts;
protected final Map<Address, BonsaiValue<Bytes>> code; protected final Map<Address, BonsaiValue<Bytes>> code;
protected final Map<Address, Map<StorageSlotKey, BonsaiValue<UInt256>>> storage; protected final Map<Address, Map<StorageSlotKey, BonsaiValue<UInt256>>> storage;
protected boolean frozen = false; protected boolean frozen = false;
@ -72,10 +74,12 @@ public class TrieLogLayer {
} }
/** Locks the layer so no new changes can be added; */ /** Locks the layer so no new changes can be added; */
void freeze() { @Override
frozen = true; // The code never bothered me anyway public void freeze() {
frozen = true; // The code never bothered me anyway 🥶
} }
@Override
public Hash getBlockHash() { public Hash getBlockHash() {
return blockHash; return blockHash;
} }
@ -86,10 +90,19 @@ public class TrieLogLayer {
return this; return this;
} }
@Override
public Optional<Long> getBlockNumber() {
return blockNumber;
}
public TrieLogLayer setBlockNumber(final long blockNumber) {
checkState(!frozen, "Layer is Frozen");
this.blockNumber = Optional.of(blockNumber);
return this;
}
public TrieLogLayer addAccountChange( public TrieLogLayer addAccountChange(
final Address address, final Address address, final AccountValue oldValue, final AccountValue newValue) {
final StateTrieAccountValue oldValue,
final StateTrieAccountValue newValue) {
checkState(!frozen, "Layer is Frozen"); checkState(!frozen, "Layer is Frozen");
accounts.put(address, new BonsaiValue<>(oldValue, newValue)); accounts.put(address, new BonsaiValue<>(oldValue, newValue));
return this; return this;
@ -114,64 +127,71 @@ public class TrieLogLayer {
return this; return this;
} }
public Stream<Map.Entry<Address, BonsaiValue<StateTrieAccountValue>>> streamAccountChanges() { @Override
return accounts.entrySet().stream(); public Map<Address, BonsaiValue<AccountValue>> getAccountChanges() {
return accounts;
} }
public Stream<Map.Entry<Address, BonsaiValue<Bytes>>> streamCodeChanges() { @Override
return code.entrySet().stream(); public Map<Address, BonsaiValue<Bytes>> getCodeChanges() {
return code;
} }
public Stream<Map.Entry<Address, Map<StorageSlotKey, BonsaiValue<UInt256>>>> @Override
streamStorageChanges() { public Map<Address, Map<StorageSlotKey, BonsaiValue<UInt256>>> getStorageChanges() {
return storage.entrySet().stream(); return storage;
} }
public boolean hasStorageChanges(final Address address) { public boolean hasStorageChanges(final Address address) {
return storage.containsKey(address); return storage.containsKey(address);
} }
public Stream<Map.Entry<StorageSlotKey, BonsaiValue<UInt256>>> streamStorageChanges( @Override
final Address address) { public Map<StorageSlotKey, BonsaiValue<UInt256>> getStorageChanges(final Address address) {
return storage.getOrDefault(address, Map.of()).entrySet().stream(); return storage.getOrDefault(address, Map.of());
} }
@Override
public Optional<Bytes> getPriorCode(final Address address) { public Optional<Bytes> getPriorCode(final Address address) {
return Optional.ofNullable(code.get(address)).map(BonsaiValue::getPrior); return Optional.ofNullable(code.get(address)).map(BonsaiValue::getPrior);
} }
@Override
public Optional<Bytes> getCode(final Address address) { public Optional<Bytes> getCode(final Address address) {
return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated); return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated);
} }
Optional<UInt256> getPriorStorageByStorageSlotKey( @Override
public Optional<UInt256> getPriorStorageByStorageSlotKey(
final Address address, final StorageSlotKey storageSlotKey) { final Address address, final StorageSlotKey storageSlotKey) {
return Optional.ofNullable(storage.get(address)) return Optional.ofNullable(storage.get(address))
.map(i -> i.get(storageSlotKey)) .map(i -> i.get(storageSlotKey))
.map(BonsaiValue::getPrior); .map(BonsaiValue::getPrior);
} }
Optional<UInt256> getStorageByStorageSlotKey( @Override
public Optional<UInt256> getStorageByStorageSlotKey(
final Address address, final StorageSlotKey storageSlotKey) { final Address address, final StorageSlotKey storageSlotKey) {
return Optional.ofNullable(storage.get(address)) return Optional.ofNullable(storage.get(address))
.map(i -> i.get(storageSlotKey)) .map(i -> i.get(storageSlotKey))
.map(BonsaiValue::getUpdated); .map(BonsaiValue::getUpdated);
} }
public Optional<StateTrieAccountValue> getPriorAccount(final Address address) { @Override
public Optional<AccountValue> getPriorAccount(final Address address) {
return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getPrior); return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getPrior);
} }
public Optional<StateTrieAccountValue> getAccount(final Address address) { @Override
public Optional<AccountValue> getAccount(final Address address) {
return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated); return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated);
} }
public String dump() { public String dump() {
final StringBuilder sb = new StringBuilder(); final StringBuilder sb = new StringBuilder();
sb.append("TrieLogLayer{" + "blockHash=").append(blockHash).append(frozen).append('}'); sb.append("TrieLog{" + "blockHash=").append(blockHash).append(frozen).append('}');
sb.append("accounts\n"); sb.append("accounts\n");
for (final Map.Entry<Address, BonsaiValue<StateTrieAccountValue>> account : for (final Map.Entry<Address, BonsaiValue<AccountValue>> account : accounts.entrySet()) {
accounts.entrySet()) {
sb.append(" : ").append(account.getKey()).append("\n"); sb.append(" : ").append(account.getKey()).append("\n");
if (Objects.equals(account.getValue().getPrior(), account.getValue().getUpdated())) { if (Objects.equals(account.getValue().getPrior(), account.getValue().getUpdated())) {
sb.append(" = ").append(account.getValue().getUpdated()).append("\n"); sb.append(" = ").append(account.getValue().getUpdated()).append("\n");

@ -19,6 +19,8 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent;
import java.util.Optional; import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
@ -47,9 +49,9 @@ public interface TrieLogManager {
void reset(); void reset();
Optional<TrieLogLayer> getTrieLogLayer(final Hash blockHash); Optional<? extends TrieLog> getTrieLogLayer(final Hash blockHash);
long subscribe(final TrieLogAddedEvent.TrieLogAddedObserver sub); long subscribe(final TrieLogEvent.TrieLogObserver sub);
void unsubscribe(final long id); void unsubscribe(final long id);
} }

@ -22,6 +22,7 @@ import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyVa
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider;
@ -261,7 +262,7 @@ public class BonsaiWorldState
// for manicured tries and composting, collect branches here (not implemented) // for manicured tries and composting, collect branches here (not implemented)
for (final Map.Entry<StorageSlotKey, BonsaiValue<UInt256>> storageUpdate : for (final Map.Entry<StorageSlotKey, BonsaiValue<UInt256>> storageUpdate :
storageAccountUpdate.getValue().entrySet()) { storageAccountUpdate.getValue().entrySet()) {
final Hash slotHash = storageUpdate.getKey().slotHash(); final Hash slotHash = storageUpdate.getKey().getSlotHash();
final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); final UInt256 updatedStorage = storageUpdate.getValue().getUpdated();
try { try {
if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) { if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
@ -503,7 +504,8 @@ public class BonsaiWorldState
@Override @Override
public UInt256 getStorageValue(final Address address, final UInt256 storageKey) { public UInt256 getStorageValue(final Address address, final UInt256 storageKey) {
return Optional.ofNullable(getStorageValue(address, storageKey)).orElse(UInt256.ZERO); return getStorageValueByStorageSlotKey(address, new StorageSlotKey(storageKey))
.orElse(UInt256.ZERO);
} }
@Override @Override

@ -16,23 +16,26 @@
package org.hyperledger.besu.ethereum.bonsai.worldview; package org.hyperledger.besu.ethereum.bonsai.worldview;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactory;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.Account;
import org.hyperledger.besu.evm.account.EvmAccount; import org.hyperledger.besu.evm.account.EvmAccount;
import org.hyperledger.besu.evm.worldstate.AbstractWorldUpdater; import org.hyperledger.besu.evm.worldstate.AbstractWorldUpdater;
import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount;
import org.hyperledger.besu.evm.worldstate.WrappedEvmAccount; import org.hyperledger.besu.evm.worldstate.WrappedEvmAccount;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogAccumulator;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
@ -55,7 +58,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class BonsaiWorldStateUpdateAccumulator public class BonsaiWorldStateUpdateAccumulator
extends AbstractWorldUpdater<BonsaiWorldView, BonsaiAccount> implements BonsaiWorldView { extends AbstractWorldUpdater<BonsaiWorldView, BonsaiAccount>
implements BonsaiWorldView, TrieLogAccumulator {
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class); LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class);
private final Consumer<BonsaiValue<BonsaiAccount>> accountPreloader; private final Consumer<BonsaiValue<BonsaiAccount>> accountPreloader;
@ -72,7 +76,7 @@ public class BonsaiWorldStateUpdateAccumulator
storageToUpdate = new ConcurrentHashMap<>(); storageToUpdate = new ConcurrentHashMap<>();
// todo plumb me from plugin service: // todo plumb me from plugin service:
TrieLogFactory<TrieLogLayer> trieLogFactory = new TrieLogFactoryImpl(); TrieLogFactory trieLogFactory = new TrieLogFactoryImpl();
private boolean isAccumulatorStateChanged; private boolean isAccumulatorStateChanged;
@ -146,10 +150,12 @@ public class BonsaiWorldStateUpdateAccumulator
return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(newAccount))); return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(newAccount)));
} }
@Override
public Map<Address, BonsaiValue<BonsaiAccount>> getAccountsToUpdate() { public Map<Address, BonsaiValue<BonsaiAccount>> getAccountsToUpdate() {
return accountsToUpdate; return accountsToUpdate;
} }
@Override
public Map<Address, BonsaiValue<Bytes>> getCodeToUpdate() { public Map<Address, BonsaiValue<Bytes>> getCodeToUpdate() {
return codeToUpdate; return codeToUpdate;
} }
@ -158,6 +164,7 @@ public class BonsaiWorldStateUpdateAccumulator
return storageToClear; return storageToClear;
} }
@Override
public Map<Address, StorageConsumingMap<StorageSlotKey, BonsaiValue<UInt256>>> public Map<Address, StorageConsumingMap<StorageSlotKey, BonsaiValue<UInt256>>>
getStorageToUpdate() { getStorageToUpdate() {
return storageToUpdate; return storageToUpdate;
@ -477,7 +484,8 @@ public class BonsaiWorldStateUpdateAccumulator
storageToUpdate.get(address); storageToUpdate.get(address);
if (bonsaiValueStorage != null) { if (bonsaiValueStorage != null) {
// hash the key to match the implied storage interface of hashed slotKey // hash the key to match the implied storage interface of hashed slotKey
bonsaiValueStorage.forEach((key, value) -> results.put(key.slotHash(), value.getUpdated())); bonsaiValueStorage.forEach(
(key, value) -> results.put(key.getSlotHash(), value.getUpdated()));
} }
return results; return results;
} }
@ -492,25 +500,22 @@ public class BonsaiWorldStateUpdateAccumulator
return wrappedWorldView().getWorldStateStorage(); return wrappedWorldView().getWorldStateStorage();
} }
public TrieLogLayer generateTrieLog(final Hash blockHash) { public TrieLog generateTrieLog(final BlockHeader blockHeader) {
return trieLogFactory.create(this, blockHash); return trieLogFactory.create(this, blockHeader);
} }
public void rollForward(final TrieLogLayer layer) { public void rollForward(final TrieLog layer) {
layer layer.getAccountChanges().entrySet().stream()
.streamAccountChanges()
.forEach( .forEach(
entry -> entry ->
rollAccountChange( rollAccountChange(
entry.getKey(), entry.getValue().getPrior(), entry.getValue().getUpdated())); entry.getKey(), entry.getValue().getPrior(), entry.getValue().getUpdated()));
layer layer.getCodeChanges().entrySet().stream()
.streamCodeChanges()
.forEach( .forEach(
entry -> entry ->
rollCodeChange( rollCodeChange(
entry.getKey(), entry.getValue().getPrior(), entry.getValue().getUpdated())); entry.getKey(), entry.getValue().getPrior(), entry.getValue().getUpdated()));
layer layer.getStorageChanges().entrySet().stream()
.streamStorageChanges()
.forEach( .forEach(
entry -> entry ->
entry entry
@ -524,21 +529,18 @@ public class BonsaiWorldStateUpdateAccumulator
value.getUpdated()))); value.getUpdated())));
} }
public void rollBack(final TrieLogLayer layer) { public void rollBack(final TrieLog layer) {
layer layer.getAccountChanges().entrySet().stream()
.streamAccountChanges()
.forEach( .forEach(
entry -> entry ->
rollAccountChange( rollAccountChange(
entry.getKey(), entry.getValue().getUpdated(), entry.getValue().getPrior())); entry.getKey(), entry.getValue().getUpdated(), entry.getValue().getPrior()));
layer layer.getCodeChanges().entrySet().stream()
.streamCodeChanges()
.forEach( .forEach(
entry -> entry ->
rollCodeChange( rollCodeChange(
entry.getKey(), entry.getValue().getUpdated(), entry.getValue().getPrior())); entry.getKey(), entry.getValue().getUpdated(), entry.getValue().getPrior()));
layer layer.getStorageChanges().entrySet().stream()
.streamStorageChanges()
.forEach( .forEach(
entry -> entry ->
entry entry
@ -554,8 +556,8 @@ public class BonsaiWorldStateUpdateAccumulator
private void rollAccountChange( private void rollAccountChange(
final Address address, final Address address,
final StateTrieAccountValue expectedValue, final AccountValue expectedValue,
final StateTrieAccountValue replacementValue) { final AccountValue replacementValue) {
if (Objects.equals(expectedValue, replacementValue)) { if (Objects.equals(expectedValue, replacementValue)) {
// non-change, a cached read. // non-change, a cached read.
return; return;

@ -18,6 +18,7 @@ package org.hyperledger.besu.ethereum.bonsai.worldview;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evm.worldstate.WorldUpdater;

@ -1,62 +0,0 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.ethereum.bonsai.worldview;
import org.hyperledger.besu.datatypes.Hash;
import java.util.Objects;
import java.util.Optional;
import org.apache.tuweni.units.bigints.UInt256;
import org.jetbrains.annotations.NotNull;
public record StorageSlotKey(Hash slotHash, Optional<UInt256> slotKey)
implements Comparable<org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey> {
public StorageSlotKey(final UInt256 slotKey) {
this(Hash.hash(slotKey), Optional.of(slotKey));
}
@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
StorageSlotKey that = (StorageSlotKey) o;
return Objects.equals(slotHash, that.slotHash);
}
@Override
public int hashCode() {
return Objects.hash(slotHash.hashCode());
}
@Override
public String toString() {
return String.format(
"StorageSlotKey{slotHash=%s, slotKey=%s}",
slotHash, slotKey.map(UInt256::toString).orElse("null"));
}
@Override
public int compareTo(
@NotNull final org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey other) {
return this.slotHash.compareTo(other.slotHash);
}
}

@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.worldstate;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPInput;
@ -26,12 +27,12 @@ import java.util.Objects;
import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.bytes.Bytes32;
/** Represents the raw values associated with an account in the world state trie. */ /** Represents the raw values associated with an account in the world state trie. */
public class StateTrieAccountValue { public class StateTrieAccountValue implements AccountValue {
private final long nonce; protected final long nonce;
private final Wei balance; protected final Wei balance;
private final Hash storageRoot; protected final Hash storageRoot;
private final Hash codeHash; protected final Hash codeHash;
public StateTrieAccountValue( public StateTrieAccountValue(
final long nonce, final Wei balance, final Hash storageRoot, final Hash codeHash) { final long nonce, final Wei balance, final Hash storageRoot, final Hash codeHash) {
@ -49,6 +50,7 @@ public class StateTrieAccountValue {
* *
* @return the account nonce. * @return the account nonce.
*/ */
@Override
public long getNonce() { public long getNonce() {
return nonce; return nonce;
} }
@ -58,6 +60,7 @@ public class StateTrieAccountValue {
* *
* @return the balance, in Wei, of the account. * @return the balance, in Wei, of the account.
*/ */
@Override
public Wei getBalance() { public Wei getBalance() {
return balance; return balance;
} }
@ -67,6 +70,7 @@ public class StateTrieAccountValue {
* *
* @return the hash of the root node of the storage trie. * @return the hash of the root node of the storage trie.
*/ */
@Override
public Hash getStorageRoot() { public Hash getStorageRoot() {
return storageRoot; return storageRoot;
} }
@ -76,6 +80,7 @@ public class StateTrieAccountValue {
* *
* @return the hash of the account code (which may be {@link Hash#EMPTY}). * @return the hash of the account code (which may be {@link Hash#EMPTY}).
*/ */
@Override
public Hash getCodeHash() { public Hash getCodeHash() {
return codeHash; return codeHash;
} }
@ -96,6 +101,7 @@ public class StateTrieAccountValue {
return Objects.hash(nonce, balance, storageRoot, codeHash); return Objects.hash(nonce, balance, storageRoot, codeHash);
} }
@Override
public void writeTo(final RLPOutput out) { public void writeTo(final RLPOutput out) {
out.startList(); out.startList();

@ -73,7 +73,8 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider {
inMemoryKeyValueStorageProvider, inMemoryKeyValueStorageProvider,
blockchain, blockchain,
cachedMerkleTrieLoader, cachedMerkleTrieLoader,
new NoOpMetricsSystem()); new NoOpMetricsSystem(),
null);
} }
public static MutableWorldState createInMemoryWorldState() { public static MutableWorldState createInMemoryWorldState() {

@ -89,7 +89,7 @@ public class BlockImportExceptionHandlingTest {
// do we need to also test with a DefaultWorldStateArchive? // do we need to also test with a DefaultWorldStateArchive?
spy( spy(
new BonsaiWorldStateProvider( new BonsaiWorldStateProvider(
storageProvider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem())); storageProvider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null));
private final BonsaiWorldState persisted = private final BonsaiWorldState persisted =
spy( spy(

@ -139,7 +139,8 @@ public abstract class AbstractIsolationTests {
blockchain, blockchain,
Optional.of(16L), Optional.of(16L),
new CachedMerkleTrieLoader(new NoOpMetricsSystem()), new CachedMerkleTrieLoader(new NoOpMetricsSystem()),
new NoOpMetricsSystem()); new NoOpMetricsSystem(),
null);
var ws = archive.getMutable(); var ws = archive.getMutable();
genesisState.writeStateTo(ws); genesisState.writeStateTo(ws);
protocolContext = new ProtocolContext(blockchain, archive, null); protocolContext = new ProtocolContext(blockchain, archive, null);

@ -20,6 +20,7 @@ import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyVa
import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never; import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy; import static org.mockito.Mockito.spy;
@ -105,7 +106,8 @@ public class BonsaiWorldStateArchiveTest {
blockchain, blockchain,
Optional.of(512L), Optional.of(512L),
new CachedMerkleTrieLoader(new NoOpMetricsSystem()), new CachedMerkleTrieLoader(new NoOpMetricsSystem()),
new NoOpMetricsSystem()); new NoOpMetricsSystem(),
null);
final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader();
final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); final BlockHeader chainHead = blockBuilder.number(512).buildHeader();
when(blockchain.getChainHeadHeader()).thenReturn(chainHead); when(blockchain.getChainHeadHeader()).thenReturn(chainHead);
@ -141,7 +143,9 @@ public class BonsaiWorldStateArchiveTest {
public void testGetMutableWithStorageInconsistencyRollbackTheState() { public void testGetMutableWithStorageInconsistencyRollbackTheState() {
when(keyValueStorage.startTransaction()).thenReturn(mock(KeyValueStorageTransaction.class)); when(keyValueStorage.startTransaction()).thenReturn(mock(KeyValueStorageTransaction.class));
when(trieLogManager.getTrieLogLayer(any())).thenReturn(Optional.of(mock(TrieLogLayer.class))); doAnswer(__ -> Optional.of(mock(TrieLogLayer.class)))
.when(trieLogManager)
.getTrieLogLayer(any(Hash.class));
var worldStateStorage = var worldStateStorage =
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem());
@ -201,7 +205,9 @@ public class BonsaiWorldStateArchiveTest {
final BlockHeader blockHeaderChainB = final BlockHeader blockHeaderChainB =
blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader(); blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader();
when(trieLogManager.getTrieLogLayer(any())).thenReturn(Optional.of(mock(TrieLogLayer.class))); doAnswer(__ -> Optional.of(mock(TrieLogLayer.class)))
.when(trieLogManager)
.getTrieLogLayer(any(Hash.class));
var worldStateStorage = var worldStateStorage =
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem());
@ -242,8 +248,9 @@ public class BonsaiWorldStateArchiveTest {
final BlockHeader blockHeaderChainB = final BlockHeader blockHeaderChainB =
blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader(); blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader();
when(trieLogManager.getTrieLogLayer(any(Hash.class))) doAnswer(__ -> Optional.of(mock(TrieLogLayer.class)))
.thenReturn(Optional.of(mock(TrieLogLayer.class))); .when(trieLogManager)
.getTrieLogLayer(any(Hash.class));
bonsaiWorldStateArchive = bonsaiWorldStateArchive =
spy( spy(

@ -24,8 +24,8 @@ import static org.mockito.Mockito.verify;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.core.TrieGenerator;
import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLP;

@ -19,9 +19,9 @@ import static org.assertj.core.api.Assertions.assertThat;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.core.TrieGenerator;
import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLP;

@ -126,7 +126,7 @@ public class LogRollingTests {
new CachedMerkleTrieLoader(new NoOpMetricsSystem()); new CachedMerkleTrieLoader(new NoOpMetricsSystem());
archive = archive =
new BonsaiWorldStateProvider( new BonsaiWorldStateProvider(
provider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem()); provider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null);
accountStorage = accountStorage =
(InMemoryKeyValueStorage) (InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
@ -149,7 +149,11 @@ public class LogRollingTests {
new CachedMerkleTrieLoader(new NoOpMetricsSystem()); new CachedMerkleTrieLoader(new NoOpMetricsSystem());
secondArchive = secondArchive =
new BonsaiWorldStateProvider( new BonsaiWorldStateProvider(
secondProvider, blockchain, secondOptimizedMerkleTrieLoader, new NoOpMetricsSystem()); secondProvider,
blockchain,
secondOptimizedMerkleTrieLoader,
new NoOpMetricsSystem(),
null);
secondAccountStorage = secondAccountStorage =
(InMemoryKeyValueStorage) (InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier( secondProvider.getStorageBySegmentIdentifier(

@ -49,7 +49,7 @@ public class RollingImport {
new CachedMerkleTrieLoader(new NoOpMetricsSystem()); new CachedMerkleTrieLoader(new NoOpMetricsSystem());
final BonsaiWorldStateProvider archive = final BonsaiWorldStateProvider archive =
new BonsaiWorldStateProvider( new BonsaiWorldStateProvider(
provider, null, cachedMerkleTrieLoader, new NoOpMetricsSystem()); provider, null, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null);
final InMemoryKeyValueStorage accountStorage = final InMemoryKeyValueStorage accountStorage =
(InMemoryKeyValueStorage) (InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);

@ -19,13 +19,15 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture;
import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256; import org.apache.tuweni.units.bigints.UInt256;
@ -36,31 +38,37 @@ import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class) @RunWith(MockitoJUnitRunner.class)
public class TrieLogFactoryTests { public class TrieLogFactoryTests {
BlockchainSetupUtil setup = BlockchainSetupUtil.forTesting(DataStorageFormat.BONSAI); final BlockchainSetupUtil setup = BlockchainSetupUtil.forTesting(DataStorageFormat.BONSAI);
final Address accountFixture = Address.fromHexString("0xdeadbeef");
final BlockHeader headerFixture =
new BlockHeaderTestFixture()
.parentHash(setup.getGenesisState().getBlock().getHash())
.coinbase(Address.ZERO)
.buildHeader();
final TrieLogLayer trieLogFixture =
new TrieLogLayer()
.setBlockHash(headerFixture.getBlockHash())
.addAccountChange(
accountFixture,
null,
new StateTrieAccountValue(0, Wei.fromEth(1), Hash.EMPTY, Hash.EMPTY))
.addCodeChange(
Address.ZERO,
null,
Bytes.fromHexString("0xfeeddeadbeef"),
headerFixture.getBlockHash())
.addStorageChange(Address.ZERO, new StorageSlotKey(UInt256.ZERO), null, UInt256.ONE);
@Test @Test
public void testSerializeDeserializeAreEqual() { public void testSerializeDeserializeAreEqual() {
BlockHeader header =
new BlockHeaderTestFixture()
.parentHash(setup.getGenesisState().getBlock().getHash())
.coinbase(Address.ZERO)
.buildHeader();
TrieLogLayer fixture =
new TrieLogLayer()
.setBlockHash(header.getBlockHash())
.addAccountChange(
Address.fromHexString("0xdeadbeef"),
null,
new StateTrieAccountValue(0, Wei.fromEth(1), Hash.EMPTY, Hash.EMPTY))
.addCodeChange(
Address.ZERO, null, Bytes.fromHexString("0xdeadbeef"), header.getBlockHash())
.addStorageChange(Address.ZERO, new StorageSlotKey(UInt256.ZERO), null, UInt256.ONE);
TrieLogFactory<TrieLogLayer> factory = new TrieLogFactoryImpl(); TrieLogFactory factory = new TrieLogFactoryImpl();
byte[] rlp = factory.serialize(fixture); byte[] rlp = factory.serialize(trieLogFixture);
TrieLogLayer layer = factory.deserialize(rlp); TrieLog layer = factory.deserialize(rlp);
assertThat(layer).isEqualTo(fixture); assertThat(layer).isEqualTo(trieLogFixture);
} }
} }

@ -15,10 +15,11 @@
*/ */
package org.hyperledger.besu.ethereum.bonsai.trielog; package org.hyperledger.besu.ethereum.bonsai.trielog;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import java.util.Optional; import java.util.Optional;
@ -66,11 +67,11 @@ public class TrieLogLayerTests {
Assertions.assertThat(trieLogLayer).isEqualTo(otherTrieLogLayer); Assertions.assertThat(trieLogLayer).isEqualTo(otherTrieLogLayer);
Optional<StateTrieAccountValue> priorAccount = trieLogLayer.getPriorAccount(address); Optional<AccountValue> priorAccount = trieLogLayer.getPriorAccount(address);
Assertions.assertThat(priorAccount).isPresent(); Assertions.assertThat(priorAccount).isPresent();
Assertions.assertThat(priorAccount.get()).isEqualTo(oldValue); Assertions.assertThat(priorAccount.get()).isEqualTo(oldValue);
Optional<StateTrieAccountValue> updatedAccount = trieLogLayer.getAccount(address); Optional<AccountValue> updatedAccount = trieLogLayer.getAccount(address);
Assertions.assertThat(updatedAccount).isPresent(); Assertions.assertThat(updatedAccount).isPresent();
Assertions.assertThat(updatedAccount.get()).isEqualTo(newValue); Assertions.assertThat(updatedAccount.get()).isEqualTo(newValue);
} }

@ -59,7 +59,12 @@ public class TrieLogManagerTests {
public void setup() { public void setup() {
trieLogManager = trieLogManager =
new CachedWorldStorageManager( new CachedWorldStorageManager(
archive, blockchain, bonsaiWorldStateKeyValueStorage, new NoOpMetricsSystem(), 512); archive,
blockchain,
bonsaiWorldStateKeyValueStorage,
new NoOpMetricsSystem(),
512,
null);
} }
@Test @Test

@ -69,7 +69,7 @@ Calculated : ${currentHash}
tasks.register('checkAPIChanges', FileStateChecker) { tasks.register('checkAPIChanges', FileStateChecker) {
description = "Checks that the API for the Plugin-API project does not change without deliberate thought" description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
files = sourceSets.main.allJava.files files = sourceSets.main.allJava.files
knownHash = 'aqSCy4h1L4eGXKojEW09GiRoyl1YGN9hLf5B2wf0oxg=' knownHash = '3/qsZ9+jA10YbctpPtQ5Rn7cvYHwKRWBv6jXa+7WQMY='
} }
check.dependsOn('checkAPIChanges') check.dependsOn('checkAPIChanges')

@ -21,6 +21,15 @@ import java.util.Optional;
/** Allows plugins to access Besu services. */ /** Allows plugins to access Besu services. */
public interface BesuContext { public interface BesuContext {
/**
* Add service.
*
* @param <T> the type parameter
* @param serviceType the service type
* @param service the service
*/
<T extends BesuService> void addService(final Class<T> serviceType, final T service);
/** /**
* Get the requested service, if it is available. There are a number of reasons that a service may * Get the requested service, if it is available. There are a number of reasons that a service may
* not be available: * not be available:

@ -0,0 +1,58 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.plugin.services;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory;
import org.hyperledger.besu.plugin.services.trielogs.TrieLogProvider;
import java.util.List;
/**
* A service interface for registering observers for trie log events.
*
* <p>Implementations must be thread-safe.
*/
public interface TrieLogService extends BesuService {
/**
* Provides list of observers to configure for trie log events.
*
* @return the list of observers to configure
*/
List<TrieLogEvent.TrieLogObserver> getObservers();
/**
* Provide a TrieLogFactory implementation to use for serializing and deserializing TrieLogs.
*
* @return the TrieLogFactory implementation
*/
TrieLogFactory getTrieLogFactory();
/**
* Configure a TrieLogProvider implementation to use for retrieving stored TrieLogs.
*
* @param provider the TrieLogProvider implementation
*/
void configureTrieLogProvider(TrieLogProvider provider);
/**
* Retrieve the configured TrieLogProvider implementation.
*
* @return the TrieLogProvider implementation
*/
TrieLogProvider getTrieLogProvider();
}

@ -0,0 +1,176 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256;
/**
* An interface for interacting with TrieLog objects, which represent changes to accounts, code, and
* storage in a blockchain state trie.
*/
public interface TrieLog {
/**
* Gets the block hash associated with this TrieLog.
*
* @return the block hash
*/
Hash getBlockHash();
/**
* Gets the block number associated with this TrieLog, if available.
*
* @return an Optional containing the block number if available, otherwise an empty Optional
*/
Optional<Long> getBlockNumber();
/** Freezes the TrieLog to prevent further modifications. */
void freeze();
/**
* Gets a map of addresses to their account value changes.
*
* @param <U> the type of LogTuple representing the account value changes
* @return a map of addresses to their account value changes
*/
<U extends LogTuple<AccountValue>> Map<Address, U> getAccountChanges();
/**
* Gets a map of addresses to their code changes.
*
* @param <U> the type of LogTuple representing the code changes
* @return a map of addresses to their code changes
*/
<U extends LogTuple<Bytes>> Map<Address, U> getCodeChanges();
/**
* Gets a map of addresses to their storage changes.
*
* @param <U> the type of LogTuple representing the storage changes
* @return a map of addresses to their storage changes
*/
<U extends LogTuple<UInt256>> Map<Address, Map<StorageSlotKey, U>> getStorageChanges();
/**
* Gets the storage changes for a specific address.
*
* @param address the address to get the storage changes for
* @param <U> the type of LogTuple representing the storage changes
* @return a map of storage slot keys to their changes
*/
<U extends LogTuple<UInt256>> Map<StorageSlotKey, U> getStorageChanges(final Address address);
/**
* Gets the prior code for a specific address, if available.
*
* @param address the address to get the prior code for
* @return an Optional containing the prior code if available, otherwise an empty Optional
*/
Optional<Bytes> getPriorCode(final Address address);
/**
* Gets the code for a specific address, if available.
*
* @param address the address to get the code for
* @return an Optional containing the code if available, otherwise an empty Optional
*/
Optional<Bytes> getCode(final Address address);
/**
* Gets the prior storage value for a specific address and storage slot key, if available.
*
* @param address the address to get the prior storage value for
* @param storageSlotKey the storage slot key to get the prior storage value for
* @return an Optional containing the prior storage value if available, otherwise an empty
* Optional
*/
Optional<UInt256> getPriorStorageByStorageSlotKey(
final Address address, final StorageSlotKey storageSlotKey);
/**
* Gets the storage value for a specific address and storage slot key, if available.
*
* @param address the address to get the storage value for
* @param storageSlotKey the storage slot key to get the storage value for
* @return an Optional containing the storage value if available, otherwise an empty Optional
*/
Optional<UInt256> getStorageByStorageSlotKey(
final Address address, final StorageSlotKey storageSlotKey);
/**
* Gets the prior account value for a specific address, if available.
*
* @param address the address to get the prior account value for
* @return an Optional containing the prior account value if available, otherwise an empty
* Optional
*/
Optional<? extends AccountValue> getPriorAccount(final Address address);
/**
* Gets the account value for a specific address, if available.
*
* @param address the address to get the account value for
* @return an Optional containing the account value if available, otherwise an empty Optional
*/
Optional<? extends AccountValue> getAccount(final Address address);
/**
* An interface representing a tuple of prior and updated values for a specific type T in a log.
* The interface also provides methods to check if the values are unchanged or cleared.
*
* @param <T> the type of values stored in the log tuple
*/
public interface LogTuple<T> {
/**
* Gets the prior value of the tuple.
*
* @return the prior value of type T
*/
T getPrior();
/**
* Gets the updated value of the tuple.
*
* @return the updated value of type T
*/
T getUpdated();
/**
* Checks if the prior and updated values are equal.
*
* @return true if the prior and updated values are equal, false otherwise
*/
default boolean isUnchanged() {
return Objects.equals(getUpdated(), getPrior());
}
/**
* Checks if the updated value represents a cleared state.
*
* @return true if the updated value is cleared, false otherwise
*/
boolean isCleared();
}
}

@ -0,0 +1,50 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
import org.hyperledger.besu.datatypes.AccountValue;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import java.util.Map;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256;
/** Accumulator interface tor provding trie updates for creating TrieLogs. */
public interface TrieLogAccumulator {
/**
* Returns the state trie accounts which have been updated.
*
* @return the accounts to update
*/
Map<Address, ? extends TrieLog.LogTuple<? extends AccountValue>> getAccountsToUpdate();
/**
* Returns code which has been updated.
*
* @return the code to update
*/
Map<Address, ? extends TrieLog.LogTuple<Bytes>> getCodeToUpdate();
/**
* Returns storage which has been updated.
*
* @return the storage to update
*/
Map<Address, ? extends Map<StorageSlotKey, ? extends TrieLog.LogTuple<UInt256>>>
getStorageToUpdate();
}

@ -0,0 +1,49 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
/** A TrieLog event. */
public interface TrieLogEvent {
/** The type of the event. */
enum Type {
/** TrieLog added event type */
ADDED
}
/**
* The type of the event.
*
* @return the type of the event
*/
TrieLogEvent.Type getType();
/**
* The TrieLog layer.
*
* @return the TrieLog layer
*/
TrieLog layer();
/** Observer interface for TrieLog events. */
interface TrieLogObserver {
/**
* Called when a TrieLog is added.
*
* @param event the TrieLog event
*/
void onTrieLogAdded(TrieLogEvent event);
}
}

@ -0,0 +1,46 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
import org.hyperledger.besu.plugin.data.BlockHeader;
/** Interface for serializing and deserializing {@link TrieLog} objects. */
public interface TrieLogFactory {
/**
* Creates a new TrieLog object.
*
* @param accumulator the accumulator
* @param blockHeader the block header
* @return a new TrieLog object
*/
TrieLog create(TrieLogAccumulator accumulator, BlockHeader blockHeader);
/**
* Deserializes a TrieLog object.
*
* @param bytes the serialized TrieLog
* @return the deserialized TrieLog
*/
TrieLog deserialize(final byte[] bytes);
/**
* Serializes a TrieLog object.
*
* @param layer the TrieLog
* @return the serialized TrieLog
*/
byte[] serialize(final TrieLog layer);
}

@ -0,0 +1,61 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.plugin.services.trielogs;
import org.hyperledger.besu.datatypes.Hash;
import java.util.List;
import java.util.Optional;
/** Trielog provider interface for a given block hash. */
public interface TrieLogProvider {
/**
* Returns the TrieLog layer for the given block hash.
*
* @param blockHash the block hash
* @return the TrieLog layer for the given block hash
* @param <T> the type of the TrieLog
*/
<T extends TrieLog.LogTuple<?>> Optional<TrieLog> getTrieLogLayer(final Hash blockHash);
/**
* Returns the TrieLog layer for the given block number.
*
* @param blockNumber the block hash
* @return the TrieLog layer for the given block hash
* @param <T> the type of the TrieLog
*/
<T extends TrieLog.LogTuple<?>> Optional<TrieLog> getTrieLogLayer(final long blockNumber);
/**
* Returns the TrieLog layers for the given block number range.
*
* @param fromBlockNumber the from block number
* @param toBlockNumber the to block number
* @return the TrieLog layers for the given block number range
* @param <T> the type of the TrieLog
*/
<T extends TrieLog.LogTuple<?>> List<TrieLogRangeTuple> getTrieLogsByRange(
long fromBlockNumber, long toBlockNumber);
/**
* Block and TrieLog layer composition, used for returning a range of TrieLog layers.
*
* @param blockHash the block hash
* @param blockNumber the block number
* @param trieLog the associated TrieLog layer
*/
record TrieLogRangeTuple(Hash blockHash, long blockNumber, TrieLog trieLog) {}
}
Loading…
Cancel
Save