Data storage option refactoring (#7807)

Renaming and refactoring the classes common to both Bonsai and Verkle to facilitate the future integration of Verkle.

---------

Signed-off-by: Karim Taam <karim.t2am@gmail.com>
pull/7800/head
Karim Taam 4 days ago committed by GitHub
parent a27dccd675
commit a3b886368f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 2
      acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java
  2. 58
      besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
  3. 6
      besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java
  4. 248
      besu/src/main/java/org/hyperledger/besu/cli/options/DataStorageOptions.java
  5. 121
      besu/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java
  6. 217
      besu/src/main/java/org/hyperledger/besu/cli/options/storage/DiffBasedSubStorageOptions.java
  7. 75
      besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java
  8. 7
      besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java
  9. 21
      besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
  10. 23
      besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java
  11. 2
      besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java
  12. 2
      besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java
  13. 1
      besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java
  14. 64
      besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java
  15. 79
      besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java
  16. 13
      besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommandTest.java
  17. 26
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java
  18. 173
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFlatDbStrategy.java
  19. 88
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFlatDbStrategyProvider.java
  20. 5
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiFullFlatDbStrategy.java
  21. 5
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/BonsaiPartialFlatDbStrategy.java
  22. 144
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java
  23. 117
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java
  24. 69
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java
  25. 95
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DiffBasedSubStorageConfiguration.java
  26. 12
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java
  27. 58
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/BonsaiFlatDbStrategyProviderTest.java
  28. 5
      ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerTest.java

@ -17,8 +17,8 @@ package org.hyperledger.besu.tests.acceptance.dsl.node;
import static com.google.common.base.Preconditions.checkState;
import static java.nio.charset.StandardCharsets.UTF_8;
import org.hyperledger.besu.cli.options.DataStorageOptions;
import org.hyperledger.besu.cli.options.TransactionPoolOptions;
import org.hyperledger.besu.cli.options.storage.DataStorageOptions;
import org.hyperledger.besu.cli.options.unstable.NetworkingOptions;
import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration;
import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;

@ -41,7 +41,6 @@ import org.hyperledger.besu.cli.converter.MetricCategoryConverter;
import org.hyperledger.besu.cli.custom.JsonRPCAllowlistHostsProperty;
import org.hyperledger.besu.cli.error.BesuExecutionExceptionHandler;
import org.hyperledger.besu.cli.error.BesuParameterExceptionHandler;
import org.hyperledger.besu.cli.options.DataStorageOptions;
import org.hyperledger.besu.cli.options.MiningOptions;
import org.hyperledger.besu.cli.options.TransactionPoolOptions;
import org.hyperledger.besu.cli.options.stable.ApiConfigurationOptions;
@ -57,6 +56,8 @@ import org.hyperledger.besu.cli.options.stable.P2PDiscoveryOptions;
import org.hyperledger.besu.cli.options.stable.PermissionsOptions;
import org.hyperledger.besu.cli.options.stable.PluginsConfigurationOptions;
import org.hyperledger.besu.cli.options.stable.RpcWebsocketOptions;
import org.hyperledger.besu.cli.options.storage.DataStorageOptions;
import org.hyperledger.besu.cli.options.storage.DiffBasedSubStorageOptions;
import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions;
import org.hyperledger.besu.cli.options.unstable.DnsOptions;
import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
@ -139,7 +140,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder;
import org.hyperledger.besu.ethereum.transaction.TransactionSimulator;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract;
import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract;
import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract;
@ -1643,7 +1646,11 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
CommandLineUtils.failIfOptionDoesntMeetRequirement(
commandLine,
"--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true",
dataStorageOptions.toDomainObject().getUnstable().getBonsaiFullFlatDbEnabled(),
dataStorageOptions
.toDomainObject()
.getDiffBasedSubStorageConfiguration()
.getUnstable()
.getFullFlatDbEnabled(),
asList(
"--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
"--Xsnapsync-synchronizer-flat-slot-healed-count-per-request"));
@ -1775,7 +1782,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
.withMiningParameters(miningParametersSupplier.get())
.withJsonRpcHttpOptions(jsonRpcHttpOptions);
final KeyValueStorageProvider storageProvider = keyValueStorageProvider(keyValueStorageName);
return controllerBuilder
BesuControllerBuilder besuControllerBuilder =
controllerBuilder
.fromEthNetworkConfig(updateNetworkConfig(network), getDefaultSyncModeIfNotSet())
.synchronizerConfiguration(buildSyncConfig())
.ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject())
@ -1790,8 +1798,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
.privacyParameters(privacyParameters())
.clock(Clock.systemUTC())
.isRevertReasonEnabled(isRevertReasonEnabled)
.isParallelTxProcessingEnabled(
dataStorageConfiguration.getUnstable().isParallelTxProcessingEnabled())
.storageProvider(storageProvider)
.gasLimitCalculator(
miningParametersSupplier.get().getTargetGasLimit().isPresent()
@ -1807,6 +1813,15 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
.cacheLastBlocks(numberOfblocksToCache)
.genesisStateHashCacheEnabled(genesisStateHashCacheEnabled)
.besuComponent(besuComponent);
if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat())) {
final DiffBasedSubStorageConfiguration subStorageConfiguration =
getDataStorageConfiguration().getDiffBasedSubStorageConfiguration();
if (subStorageConfiguration.getLimitTrieLogsEnabled()) {
besuControllerBuilder.isParallelTxProcessingEnabled(
subStorageConfiguration.getUnstable().isParallelTxProcessingEnabled());
}
}
return besuControllerBuilder;
}
private JsonRpcConfiguration createEngineJsonRpcConfiguration() {
@ -2126,30 +2141,35 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
}
if (SyncMode.FULL.equals(getDefaultSyncModeIfNotSet())
&& DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())
&& dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()) {
&& DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) {
final DiffBasedSubStorageConfiguration diffBasedSubStorageConfiguration =
dataStorageConfiguration.getDiffBasedSubStorageConfiguration();
if (diffBasedSubStorageConfiguration.getLimitTrieLogsEnabled()) {
if (CommandLineUtils.isOptionSet(
commandLine, DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED)) {
commandLine, DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED)) {
throw new ParameterException(
commandLine,
String.format(
"Cannot enable %s with --sync-mode=%s and --data-storage-format=%s. You must set %s or use a different sync-mode",
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED,
DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED,
SyncMode.FULL,
DataStorageFormat.BONSAI,
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false"));
DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED + "=false"));
}
dataStorageConfiguration =
ImmutableDataStorageConfiguration.copyOf(dataStorageConfiguration)
.withBonsaiLimitTrieLogsEnabled(false);
.withDiffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.copyOf(
dataStorageConfiguration.getDiffBasedSubStorageConfiguration())
.withLimitTrieLogsEnabled(false));
logger.warn(
"Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.",
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false",
DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED + "=false",
SyncMode.FULL,
DataStorageFormat.BONSAI);
}
}
return dataStorageConfiguration;
}
@ -2715,12 +2735,14 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
builder.setHighSpecEnabled();
}
if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat())
&& getDataStorageConfiguration().getBonsaiLimitTrieLogsEnabled()) {
if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat())) {
final DiffBasedSubStorageConfiguration subStorageConfiguration =
getDataStorageConfiguration().getDiffBasedSubStorageConfiguration();
if (subStorageConfiguration.getLimitTrieLogsEnabled()) {
builder.setLimitTrieLogsEnabled();
builder.setTrieLogRetentionLimit(getDataStorageConfiguration().getBonsaiMaxLayersToLoad());
builder.setTrieLogsPruningWindowSize(
getDataStorageConfiguration().getBonsaiTrieLogPruningWindowSize());
builder.setTrieLogRetentionLimit(subStorageConfiguration.getMaxLayersToLoad());
builder.setTrieLogsPruningWindowSize(subStorageConfiguration.getTrieLogPruningWindowSize());
}
}
builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled());

@ -53,7 +53,7 @@ public class ConfigurationOverviewBuilder {
private Collection<String> engineApis;
private String engineJwtFilePath;
private boolean isHighSpec = false;
private boolean isBonsaiLimitTrieLogsEnabled = false;
private boolean isLimitTrieLogsEnabled = false;
private long trieLogRetentionLimit = 0;
private Integer trieLogsPruningWindowSize = null;
private boolean isSnapServerEnabled = false;
@ -220,7 +220,7 @@ public class ConfigurationOverviewBuilder {
* @return the builder
*/
public ConfigurationOverviewBuilder setLimitTrieLogsEnabled() {
isBonsaiLimitTrieLogsEnabled = true;
isLimitTrieLogsEnabled = true;
return this;
}
@ -389,7 +389,7 @@ public class ConfigurationOverviewBuilder {
lines.add("Experimental Snap Sync for BFT enabled");
}
if (isBonsaiLimitTrieLogsEnabled) {
if (isLimitTrieLogsEnabled) {
final StringBuilder trieLogPruningString = new StringBuilder();
trieLogPruningString
.append("Limit trie logs enabled: retention: ")

@ -1,248 +0,0 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.cli.options;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
import org.hyperledger.besu.cli.util.CommandLineUtils;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import java.util.List;
import java.util.Locale;
import org.apache.commons.lang3.StringUtils;
import picocli.CommandLine;
import picocli.CommandLine.Option;
/** The Data storage CLI options. */
public class DataStorageOptions implements CLIOptions<DataStorageConfiguration> {
private static final String DATA_STORAGE_FORMAT = "--data-storage-format";
/** The maximum number of historical layers to load. */
public static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD =
"--bonsai-historical-block-limit";
// Use Bonsai DB
@Option(
names = {DATA_STORAGE_FORMAT},
description =
"Format to store trie data in. Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).",
arity = "1")
private DataStorageFormat dataStorageFormat = DataStorageFormat.BONSAI;
@Option(
names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"},
paramLabel = "<LONG>",
description =
"Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using "
+ BONSAI_LIMIT_TRIE_LOGS_ENABLED
+ " it will also be used as the number of layers of trie logs to retain.",
arity = "1")
private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
/** The bonsai limit trie logs enabled option name */
public static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--bonsai-limit-trie-logs-enabled";
/** The bonsai trie logs pruning window size. */
public static final String BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE =
"--bonsai-trie-logs-pruning-window-size";
// TODO --Xbonsai-limit-trie-logs-enabled and --Xbonsai-trie-log-pruning-enabled are deprecated,
// remove in a future release
@SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed")
@CommandLine.Option(
names = {
BONSAI_LIMIT_TRIE_LOGS_ENABLED,
"--Xbonsai-limit-trie-logs-enabled", // deprecated
"--Xbonsai-trie-log-pruning-enabled" // deprecated
},
fallbackValue = "true",
description = "Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})")
private Boolean bonsaiLimitTrieLogsEnabled = DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
// TODO --Xbonsai-trie-logs-pruning-window-size is deprecated, remove in a future release
@SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed")
@CommandLine.Option(
names = {
BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE,
"--Xbonsai-trie-logs-pruning-window-size" // deprecated
},
description =
"The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
private Integer bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
@Option(
names = "--receipt-compaction-enabled",
description = "Enables compact storing of receipts (default: ${DEFAULT-VALUE})",
fallbackValue = "true")
private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED;
@CommandLine.ArgGroup(validate = false)
private final DataStorageOptions.Unstable unstableOptions = new Unstable();
/** Default Constructor. */
DataStorageOptions() {}
/** The unstable options for data storage. */
public static class Unstable {
// TODO: --Xsnapsync-synchronizer-flat-db-healing-enabled is deprecated, remove it in a future
// release
@CommandLine.Option(
hidden = true,
names = {
"--Xbonsai-full-flat-db-enabled",
"--Xsnapsync-synchronizer-flat-db-healing-enabled"
},
arity = "1",
description = "Enables bonsai full flat database strategy. (default: ${DEFAULT-VALUE})")
private Boolean bonsaiFullFlatDbEnabled = DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
@CommandLine.Option(
hidden = true,
names = {"--Xbonsai-code-using-code-hash-enabled"},
arity = "1",
description =
"Enables code storage using code hash instead of by account hash. (default: ${DEFAULT-VALUE})")
private boolean bonsaiCodeUsingCodeHashEnabled = DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
@CommandLine.Option(
hidden = true,
names = {"--Xbonsai-parallel-tx-processing-enabled"},
arity = "1",
description =
"Enables parallelization of transactions to optimize processing speed by concurrently loading and executing necessary data in advance. (default: ${DEFAULT-VALUE})")
private Boolean isParallelTxProcessingEnabled = false;
/** Default Constructor. */
Unstable() {}
}
/**
* Create data storage options.
*
* @return the data storage options
*/
public static DataStorageOptions create() {
return new DataStorageOptions();
}
/**
* Validates the data storage options
*
* @param commandLine the full commandLine to check all the options specified by the user
*/
public void validate(final CommandLine commandLine) {
if (DataStorageFormat.BONSAI == dataStorageFormat) {
if (bonsaiLimitTrieLogsEnabled) {
if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d",
MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT));
}
if (bonsaiTrieLogPruningWindowSize <= 0) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0",
bonsaiTrieLogPruningWindowSize));
}
if (bonsaiTrieLogPruningWindowSize <= bonsaiMaxLayersToLoad) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE
+ "=%d must be greater than "
+ BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD
+ "=%d",
bonsaiTrieLogPruningWindowSize,
bonsaiMaxLayersToLoad));
}
}
} else {
if (unstableOptions.isParallelTxProcessingEnabled) {
throw new CommandLine.ParameterException(
commandLine,
"Transaction parallelization is not supported unless operating in a 'diffbased' mode, such as Bonsai.");
}
}
}
/**
* Converts to options from the configuration
*
* @param domainObject to be reversed
* @return the options that correspond to the configuration
*/
public static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject) {
final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat();
dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad();
dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled();
dataStorageOptions.bonsaiLimitTrieLogsEnabled = domainObject.getBonsaiLimitTrieLogsEnabled();
dataStorageOptions.bonsaiTrieLogPruningWindowSize =
domainObject.getBonsaiTrieLogPruningWindowSize();
dataStorageOptions.unstableOptions.bonsaiFullFlatDbEnabled =
domainObject.getUnstable().getBonsaiFullFlatDbEnabled();
dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled =
domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled();
dataStorageOptions.unstableOptions.isParallelTxProcessingEnabled =
domainObject.getUnstable().isParallelTxProcessingEnabled();
return dataStorageOptions;
}
@Override
public DataStorageConfiguration toDomainObject() {
return ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(dataStorageFormat)
.bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad)
.receiptCompactionEnabled(receiptCompactionEnabled)
.bonsaiLimitTrieLogsEnabled(bonsaiLimitTrieLogsEnabled)
.bonsaiTrieLogPruningWindowSize(bonsaiTrieLogPruningWindowSize)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiFullFlatDbEnabled(unstableOptions.bonsaiFullFlatDbEnabled)
.bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled)
.isParallelTxProcessingEnabled(unstableOptions.isParallelTxProcessingEnabled)
.build())
.build();
}
@Override
public List<String> getCLIOptions() {
return CommandLineUtils.getCLIOptions(this, new DataStorageOptions());
}
/**
* Normalize data storage format string.
*
* @return the normalized string
*/
public String normalizeDataStorageFormat() {
return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase(Locale.ROOT));
}
}

@ -0,0 +1,121 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.cli.options.storage;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED;
import org.hyperledger.besu.cli.options.CLIOptions;
import org.hyperledger.besu.cli.util.CommandLineUtils;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import java.util.List;
import java.util.Locale;
import org.apache.commons.lang3.StringUtils;
import picocli.CommandLine;
import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
/** The Data storage CLI options. */
public class DataStorageOptions implements CLIOptions<DataStorageConfiguration> {
private static final String DATA_STORAGE_FORMAT = "--data-storage-format";
// Use Bonsai DB
@Option(
names = {DATA_STORAGE_FORMAT},
description =
"Format to store trie data in. Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).",
arity = "1")
private DataStorageFormat dataStorageFormat = DataStorageFormat.BONSAI;
@Option(
names = "--receipt-compaction-enabled",
description = "Enables compact storing of receipts (default: ${DEFAULT-VALUE})",
fallbackValue = "true")
private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED;
/**
* Options specific to diff-based storage modes. Holds the necessary parameters to configure
* diff-based storage, such as the Bonsai mode or Verkle in the future.
*/
@Mixin
private DiffBasedSubStorageOptions diffBasedSubStorageOptions =
DiffBasedSubStorageOptions.create();
/** Default Constructor. */
DataStorageOptions() {}
/**
* Create data storage options.
*
* @return the data storage options
*/
public static DataStorageOptions create() {
return new DataStorageOptions();
}
/**
* Validates the data storage options
*
* @param commandLine the full commandLine to check all the options specified by the user
*/
public void validate(final CommandLine commandLine) {
diffBasedSubStorageOptions.validate(commandLine, dataStorageFormat);
}
/**
* Converts to options from the configuration
*
* @param domainObject to be reversed
* @return the options that correspond to the configuration
*/
public static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject) {
final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat();
dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled();
dataStorageOptions.diffBasedSubStorageOptions =
DiffBasedSubStorageOptions.fromConfig(domainObject.getDiffBasedSubStorageConfiguration());
return dataStorageOptions;
}
@Override
public DataStorageConfiguration toDomainObject() {
final ImmutableDataStorageConfiguration.Builder builder =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(dataStorageFormat)
.receiptCompactionEnabled(receiptCompactionEnabled)
.diffBasedSubStorageConfiguration(diffBasedSubStorageOptions.toDomainObject());
return builder.build();
}
@Override
public List<String> getCLIOptions() {
final List<String> cliOptions = CommandLineUtils.getCLIOptions(this, new DataStorageOptions());
cliOptions.addAll(diffBasedSubStorageOptions.getCLIOptions());
return cliOptions;
}
/**
* Normalize data storage format string.
*
* @return the normalized string
*/
public String normalizeDataStorageFormat() {
return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase(Locale.ROOT));
}
}

@ -0,0 +1,217 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.cli.options.storage;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_LIMIT_TRIE_LOGS_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DiffBasedUnstable.DEFAULT_CODE_USING_CODE_HASH_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DiffBasedUnstable.DEFAULT_FULL_FLAT_DB_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.MINIMUM_TRIE_LOG_RETENTION_LIMIT;
import org.hyperledger.besu.cli.options.CLIOptions;
import org.hyperledger.besu.cli.util.CommandLineUtils;
import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import java.util.List;
import picocli.CommandLine;
import picocli.CommandLine.Option;
/** The Data storage CLI options. */
public class DiffBasedSubStorageOptions implements CLIOptions<DiffBasedSubStorageConfiguration> {
/** The maximum number of historical layers to load. */
public static final String MAX_LAYERS_TO_LOAD = "--bonsai-historical-block-limit";
@Option(
names = {MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"},
paramLabel = "<LONG>",
description =
"Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using "
+ LIMIT_TRIE_LOGS_ENABLED
+ " it will also be used as the number of layers of trie logs to retain.",
arity = "1")
private Long maxLayersToLoad = DEFAULT_MAX_LAYERS_TO_LOAD;
/** The bonsai limit trie logs enabled option name */
public static final String LIMIT_TRIE_LOGS_ENABLED = "--bonsai-limit-trie-logs-enabled";
/** The bonsai trie logs pruning window size. */
public static final String TRIE_LOG_PRUNING_WINDOW_SIZE =
"--bonsai-trie-logs-pruning-window-size";
// TODO --Xbonsai-limit-trie-logs-enabled and --Xbonsai-trie-log-pruning-enabled are deprecated,
// remove in a future release
@SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed")
@Option(
names = {
LIMIT_TRIE_LOGS_ENABLED,
"--Xbonsai-limit-trie-logs-enabled", // deprecated
"--Xbonsai-trie-log-pruning-enabled" // deprecated
},
fallbackValue = "true",
description = "Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})")
private Boolean limitTrieLogsEnabled = DEFAULT_LIMIT_TRIE_LOGS_ENABLED;
// TODO --Xbonsai-trie-logs-pruning-window-size is deprecated, remove in a future release
@SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed")
@Option(
names = {
TRIE_LOG_PRUNING_WINDOW_SIZE,
"--Xbonsai-trie-logs-pruning-window-size" // deprecated
},
description =
"The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
private Integer trieLogPruningWindowSize = DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE;
@CommandLine.ArgGroup(validate = false)
private final DiffBasedSubStorageOptions.Unstable unstableOptions = new Unstable();
/** Default Constructor. */
DiffBasedSubStorageOptions() {}
/** The unstable options for data storage. */
public static class Unstable {
// TODO: --Xsnapsync-synchronizer-flat-db-healing-enabled is deprecated, remove it in a future
// release
@Option(
hidden = true,
names = {
"--Xbonsai-full-flat-db-enabled",
"--Xsnapsync-synchronizer-flat-db-healing-enabled"
},
arity = "1",
description = "Enables bonsai full flat database strategy. (default: ${DEFAULT-VALUE})")
private Boolean fullFlatDbEnabled = DEFAULT_FULL_FLAT_DB_ENABLED;
@Option(
hidden = true,
names = {"--Xbonsai-code-using-code-hash-enabled"},
arity = "1",
description =
"Enables code storage using code hash instead of by account hash. (default: ${DEFAULT-VALUE})")
private boolean codeUsingCodeHashEnabled = DEFAULT_CODE_USING_CODE_HASH_ENABLED;
@Option(
hidden = true,
names = {"--Xbonsai-parallel-tx-processing-enabled"},
arity = "1",
description =
"Enables parallelization of transactions to optimize processing speed by concurrently loading and executing necessary data in advance. (default: ${DEFAULT-VALUE})")
private Boolean isParallelTxProcessingEnabled = false;
/** Default Constructor. */
Unstable() {}
}
/**
* Create data storage options.
*
* @return the data storage options
*/
public static DiffBasedSubStorageOptions create() {
return new DiffBasedSubStorageOptions();
}
/**
* Validates the data storage options
*
* @param commandLine the full commandLine to check all the options specified by the user
* @param dataStorageFormat the selected data storage format which determines the validation rules
* to apply.
*/
public void validate(final CommandLine commandLine, final DataStorageFormat dataStorageFormat) {
if (DataStorageFormat.BONSAI == dataStorageFormat) {
if (limitTrieLogsEnabled) {
if (maxLayersToLoad < MINIMUM_TRIE_LOG_RETENTION_LIMIT) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
MAX_LAYERS_TO_LOAD + " minimum value is %d", MINIMUM_TRIE_LOG_RETENTION_LIMIT));
}
if (trieLogPruningWindowSize <= 0) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0",
trieLogPruningWindowSize));
}
if (trieLogPruningWindowSize <= maxLayersToLoad) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
TRIE_LOG_PRUNING_WINDOW_SIZE
+ "=%d must be greater than "
+ MAX_LAYERS_TO_LOAD
+ "=%d",
trieLogPruningWindowSize,
maxLayersToLoad));
}
}
} else {
if (unstableOptions.isParallelTxProcessingEnabled) {
throw new CommandLine.ParameterException(
commandLine,
"Transaction parallelization is not supported unless operating in a 'diffbased' mode, such as Bonsai.");
}
}
}
/**
* Converts to options from the configuration
*
* @param domainObject to be reversed
* @return the options that correspond to the configuration
*/
public static DiffBasedSubStorageOptions fromConfig(
final DiffBasedSubStorageConfiguration domainObject) {
final DiffBasedSubStorageOptions dataStorageOptions = DiffBasedSubStorageOptions.create();
dataStorageOptions.maxLayersToLoad = domainObject.getMaxLayersToLoad();
dataStorageOptions.limitTrieLogsEnabled = domainObject.getLimitTrieLogsEnabled();
dataStorageOptions.trieLogPruningWindowSize = domainObject.getTrieLogPruningWindowSize();
dataStorageOptions.unstableOptions.fullFlatDbEnabled =
domainObject.getUnstable().getFullFlatDbEnabled();
dataStorageOptions.unstableOptions.codeUsingCodeHashEnabled =
domainObject.getUnstable().getCodeStoredByCodeHashEnabled();
dataStorageOptions.unstableOptions.isParallelTxProcessingEnabled =
domainObject.getUnstable().isParallelTxProcessingEnabled();
return dataStorageOptions;
}
@Override
public final DiffBasedSubStorageConfiguration toDomainObject() {
return ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(maxLayersToLoad)
.limitTrieLogsEnabled(limitTrieLogsEnabled)
.trieLogPruningWindowSize(trieLogPruningWindowSize)
.unstable(
ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder()
.fullFlatDbEnabled(unstableOptions.fullFlatDbEnabled)
.codeStoredByCodeHashEnabled(unstableOptions.codeUsingCodeHashEnabled)
.isParallelTxProcessingEnabled(unstableOptions.isParallelTxProcessingEnabled)
.build())
.build();
}
@Override
public List<String> getCLIOptions() {
return CommandLineUtils.getCLIOptions(this, new DiffBasedSubStorageOptions());
}
}

@ -15,21 +15,22 @@
package org.hyperledger.besu.cli.subcommands.storage;
import static com.google.common.base.Preconditions.checkArgument;
import static org.hyperledger.besu.cli.options.DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD;
import static org.hyperledger.besu.cli.options.storage.DiffBasedSubStorageOptions.MAX_LAYERS_TO_LOAD;
import static org.hyperledger.besu.cli.options.storage.DiffBasedSubStorageOptions.TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE;
import org.hyperledger.besu.cli.options.DataStorageOptions;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration;
import java.io.File;
import java.io.FileInputStream;
@ -64,7 +65,7 @@ public class TrieLogHelper {
boolean prune(
final DataStorageConfiguration config,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final MutableBlockchain blockchain,
final Path dataDirectoryPath) {
@ -73,7 +74,7 @@ public class TrieLogHelper {
validatePruneConfiguration(config);
final long layersToRetain = config.getBonsaiMaxLayersToLoad();
final long layersToRetain = config.getDiffBasedSubStorageConfiguration().getMaxLayersToLoad();
final long chainHeight = blockchain.getChainHeadBlockNumber();
@ -102,7 +103,7 @@ public class TrieLogHelper {
// Should only be layersToRetain left but loading extra just in case of an unforeseen bug
final long countAfterPrune =
rootWorldStateStorage
.streamTrieLogKeys(layersToRetain + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE)
.streamTrieLogKeys(layersToRetain + DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE)
.count();
if (countAfterPrune == layersToRetain) {
if (deleteFiles(batchFileNameBase, numberOfBatches)) {
@ -115,15 +116,12 @@ public class TrieLogHelper {
throw new IllegalStateException(
String.format(
"Remaining trie logs (%d) did not match %s (%d). Trie logs backup files (in %s) have not been deleted, it is safe to rerun the subcommand.",
countAfterPrune,
BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD,
layersToRetain,
batchFileNameBase));
countAfterPrune, MAX_LAYERS_TO_LOAD, layersToRetain, batchFileNameBase));
}
}
private void processTrieLogBatches(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final MutableBlockchain blockchain,
final long chainHeight,
final long lastBlockNumberToRetainTrieLogsFor,
@ -152,7 +150,7 @@ public class TrieLogHelper {
private void saveTrieLogBatches(
final String batchFileName,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final List<Hash> trieLogKeys) {
try {
@ -164,7 +162,7 @@ public class TrieLogHelper {
}
private void restoreTrieLogBatches(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final long batchNumber,
final String batchFileNameBase) {
@ -217,7 +215,7 @@ public class TrieLogHelper {
final MutableBlockchain blockchain,
final long chainHeight,
final long lastBlockNumberToRetainTrieLogsFor,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final long layersToRetain) {
if (lastBlockNumberToRetainTrieLogsFor < 0) {
@ -231,7 +229,7 @@ public class TrieLogHelper {
// plus extra threshold to account forks and orphans
final long clampedCountBeforePruning =
rootWorldStateStorage
.streamTrieLogKeys(layersToRetain + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE)
.streamTrieLogKeys(layersToRetain + DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE)
.count();
if (clampedCountBeforePruning < layersToRetain) {
throw new IllegalArgumentException(
@ -257,7 +255,7 @@ public class TrieLogHelper {
}
private void recreateTrieLogs(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final long batchNumber,
final String batchFileNameBase)
throws IOException {
@ -277,7 +275,7 @@ public class TrieLogHelper {
final int chunkSize,
final List<byte[]> keys,
final IdentityHashMap<byte[], byte[]> trieLogsToRetain,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) {
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage) {
var updater = rootWorldStateStorage.updater();
int endIndex = Math.min(startIndex + chunkSize, keys.size());
@ -294,31 +292,31 @@ public class TrieLogHelper {
@VisibleForTesting
void validatePruneConfiguration(final DataStorageConfiguration config) {
final DiffBasedSubStorageConfiguration subStorageConfiguration =
config.getDiffBasedSubStorageConfiguration();
checkArgument(
config.getBonsaiMaxLayersToLoad()
>= DataStorageConfiguration.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT,
subStorageConfiguration.getMaxLayersToLoad()
>= DiffBasedSubStorageConfiguration.MINIMUM_TRIE_LOG_RETENTION_LIMIT,
String.format(
BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d",
DataStorageConfiguration.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT));
MAX_LAYERS_TO_LOAD + " minimum value is %d",
DiffBasedSubStorageConfiguration.MINIMUM_TRIE_LOG_RETENTION_LIMIT));
checkArgument(
config.getBonsaiTrieLogPruningWindowSize() > 0,
subStorageConfiguration.getTrieLogPruningWindowSize() > 0,
String.format(
DataStorageOptions.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0",
config.getBonsaiTrieLogPruningWindowSize()));
TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0",
subStorageConfiguration.getTrieLogPruningWindowSize()));
checkArgument(
config.getBonsaiTrieLogPruningWindowSize() > config.getBonsaiMaxLayersToLoad(),
subStorageConfiguration.getTrieLogPruningWindowSize()
> subStorageConfiguration.getMaxLayersToLoad(),
String.format(
DataStorageOptions.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE
+ "=%d must be greater than "
+ BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD
+ "=%d",
config.getBonsaiTrieLogPruningWindowSize(),
config.getBonsaiMaxLayersToLoad()));
TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than " + MAX_LAYERS_TO_LOAD + "=%d",
subStorageConfiguration.getTrieLogPruningWindowSize(),
subStorageConfiguration.getMaxLayersToLoad()));
}
private void saveTrieLogsInFile(
final List<Hash> trieLogsKeys,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final String batchFileName)
throws IOException {
@ -355,7 +353,7 @@ public class TrieLogHelper {
private void saveTrieLogsAsRlpInFile(
final List<Hash> trieLogsKeys,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final String batchFileName) {
File file = new File(batchFileName);
if (file.exists()) {
@ -400,7 +398,8 @@ public class TrieLogHelper {
}
private IdentityHashMap<byte[], byte[]> getTrieLogs(
final List<Hash> trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) {
final List<Hash> trieLogKeys,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage) {
IdentityHashMap<byte[], byte[]> trieLogsToRetain = new IdentityHashMap<>();
LOG.info("Obtaining trielogs from db, this may take a few minutes...");
@ -413,7 +412,7 @@ public class TrieLogHelper {
}
TrieLogCount getCount(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final int limit,
final Blockchain blockchain) {
final AtomicInteger total = new AtomicInteger();
@ -454,7 +453,7 @@ public class TrieLogHelper {
}
void importTrieLog(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) {
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) {
var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString());
@ -464,7 +463,7 @@ public class TrieLogHelper {
}
void exportTrieLog(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage,
final List<Hash> trieLogHash,
final Path directoryPath)
throws IOException {

@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldSt
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import java.io.IOException;
@ -89,7 +90,11 @@ public class TrieLogSubCommand implements Runnable {
.besuCommand
.setupControllerBuilder()
.dataStorageConfiguration(
ImmutableDataStorageConfiguration.copyOf(config).withBonsaiLimitTrieLogsEnabled(false))
ImmutableDataStorageConfiguration.copyOf(config)
.withDiffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.copyOf(
config.getDiffBasedSubStorageConfiguration())
.withLimitTrieLogsEnabled(false)))
.build();
}

@ -89,6 +89,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManage
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
@ -741,8 +742,10 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
final JsonRpcMethods additionalJsonRpcMethodFactory =
createAdditionalJsonRpcMethodFactory(protocolContext, protocolSchedule, miningParameters);
if (dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()
&& DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) {
if (DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) {
final DiffBasedSubStorageConfiguration subStorageConfiguration =
dataStorageConfiguration.getDiffBasedSubStorageConfiguration();
if (subStorageConfiguration.getLimitTrieLogsEnabled()) {
final TrieLogManager trieLogManager =
((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager();
final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
@ -751,6 +754,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
createTrieLogPruner(worldStateKeyValueStorage, blockchain, scheduler);
trieLogManager.subscribe(trieLogPruner);
}
}
final List<Closeable> closeables = new ArrayList<>();
closeables.add(protocolContext.getWorldStateArchive());
@ -803,14 +807,15 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
final Blockchain blockchain,
final EthScheduler scheduler) {
final boolean isProofOfStake = genesisConfigOptions.getTerminalTotalDifficulty().isPresent();
final DiffBasedSubStorageConfiguration subStorageConfiguration =
dataStorageConfiguration.getDiffBasedSubStorageConfiguration();
final TrieLogPruner trieLogPruner =
new TrieLogPruner(
(BonsaiWorldStateKeyValueStorage) worldStateStorage,
blockchain,
scheduler::executeServiceTask,
dataStorageConfiguration.getBonsaiMaxLayersToLoad(),
dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize(),
subStorageConfiguration.getMaxLayersToLoad(),
subStorageConfiguration.getTrieLogPruningWindowSize(),
isProofOfStake,
metricsSystem);
trieLogPruner.initialize();
@ -1092,10 +1097,14 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
case BONSAI -> {
final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class);
yield new BonsaiWorldStateProvider(
worldStateKeyValueStorage,
blockchain,
Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
Optional.of(
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getMaxLayersToLoad()),
bonsaiCachedMerkleTrieLoader,
besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
evmConfiguration);

@ -1390,7 +1390,7 @@ public class BesuCommandTest extends CommandTestAbstract {
}
@Test
public void bonsaiLimitTrieLogsEnabledByDefault() {
public void diffbasedLimitTrieLogsEnabledByDefault() {
parseCommand();
verify(mockControllerBuilder)
.dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture());
@ -1398,7 +1398,11 @@ public class BesuCommandTest extends CommandTestAbstract {
final DataStorageConfiguration dataStorageConfiguration =
dataStorageConfigurationArgumentCaptor.getValue();
assertThat(dataStorageConfiguration.getDataStorageFormat()).isEqualTo(BONSAI);
assertThat(dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()).isTrue();
assertThat(
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getLimitTrieLogsEnabled())
.isTrue();
assertThat(commandOutput.toString(UTF_8)).isEmpty();
assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
}
@ -1413,7 +1417,11 @@ public class BesuCommandTest extends CommandTestAbstract {
final DataStorageConfiguration dataStorageConfiguration =
dataStorageConfigurationArgumentCaptor.getValue();
assertThat(dataStorageConfiguration.getDataStorageFormat()).isEqualTo(BONSAI);
assertThat(dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()).isFalse();
assertThat(
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getLimitTrieLogsEnabled())
.isFalse();
verify(mockLogger)
.warn(
"Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.",
@ -1448,7 +1456,8 @@ public class BesuCommandTest extends CommandTestAbstract {
final DataStorageConfiguration dataStorageConfiguration =
dataStorageConfigurationArgumentCaptor.getValue();
assertThat(dataStorageConfiguration.getDataStorageFormat()).isEqualTo(BONSAI);
assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()).isEqualTo(11);
assertThat(dataStorageConfiguration.getDiffBasedSubStorageConfiguration().getMaxLayersToLoad())
.isEqualTo(11);
assertThat(commandOutput.toString(UTF_8)).isEmpty();
assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
}
@ -2551,8 +2560,9 @@ public class BesuCommandTest extends CommandTestAbstract {
besuCommand
.getDataStorageOptions()
.toDomainObject()
.getDiffBasedSubStorageConfiguration()
.getUnstable()
.getBonsaiFullFlatDbEnabled())
.getFullFlatDbEnabled())
.isTrue();
}
@ -2563,8 +2573,9 @@ public class BesuCommandTest extends CommandTestAbstract {
besuCommand
.dataStorageOptions
.toDomainObject()
.getDiffBasedSubStorageConfiguration()
.getUnstable()
.getBonsaiFullFlatDbEnabled())
.getFullFlatDbEnabled())
.isFalse();
}

@ -34,10 +34,10 @@ import org.hyperledger.besu.chainexport.RlpBlockExporter;
import org.hyperledger.besu.chainimport.JsonBlockImporter;
import org.hyperledger.besu.chainimport.RlpBlockImporter;
import org.hyperledger.besu.cli.config.EthNetworkConfig;
import org.hyperledger.besu.cli.options.DataStorageOptions;
import org.hyperledger.besu.cli.options.MiningOptions;
import org.hyperledger.besu.cli.options.TransactionPoolOptions;
import org.hyperledger.besu.cli.options.stable.EthstatsOptions;
import org.hyperledger.besu.cli.options.storage.DataStorageOptions;
import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
import org.hyperledger.besu.cli.options.unstable.NetworkingOptions;
import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions;

@ -162,7 +162,7 @@ class ConfigurationOverviewBuilderTest {
}
@Test
void setBonsaiLimitTrieLogsEnabled() {
void setDiffbasedLimitTrieLogsEnabled() {
final String noTrieLogRetentionLimitSet = builder.build();
assertThat(noTrieLogRetentionLimitSet).doesNotContain("Limit trie logs enabled");

@ -64,7 +64,6 @@ public abstract class AbstractCLIOptionsTest<D, T extends CLIOptions<D>>
private void getCLIOptions(final D domainObject) {
T options = optionsFromDomainObject(domainObject);
final String[] cliOptions = options.getCLIOptions().toArray(new String[0]);
final TestBesuCommand cmd = parseCommand(cliOptions);
final T optionsFromCommand = getOptionsFromBesuCommand(cmd);

@ -15,12 +15,13 @@
package org.hyperledger.besu.cli.options.stable;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.MINIMUM_TRIE_LOG_RETENTION_LIMIT;
import org.hyperledger.besu.cli.options.AbstractCLIOptionsTest;
import org.hyperledger.besu.cli.options.DataStorageOptions;
import org.hyperledger.besu.cli.options.storage.DataStorageOptions;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import org.junit.jupiter.api.Test;
@ -32,7 +33,11 @@ public class DataStorageOptionsTest
public void bonsaiTrieLogPruningLimitOption() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize()).isEqualTo(600),
assertThat(
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getTrieLogPruningWindowSize())
.isEqualTo(600),
"--bonsai-limit-trie-logs-enabled",
"--bonsai-trie-logs-pruning-window-size",
"600");
@ -42,7 +47,11 @@ public class DataStorageOptionsTest
public void bonsaiTrieLogPruningLimitLegacyOption() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getBonsaiTrieLogPruningWindowSize()).isEqualTo(600),
assertThat(
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getTrieLogPruningWindowSize())
.isEqualTo(600),
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-pruning-window-size",
"600");
@ -52,12 +61,16 @@ public class DataStorageOptionsTest
public void bonsaiTrieLogsEnabled_explicitlySetToFalse() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()).isEqualTo(false),
assertThat(
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getLimitTrieLogsEnabled())
.isEqualTo(false),
"--bonsai-limit-trie-logs-enabled=false");
}
@Test
public void bonsaiTrieLogPruningWindowSizeShouldBePositive() {
public void diffbasedTrieLogPruningWindowSizeShouldBePositive() {
internalTestFailure(
"--bonsai-trie-logs-pruning-window-size=0 must be greater than 0",
"--bonsai-limit-trie-logs-enabled",
@ -66,7 +79,7 @@ public class DataStorageOptionsTest
}
@Test
public void bonsaiTrieLogPruningWindowSizeShouldBeAboveRetentionLimit() {
public void diffbasedTrieLogPruningWindowSizeShouldBeAboveRetentionLimit() {
internalTestFailure(
"--bonsai-trie-logs-pruning-window-size=512 must be greater than --bonsai-historical-block-limit=512",
"--bonsai-limit-trie-logs-enabled",
@ -78,8 +91,11 @@ public class DataStorageOptionsTest
public void bonsaiTrieLogRetentionLimitOption() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad())
.isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT + 1),
assertThat(
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getMaxLayersToLoad())
.isEqualTo(MINIMUM_TRIE_LOG_RETENTION_LIMIT + 1),
"--bonsai-limit-trie-logs-enabled",
"--bonsai-historical-block-limit",
"513");
@ -89,8 +105,11 @@ public class DataStorageOptionsTest
public void bonsaiTrieLogRetentionLimitOption_boundaryTest() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad())
.isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT),
assertThat(
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getMaxLayersToLoad())
.isEqualTo(MINIMUM_TRIE_LOG_RETENTION_LIMIT),
"--bonsai-limit-trie-logs-enabled",
"--bonsai-historical-block-limit",
"512");
@ -106,22 +125,28 @@ public class DataStorageOptionsTest
}
@Test
public void bonsaiCodeUsingCodeHashEnabledCanBeEnabled() {
public void diffbasedCodeUsingCodeHashEnabledCanBeEnabled() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(
dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled())
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getUnstable()
.getCodeStoredByCodeHashEnabled())
.isEqualTo(true),
"--Xbonsai-code-using-code-hash-enabled",
"true");
}
@Test
public void bonsaiCodeUsingCodeHashEnabledCanBeDisabled() {
public void diffbasedCodeUsingCodeHashEnabledCanBeDisabled() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(
dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled())
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getUnstable()
.getCodeStoredByCodeHashEnabled())
.isEqualTo(false),
"--Xbonsai-code-using-code-hash-enabled",
"false");
@ -160,9 +185,12 @@ public class DataStorageOptionsTest
protected DataStorageConfiguration createCustomizedDomainObject() {
return ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(513L)
.bonsaiLimitTrieLogsEnabled(true)
.bonsaiTrieLogPruningWindowSize(514)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(513L)
.limitTrieLogsEnabled(true)
.trieLogPruningWindowSize(514)
.build())
.build();
}

@ -17,7 +17,7 @@ package org.hyperledger.besu.cli.subcommands.storage;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.spy;
@ -35,6 +35,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactor
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import java.io.FileNotFoundException;
@ -134,8 +135,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(3L)
.bonsaiLimitTrieLogsEnabled(true)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(3L)
.limitTrieLogsEnabled(true)
.build())
.build();
mockBlockchainBase();
@ -172,8 +176,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(2L)
.bonsaiLimitTrieLogsEnabled(true)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(2L)
.limitTrieLogsEnabled(true)
.build())
.build();
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L);
@ -192,8 +199,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(10L)
.bonsaiLimitTrieLogsEnabled(true)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(10L)
.limitTrieLogsEnabled(true)
.build())
.build();
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L);
@ -212,8 +222,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(2L)
.bonsaiLimitTrieLogsEnabled(true)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(2L)
.limitTrieLogsEnabled(true)
.build())
.build();
mockBlockchainBase();
@ -233,8 +246,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(6L)
.bonsaiLimitTrieLogsEnabled(true)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(6L)
.limitTrieLogsEnabled(true)
.build())
.build();
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L);
@ -255,8 +271,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(3L)
.bonsaiLimitTrieLogsEnabled(true)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(3L)
.limitTrieLogsEnabled(true)
.build())
.build();
mockBlockchainBase();
@ -266,7 +285,7 @@ class TrieLogHelperTest {
final BonsaiWorldStateKeyValueStorage inMemoryWorldStateSpy = spy(inMemoryWorldState);
// force a different value the second time the trie log count is called
when(inMemoryWorldStateSpy.streamTrieLogKeys(3L + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE))
when(inMemoryWorldStateSpy.streamTrieLogKeys(3L + DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE))
.thenCallRealMethod()
.thenReturn(Stream.empty());
assertThatThrownBy(
@ -284,8 +303,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(511L)
.bonsaiLimitTrieLogsEnabled(true)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(511L)
.limitTrieLogsEnabled(true)
.build())
.build();
TrieLogHelper helper = new TrieLogHelper();
@ -302,9 +324,12 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(512L)
.bonsaiLimitTrieLogsEnabled(true)
.bonsaiTrieLogPruningWindowSize(0)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(512L)
.limitTrieLogsEnabled(true)
.trieLogPruningWindowSize(0)
.build())
.build();
TrieLogHelper helper = new TrieLogHelper();
@ -320,9 +345,12 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(512L)
.bonsaiLimitTrieLogsEnabled(true)
.bonsaiTrieLogPruningWindowSize(512)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(512L)
.limitTrieLogsEnabled(true)
.trieLogPruningWindowSize(512)
.build())
.build();
TrieLogHelper helper = new TrieLogHelper();
@ -340,8 +368,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(3L)
.bonsaiLimitTrieLogsEnabled(true)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(3L)
.limitTrieLogsEnabled(true)
.build())
.build();
mockBlockchainBase();

@ -58,8 +58,10 @@ class TrieLogSubCommandTest extends CommandTestAbstract {
.dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture());
final List<DataStorageConfiguration> configs =
dataStorageConfigurationArgumentCaptor.getAllValues();
assertThat(configs.get(0).getBonsaiLimitTrieLogsEnabled()).isTrue();
assertThat(configs.get(1).getBonsaiLimitTrieLogsEnabled()).isFalse();
assertThat(configs.get(0).getDiffBasedSubStorageConfiguration().getLimitTrieLogsEnabled())
.isTrue();
assertThat(configs.get(1).getDiffBasedSubStorageConfiguration().getLimitTrieLogsEnabled())
.isFalse();
}
@Test
@ -69,6 +71,11 @@ class TrieLogSubCommandTest extends CommandTestAbstract {
.dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture());
final List<DataStorageConfiguration> configs =
dataStorageConfigurationArgumentCaptor.getAllValues();
assertThat(configs).allMatch(DataStorageConfiguration::getBonsaiLimitTrieLogsEnabled);
assertThat(configs)
.allMatch(
dataStorageConfiguration ->
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getLimitTrieLogsEnabled());
}
}

@ -24,9 +24,10 @@ import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.trie.MerkleTrie;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFlatDbStrategyProvider;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
@ -49,7 +50,7 @@ import org.apache.tuweni.bytes.Bytes32;
public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValueStorage
implements WorldStateKeyValueStorage {
protected final FlatDbStrategyProvider flatDbStrategyProvider;
protected final BonsaiFlatDbStrategyProvider flatDbStrategyProvider;
public BonsaiWorldStateKeyValueStorage(
final StorageProvider provider,
@ -61,12 +62,12 @@ public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValue
ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)),
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE));
this.flatDbStrategyProvider =
new FlatDbStrategyProvider(metricsSystem, dataStorageConfiguration);
new BonsaiFlatDbStrategyProvider(metricsSystem, dataStorageConfiguration);
flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
}
public BonsaiWorldStateKeyValueStorage(
final FlatDbStrategyProvider flatDbStrategyProvider,
final BonsaiFlatDbStrategyProvider flatDbStrategyProvider,
final SegmentedKeyValueStorage composedWorldStateStorage,
final KeyValueStorage trieLogStorage) {
super(composedWorldStateStorage, trieLogStorage);
@ -87,15 +88,12 @@ public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValue
if (codeHash.equals(Hash.EMPTY)) {
return Optional.of(Bytes.EMPTY);
} else {
return flatDbStrategyProvider
.getFlatDbStrategy(composedWorldStateStorage)
.getFlatCode(codeHash, accountHash, composedWorldStateStorage);
return getFlatDbStrategy().getFlatCode(codeHash, accountHash, composedWorldStateStorage);
}
}
public Optional<Bytes> getAccount(final Hash accountHash) {
return flatDbStrategyProvider
.getFlatDbStrategy(composedWorldStateStorage)
return getFlatDbStrategy()
.getFlatAccount(
this::getWorldStateRootHash,
this::getAccountStateTrieNode,
@ -148,8 +146,7 @@ public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValue
final Supplier<Optional<Hash>> storageRootSupplier,
final Hash accountHash,
final StorageSlotKey storageSlotKey) {
return flatDbStrategyProvider
.getFlatDbStrategy(composedWorldStateStorage)
return getFlatDbStrategy()
.getFlatStorageValueByStorageSlotKey(
this::getWorldStateRootHash,
storageRootSupplier,
@ -180,8 +177,9 @@ public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValue
}
@Override
public FlatDbStrategy getFlatDbStrategy() {
return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage);
public BonsaiFlatDbStrategy getFlatDbStrategy() {
return (BonsaiFlatDbStrategy)
flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage);
}
@Override
@ -189,7 +187,7 @@ public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValue
return new Updater(
composedWorldStateStorage.startTransaction(),
trieLogStorage.startTransaction(),
flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage));
getFlatDbStrategy());
}
public static class Updater implements DiffBasedWorldStateKeyValueStorage.Updater {

@ -0,0 +1,173 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.trie.NodeLoader;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Stream;
import kotlin.Pair;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
/**
* This class represents a FlatDbReaderStrategy, which is responsible for reading and writing data
* from flat databases. It implements various methods for storing and retrieving account data, code
* data, and storage data from the corresponding KeyValueStorage.
*/
public abstract class BonsaiFlatDbStrategy extends FlatDbStrategy {
public BonsaiFlatDbStrategy(
final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) {
super(metricsSystem, codeStorageStrategy);
}
/*
* Retrieves the account data for the given account hash, using the world state root hash supplier and node loader.
*/
public abstract Optional<Bytes> getFlatAccount(
Supplier<Optional<Bytes>> worldStateRootHashSupplier,
NodeLoader nodeLoader,
Hash accountHash,
SegmentedKeyValueStorage storage);
/*
* Retrieves the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
*/
public abstract Optional<Bytes> getFlatStorageValueByStorageSlotKey(
Supplier<Optional<Bytes>> worldStateRootHashSupplier,
Supplier<Optional<Hash>> storageRootSupplier,
NodeLoader nodeLoader,
Hash accountHash,
StorageSlotKey storageSlotKey,
SegmentedKeyValueStorage storageStorage);
@Override
public void putFlatAccount(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Bytes accountValue) {
transaction.put(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe());
}
@Override
public void removeFlatAccount(
final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash) {
transaction.remove(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe());
}
@Override
public void putFlatAccountStorageValueByStorageSlotHash(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash slotHash,
final Bytes storage) {
transaction.put(
ACCOUNT_STORAGE_STORAGE,
Bytes.concatenate(accountHash, slotHash).toArrayUnsafe(),
storage.toArrayUnsafe());
}
@Override
public void removeFlatAccountStorageValueByStorageSlotHash(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash slotHash) {
transaction.remove(
ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, slotHash).toArrayUnsafe());
}
@Override
public void clearAll(final SegmentedKeyValueStorage storage) {
storage.clear(ACCOUNT_INFO_STATE);
storage.clear(ACCOUNT_STORAGE_STORAGE);
storage.clear(CODE_STORAGE);
}
@Override
public void resetOnResync(final SegmentedKeyValueStorage storage) {
storage.clear(ACCOUNT_INFO_STATE);
storage.clear(ACCOUNT_STORAGE_STORAGE);
}
@Override
protected Stream<Pair<Bytes32, Bytes>> storageToPairStream(
final SegmentedKeyValueStorage storage,
final Hash accountHash,
final Bytes startKeyHash,
final Function<Bytes, Bytes> valueMapper) {
return storage
.streamFromKey(
ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe())
.takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash))
.map(
pair ->
new Pair<>(
Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)),
valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros())));
}
@Override
protected Stream<Pair<Bytes32, Bytes>> storageToPairStream(
final SegmentedKeyValueStorage storage,
final Hash accountHash,
final Bytes startKeyHash,
final Bytes32 endKeyHash,
final Function<Bytes, Bytes> valueMapper) {
return storage
.streamFromKey(
ACCOUNT_STORAGE_STORAGE,
Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe(),
Bytes.concatenate(accountHash, endKeyHash).toArrayUnsafe())
.map(
pair ->
new Pair<>(
Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)),
valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros())));
}
@Override
protected Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash) {
return storage
.streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe(), endKeyHash.toArrayUnsafe())
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())));
}
@Override
protected Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
final SegmentedKeyValueStorage storage, final Bytes startKeyHash) {
return storage
.streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe())
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())));
}
}

@ -0,0 +1,88 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BonsaiFlatDbStrategyProvider extends FlatDbStrategyProvider {
private static final Logger LOG = LoggerFactory.getLogger(BonsaiFlatDbStrategyProvider.class);
public BonsaiFlatDbStrategyProvider(
final MetricsSystem metricsSystem, final DataStorageConfiguration dataStorageConfiguration) {
super(metricsSystem, dataStorageConfiguration);
}
@Override
protected FlatDbMode getRequestedFlatDbMode(
final DataStorageConfiguration dataStorageConfiguration) {
return dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getUnstable()
.getFullFlatDbEnabled()
? FlatDbMode.FULL
: FlatDbMode.PARTIAL;
}
@Override
protected FlatDbMode alternativeFlatDbModeForExistingDatabase() {
return FlatDbMode.PARTIAL;
}
public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) {
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
LOG.info("setting FlatDbStrategy to FULL");
transaction.put(
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
transaction.commit();
loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy
}
public void downgradeToPartialFlatDbMode(
final SegmentedKeyValueStorage composedWorldStateStorage) {
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
LOG.info("setting FlatDbStrategy to PARTIAL");
transaction.put(
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe());
transaction.commit();
loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy
}
@Override
protected FlatDbStrategy createFlatDbStrategy(
final FlatDbMode flatDbMode,
final MetricsSystem metricsSystem,
final CodeStorageStrategy codeStorageStrategy) {
if (flatDbMode == FlatDbMode.FULL) {
return new BonsaiFullFlatDbStrategy(metricsSystem, codeStorageStrategy);
} else {
return new BonsaiPartialFlatDbStrategy(metricsSystem, codeStorageStrategy);
}
}
}

@ -21,7 +21,6 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.trie.NodeLoader;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy;
import org.hyperledger.besu.metrics.BesuMetricCategory;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.plugin.services.metrics.Counter;
@ -32,13 +31,13 @@ import java.util.function.Supplier;
import org.apache.tuweni.bytes.Bytes;
public class FullFlatDbStrategy extends FlatDbStrategy {
public class BonsaiFullFlatDbStrategy extends BonsaiFlatDbStrategy {
protected final Counter getAccountNotFoundInFlatDatabaseCounter;
protected final Counter getStorageValueNotFoundInFlatDatabaseCounter;
public FullFlatDbStrategy(
public BonsaiFullFlatDbStrategy(
final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) {
super(metricsSystem, codeStorageStrategy);

@ -21,7 +21,6 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.trie.NodeLoader;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie;
import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory;
import org.hyperledger.besu.metrics.BesuMetricCategory;
@ -45,7 +44,7 @@ import org.apache.tuweni.rlp.RLP;
* methods, which checks if the data is present in the flat database, and if not, queries the merkle
* trie
*/
public class PartialFlatDbStrategy extends FlatDbStrategy {
public class BonsaiPartialFlatDbStrategy extends BonsaiFlatDbStrategy {
protected final Counter getAccountMerkleTrieCounter;
protected final Counter getAccountMissingMerkleTrieCounter;
@ -53,7 +52,7 @@ public class PartialFlatDbStrategy extends FlatDbStrategy {
protected final Counter getStorageValueMerkleTrieCounter;
protected final Counter getStorageValueMissingMerkleTrieCounter;
public PartialFlatDbStrategy(
public BonsaiPartialFlatDbStrategy(
final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) {
super(metricsSystem, codeStorageStrategy);
getAccountMerkleTrieCounter =

@ -14,13 +14,7 @@
*/
package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.trie.NodeLoader;
import org.hyperledger.besu.metrics.BesuMetricCategory;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.plugin.services.metrics.Counter;
@ -33,7 +27,6 @@ import java.util.Optional;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -86,27 +79,6 @@ public abstract class FlatDbStrategy {
"Number of storage slots found in the flat database");
}
/*
* Retrieves the account data for the given account hash, using the world state root hash supplier and node loader.
*/
public abstract Optional<Bytes> getFlatAccount(
Supplier<Optional<Bytes>> worldStateRootHashSupplier,
NodeLoader nodeLoader,
Hash accountHash,
SegmentedKeyValueStorage storage);
/*
* Retrieves the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
*/
public abstract Optional<Bytes> getFlatStorageValueByStorageSlotKey(
Supplier<Optional<Bytes>> worldStateRootHashSupplier,
Supplier<Optional<Hash>> storageRootSupplier,
NodeLoader nodeLoader,
Hash accountHash,
StorageSlotKey storageSlotKey,
SegmentedKeyValueStorage storageStorage);
public boolean isCodeByCodeHash() {
return codeStorageStrategy instanceof CodeHashCodeStorageStrategy;
}
@ -124,76 +96,57 @@ public abstract class FlatDbStrategy {
}
/*
* Puts the account data for the given account hash, using the world state root hash supplier and node loader.
* Removes code for the given account hash.
*/
public void putFlatAccount(
public void removeFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Bytes accountValue) {
transaction.put(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe());
}
public void removeFlatAccount(
final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash) {
transaction.remove(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe());
final Hash codeHash) {
codeStorageStrategy.removeFlatCode(transaction, accountHash, codeHash);
}
/*
* Puts the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
* Puts the code data for the given code hash and account hash.
*/
public void putFlatAccountStorageValueByStorageSlotHash(
public void putFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash slotHash,
final Bytes storage) {
transaction.put(
ACCOUNT_STORAGE_STORAGE,
Bytes.concatenate(accountHash, slotHash).toArrayUnsafe(),
storage.toArrayUnsafe());
final Hash codeHash,
final Bytes code) {
codeStorageStrategy.putFlatCode(transaction, accountHash, codeHash, code);
}
/*
* Removes the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
* Puts the account data for the given account hash, using the world state root hash supplier and node loader.
*/
public void removeFlatAccountStorageValueByStorageSlotHash(
public abstract void putFlatAccount(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash slotHash) {
transaction.remove(
ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, slotHash).toArrayUnsafe());
}
final Bytes accountValue);
public abstract void removeFlatAccount(
final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash);
/*
* Removes code for the given account hash.
* Puts the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
*/
public void removeFlatCode(
public abstract void putFlatAccountStorageValueByStorageSlotHash(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash) {
codeStorageStrategy.removeFlatCode(transaction, accountHash, codeHash);
}
final Hash slotHash,
final Bytes storage);
/*
* Puts the code data for the given code hash and account hash.
* Removes the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader.
*/
public void putFlatCode(
public abstract void removeFlatAccountStorageValueByStorageSlotHash(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash,
final Bytes code) {
codeStorageStrategy.putFlatCode(transaction, accountHash, codeHash, code);
}
final Hash slotHash);
public void clearAll(final SegmentedKeyValueStorage storage) {
storage.clear(ACCOUNT_INFO_STATE);
storage.clear(ACCOUNT_STORAGE_STORAGE);
storage.clear(CODE_STORAGE);
}
public abstract void clearAll(final SegmentedKeyValueStorage storage);
public void resetOnResync(final SegmentedKeyValueStorage storage) {
storage.clear(ACCOUNT_INFO_STATE);
storage.clear(ACCOUNT_STORAGE_STORAGE);
}
public abstract void resetOnResync(final SegmentedKeyValueStorage storage);
public NavigableMap<Bytes32, Bytes> streamAccountFlatDatabase(
final SegmentedKeyValueStorage storage,
@ -249,57 +202,26 @@ public abstract class FlatDbStrategy {
.takeWhile(takeWhile));
}
private static Stream<Pair<Bytes32, Bytes>> storageToPairStream(
protected abstract Stream<Pair<Bytes32, Bytes>> storageToPairStream(
final SegmentedKeyValueStorage storage,
final Hash accountHash,
final Bytes startKeyHash,
final Function<Bytes, Bytes> valueMapper) {
return storage
.streamFromKey(
ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe())
.takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash))
.map(
pair ->
new Pair<>(
Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)),
valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros())));
}
final Function<Bytes, Bytes> valueMapper);
private static Stream<Pair<Bytes32, Bytes>> storageToPairStream(
protected abstract Stream<Pair<Bytes32, Bytes>> storageToPairStream(
final SegmentedKeyValueStorage storage,
final Hash accountHash,
final Bytes startKeyHash,
final Bytes32 endKeyHash,
final Function<Bytes, Bytes> valueMapper) {
return storage
.streamFromKey(
ACCOUNT_STORAGE_STORAGE,
Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe(),
Bytes.concatenate(accountHash, endKeyHash).toArrayUnsafe())
.map(
pair ->
new Pair<>(
Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)),
valueMapper.apply(Bytes.wrap(pair.getValue()).trimLeadingZeros())));
}
final Function<Bytes, Bytes> valueMapper);
private static Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash) {
return storage
.streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe(), endKeyHash.toArrayUnsafe())
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())));
}
protected abstract Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash);
private static Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
final SegmentedKeyValueStorage storage, final Bytes startKeyHash) {
return storage
.streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe())
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())));
}
protected abstract Stream<Pair<Bytes32, Bytes>> accountsToPairStream(
final SegmentedKeyValueStorage storage, final Bytes startKeyHash);
private static NavigableMap<Bytes32, Bytes> toNavigableMap(
private NavigableMap<Bytes32, Bytes> toNavigableMap(
final Stream<Pair<Bytes32, Bytes>> pairStream) {
final TreeMap<Bytes32, Bytes> collected =
pairStream.collect(

@ -18,13 +18,10 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIden
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
@ -34,7 +31,7 @@ import org.apache.tuweni.bytes.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FlatDbStrategyProvider {
public abstract class FlatDbStrategyProvider {
private static final Logger LOG = LoggerFactory.getLogger(FlatDbStrategyProvider.class);
// 0x666C61744462537461747573
@ -61,20 +58,48 @@ public class FlatDbStrategyProvider {
deriveUseCodeStorageByHash(composedWorldStateStorage)
? new CodeHashCodeStorageStrategy()
: new AccountHashCodeStorageStrategy();
if (flatDbMode == FlatDbMode.FULL) {
this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem, codeStorageStrategy);
} else {
this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem, codeStorageStrategy);
this.flatDbStrategy = createFlatDbStrategy(flatDbMode, metricsSystem, codeStorageStrategy);
}
}
protected boolean deriveUseCodeStorageByHash(
final SegmentedKeyValueStorage composedWorldStateStorage) {
final boolean configCodeUsingHash =
dataStorageConfiguration
.getDiffBasedSubStorageConfiguration()
.getUnstable()
.getCodeStoredByCodeHashEnabled();
boolean codeUsingCodeByHash =
detectCodeStorageByHash(composedWorldStateStorage)
.map(
dbCodeUsingHash -> {
if (dbCodeUsingHash != configCodeUsingHash) {
LOG.warn(
"Bonsai db is using code storage mode {} but config specifies mode {}. Using mode from database",
dbCodeUsingHash,
configCodeUsingHash);
}
return dbCodeUsingHash;
})
.orElse(configCodeUsingHash);
LOG.info("DB mode with code stored using code hash enabled = {}", codeUsingCodeByHash);
return codeUsingCodeByHash;
}
private Optional<Boolean> detectCodeStorageByHash(
final SegmentedKeyValueStorage composedWorldStateStorage) {
return composedWorldStateStorage.stream(CODE_STORAGE)
.limit(1)
.findFirst()
.map(
keypair ->
CodeHashCodeStorageStrategy.isCodeHashValue(keypair.getKey(), keypair.getValue()));
}
@VisibleForTesting
FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) {
final FlatDbMode requestedFlatDbMode =
dataStorageConfiguration.getUnstable().getBonsaiFullFlatDbEnabled()
? FlatDbMode.FULL
: FlatDbMode.PARTIAL;
private FlatDbMode deriveFlatDbStrategy(
final SegmentedKeyValueStorage composedWorldStateStorage) {
final FlatDbMode requestedFlatDbMode = getRequestedFlatDbMode(dataStorageConfiguration);
final var existingTrieData =
composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).isPresent();
@ -91,7 +116,7 @@ public class FlatDbStrategyProvider {
// and default to the storage config otherwise
var flatDbModeVal =
existingTrieData
? FlatDbMode.PARTIAL.getVersion()
? alternativeFlatDbModeForExistingDatabase().getVersion()
: requestedFlatDbMode.getVersion();
// persist this config in the db
var setDbModeTx = composedWorldStateStorage.startTransaction();
@ -101,42 +126,11 @@ public class FlatDbStrategyProvider {
return flatDbModeVal;
}));
LOG.info("Bonsai flat db mode found {}", flatDbMode);
LOG.info("Flat db mode found {}", flatDbMode);
return flatDbMode;
}
protected boolean deriveUseCodeStorageByHash(
final SegmentedKeyValueStorage composedWorldStateStorage) {
final boolean configCodeUsingHash =
dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled();
boolean codeUsingCodeByHash =
detectCodeStorageByHash(composedWorldStateStorage)
.map(
dbCodeUsingHash -> {
if (dbCodeUsingHash != configCodeUsingHash) {
LOG.warn(
"Bonsai db is using code storage mode {} but config specifies mode {}. Using mode from database",
dbCodeUsingHash,
configCodeUsingHash);
}
return dbCodeUsingHash;
})
.orElse(configCodeUsingHash);
LOG.info("Bonsai db mode with code stored using code hash enabled = {}", codeUsingCodeByHash);
return codeUsingCodeByHash;
}
private Optional<Boolean> detectCodeStorageByHash(
final SegmentedKeyValueStorage composedWorldStateStorage) {
return composedWorldStateStorage.stream(CODE_STORAGE)
.limit(1)
.findFirst()
.map(
keypair ->
CodeHashCodeStorageStrategy.isCodeHashValue(keypair.getKey(), keypair.getValue()));
}
public FlatDbStrategy getFlatDbStrategy(
final SegmentedKeyValueStorage composedWorldStateStorage) {
if (flatDbStrategy == null) {
@ -145,28 +139,17 @@ public class FlatDbStrategyProvider {
return flatDbStrategy;
}
public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) {
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
LOG.info("setting FlatDbStrategy to FULL");
transaction.put(
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
transaction.commit();
loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy
}
public void downgradeToPartialFlatDbMode(
final SegmentedKeyValueStorage composedWorldStateStorage) {
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
LOG.info("setting FlatDbStrategy to PARTIAL");
transaction.put(
TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe());
transaction.commit();
loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy
}
public FlatDbMode getFlatDbMode() {
return flatDbMode;
}
protected abstract FlatDbMode getRequestedFlatDbMode(
final DataStorageConfiguration dataStorageConfiguration);
protected abstract FlatDbMode alternativeFlatDbModeForExistingDatabase();
protected abstract FlatDbStrategy createFlatDbStrategy(
final FlatDbMode flatDbMode,
final MetricsSystem metricsSystem,
final CodeStorageStrategy codeStorageStrategy);
}

@ -14,6 +14,7 @@
*/
package org.hyperledger.besu.ethereum.worldstate;
import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DiffBasedUnstable;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import org.immutables.value.Value;
@ -22,90 +23,40 @@ import org.immutables.value.Value;
@Value.Enclosing
public interface DataStorageConfiguration {
long DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD = 512;
boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = true;
long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 5_000;
boolean DEFAULT_RECEIPT_COMPACTION_ENABLED = true;
DataStorageConfiguration DEFAULT_CONFIG =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.unstable(Unstable.DEFAULT)
.diffBasedSubStorageConfiguration(DiffBasedSubStorageConfiguration.DEFAULT)
.build();
DataStorageConfiguration DEFAULT_BONSAI_CONFIG =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.build();
DataStorageConfiguration DEFAULT_BONSAI_CONFIG = DEFAULT_CONFIG;
DataStorageConfiguration DEFAULT_BONSAI_PARTIAL_DB_CONFIG =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.unstable(Unstable.DEFAULT_PARTIAL)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.unstable(DiffBasedUnstable.PARTIAL_MODE)
.build())
.build();
DataStorageConfiguration DEFAULT_FOREST_CONFIG =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.FOREST)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.unstable(Unstable.DEFAULT)
.diffBasedSubStorageConfiguration(DiffBasedSubStorageConfiguration.DISABLED)
.build();
DataStorageFormat getDataStorageFormat();
Long getBonsaiMaxLayersToLoad();
@Value.Default
default boolean getBonsaiLimitTrieLogsEnabled() {
return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
}
@Value.Default
default int getBonsaiTrieLogPruningWindowSize() {
return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
default DiffBasedSubStorageConfiguration getDiffBasedSubStorageConfiguration() {
return DiffBasedSubStorageConfiguration.DEFAULT;
}
@Value.Default
default boolean getReceiptCompactionEnabled() {
return DEFAULT_RECEIPT_COMPACTION_ENABLED;
}
@Value.Default
default Unstable getUnstable() {
return Unstable.DEFAULT;
}
@Value.Immutable
interface Unstable {
boolean DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED = true;
boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = true;
boolean DEFAULT_PARALLEL_TRX_ENABLED = false;
DataStorageConfiguration.Unstable DEFAULT =
ImmutableDataStorageConfiguration.Unstable.builder().build();
DataStorageConfiguration.Unstable DEFAULT_PARTIAL =
ImmutableDataStorageConfiguration.Unstable.builder().bonsaiFullFlatDbEnabled(false).build();
@Value.Default
default boolean getBonsaiFullFlatDbEnabled() {
return DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
}
@Value.Default
default boolean getBonsaiCodeStoredByCodeHashEnabled() {
return DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
}
@Value.Default
default boolean isParallelTxProcessingEnabled() {
return DEFAULT_PARALLEL_TRX_ENABLED;
}
}
}

@ -0,0 +1,95 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.worldstate;
import org.immutables.value.Value;
@Value.Immutable
@Value.Enclosing
public interface DiffBasedSubStorageConfiguration {
DiffBasedSubStorageConfiguration DEFAULT =
ImmutableDiffBasedSubStorageConfiguration.builder().build();
DiffBasedSubStorageConfiguration DISABLED =
ImmutableDiffBasedSubStorageConfiguration.builder()
.limitTrieLogsEnabled(false)
.unstable(DiffBasedUnstable.DISABLED)
.build();
long DEFAULT_MAX_LAYERS_TO_LOAD = 512;
boolean DEFAULT_LIMIT_TRIE_LOGS_ENABLED = true;
long MINIMUM_TRIE_LOG_RETENTION_LIMIT = DEFAULT_MAX_LAYERS_TO_LOAD;
int DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE = 5_000;
@Value.Default
default Long getMaxLayersToLoad() {
return DEFAULT_MAX_LAYERS_TO_LOAD;
}
@Value.Default
default boolean getLimitTrieLogsEnabled() {
return DEFAULT_LIMIT_TRIE_LOGS_ENABLED;
}
@Value.Default
default int getTrieLogPruningWindowSize() {
return DEFAULT_TRIE_LOG_PRUNING_WINDOW_SIZE;
}
@Value.Default
default DiffBasedUnstable getUnstable() {
return DiffBasedUnstable.DEFAULT;
}
@Value.Immutable
interface DiffBasedUnstable {
DiffBasedSubStorageConfiguration.DiffBasedUnstable DEFAULT =
ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder().build();
DiffBasedSubStorageConfiguration.DiffBasedUnstable PARTIAL_MODE =
ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder()
.fullFlatDbEnabled(false)
.build();
DiffBasedSubStorageConfiguration.DiffBasedUnstable DISABLED =
ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder()
.fullFlatDbEnabled(false)
.codeStoredByCodeHashEnabled(false)
.isParallelTxProcessingEnabled(false)
.build();
boolean DEFAULT_FULL_FLAT_DB_ENABLED = true;
boolean DEFAULT_CODE_USING_CODE_HASH_ENABLED = true;
boolean DEFAULT_PARALLEL_TRX_ENABLED = false;
@Value.Default
default boolean getFullFlatDbEnabled() {
return DEFAULT_FULL_FLAT_DB_ENABLED;
}
@Value.Default
default boolean getCodeStoredByCodeHashEnabled() {
return DEFAULT_CODE_USING_CODE_HASH_ENABLED;
}
@Value.Default
default boolean isParallelTxProcessingEnabled() {
return DEFAULT_PARALLEL_TRX_ENABLED;
}
}
}

@ -17,7 +17,7 @@ package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
import static org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
@ -472,10 +473,13 @@ public class BonsaiWorldStateKeyValueStorageTest {
new NoOpMetricsSystem(),
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(DEFAULT_MAX_LAYERS_TO_LOAD)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiCodeStoredByCodeHashEnabled(useCodeHashStorage)
ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder()
.codeStoredByCodeHashEnabled(useCodeHashStorage)
.build())
.build())
.build());
}

@ -15,15 +15,17 @@
package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFlatDbStrategyProvider;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFullFlatDbStrategy;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiPartialFlatDbStrategy;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
@ -42,9 +44,10 @@ import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
class FlatDbStrategyProviderTest {
private final FlatDbStrategyProvider flatDbStrategyProvider =
new FlatDbStrategyProvider(new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG);
class BonsaiFlatDbStrategyProviderTest {
private final BonsaiFlatDbStrategyProvider flatDbStrategyProvider =
new BonsaiFlatDbStrategyProvider(
new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG);
private final SegmentedKeyValueStorage composedWorldStateStorage =
new SegmentedInMemoryKeyValueStorage(
List.of(
@ -74,7 +77,7 @@ class FlatDbStrategyProviderTest {
assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL);
assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull();
assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage))
.isInstanceOf(FullFlatDbStrategy.class);
.isInstanceOf(BonsaiFullFlatDbStrategy.class);
assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy)
.isInstanceOf(CodeHashCodeStorageStrategy.class);
}
@ -85,14 +88,17 @@ class FlatDbStrategyProviderTest {
final DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled)
ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder()
.codeStoredByCodeHashEnabled(codeByHashEnabled)
.build())
.build())
.build();
final FlatDbStrategyProvider flatDbStrategyProvider =
new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration);
final BonsaiFlatDbStrategyProvider flatDbStrategyProvider =
new BonsaiFlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration);
flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
final Class<? extends CodeStorageStrategy> expectedCodeStorageClass =
@ -110,14 +116,17 @@ class FlatDbStrategyProviderTest {
final DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled)
ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder()
.codeStoredByCodeHashEnabled(codeByHashEnabled)
.build())
.build())
.build();
final FlatDbStrategyProvider flatDbStrategyProvider =
new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration);
final BonsaiFlatDbStrategyProvider flatDbStrategyProvider =
new BonsaiFlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration);
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
@ -140,14 +149,17 @@ class FlatDbStrategyProviderTest {
final DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
.diffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.builder()
.maxLayersToLoad(DiffBasedSubStorageConfiguration.DEFAULT_MAX_LAYERS_TO_LOAD)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled)
ImmutableDiffBasedSubStorageConfiguration.DiffBasedUnstable.builder()
.codeStoredByCodeHashEnabled(codeByHashEnabled)
.build())
.build())
.build();
final FlatDbStrategyProvider flatDbStrategyProvider =
new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration);
final BonsaiFlatDbStrategyProvider flatDbStrategyProvider =
new BonsaiFlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration);
final SegmentedKeyValueStorageTransaction transaction =
composedWorldStateStorage.startTransaction();
@ -171,7 +183,7 @@ class FlatDbStrategyProviderTest {
assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL);
assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull();
assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage))
.isInstanceOf(PartialFlatDbStrategy.class);
.isInstanceOf(BonsaiPartialFlatDbStrategy.class);
}
private void updateFlatDbMode(final FlatDbMode flatDbMode) {

@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.CompactEncoding;
import org.hyperledger.besu.ethereum.trie.MerkleTrie;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.BonsaiFlatDbStrategyProvider;
import org.hyperledger.besu.ethereum.trie.patricia.SimpleMerklePatriciaTrie;
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
@ -85,7 +85,8 @@ public class SnapServerTest {
// force a full flat db with code stored by code hash:
final BonsaiWorldStateKeyValueStorage inMemoryStorage =
new BonsaiWorldStateKeyValueStorage(
new FlatDbStrategyProvider(noopMetrics, DataStorageConfiguration.DEFAULT_BONSAI_CONFIG) {
new BonsaiFlatDbStrategyProvider(
noopMetrics, DataStorageConfiguration.DEFAULT_BONSAI_CONFIG) {
@Override
public FlatDbMode getFlatDbMode() {
return FlatDbMode.FULL;

Loading…
Cancel
Save