CipherLabs
/
besu
mirror of https://github.com/hyperledger/besu
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reuse --bonsai-historical-block-limit for limit trie logs feature (#6445)

Browse Source 
Renames and tidy-up
Add change log for feature
---------

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>
pull/6453/head
Simon Dudley 10 months ago committed by GitHub
parent 3fc3fb1225
commit b90d0ed521
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 356 additions and 259 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      CHANGELOG.md
  2. 12
      besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
  3. 38
      besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java
  4. 79
      besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
  5. 73
      besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java
  6. 25
      besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java
  7. 6
      besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
  8. 26
      besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java
  9. 54
      besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java
  10. 280
      besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java
  11. 20
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java

2
CHANGELOG.md
Unescape View File

@ -35,6 +35,8 @@
- Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397)
- Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414)
- Options to tune the max allowed time that can be spent selecting transactions during block creation are now stable [#6423](https://github.com/hyperledger/besu/pull/6423)
- Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390)
- Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303)
### Bug fixes
- INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344)

12
besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
Unescape View File

@ -3555,12 +3555,12 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
builder.setHighSpecEnabled();
}
if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningEnabled()) {
builder.setTrieLogPruningEnabled();
builder.setTrieLogRetentionThreshold(
dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogRetentionThreshold());
builder.setTrieLogPruningLimit(
dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningLimit());
if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiLimitTrieLogsEnabled()) {
builder.setLimitTrieLogsEnabled();
builder.setTrieLogRetentionLimit(
dataStorageOptions.toDomainObject().getBonsaiMaxLayersToLoad());
builder.setTrieLogsPruningWindowSize(
dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningWindowSize());
}
builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation());

38
besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java
Unescape View File

@ -52,9 +52,9 @@ public class ConfigurationOverviewBuilder {
private Collection<String> engineApis;
private String engineJwtFilePath;
private boolean isHighSpec = false;
private boolean isTrieLogPruningEnabled = false;
private long trieLogRetentionThreshold = 0;
private Integer trieLogPruningLimit = null;
private boolean isBonsaiLimitTrieLogsEnabled = false;
private long trieLogRetentionLimit = 0;
private Integer trieLogsPruningWindowSize = null;
private TransactionPoolConfiguration.Implementation txPoolImplementation;
private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode;
private Map<String, String> environment;
@ -199,34 +199,34 @@ public class ConfigurationOverviewBuilder {
}
/**
* Sets trie log pruning enabled
* Sets limit trie logs enabled
*
* @return the builder
*/
public ConfigurationOverviewBuilder setTrieLogPruningEnabled() {
isTrieLogPruningEnabled = true;
public ConfigurationOverviewBuilder setLimitTrieLogsEnabled() {
isBonsaiLimitTrieLogsEnabled = true;
return this;
}
/**
* Sets trie log retention threshold
* Sets trie log retention limit
*
* @param threshold the number of blocks to retain trie logs for
* @param limit the number of blocks to retain trie logs for
* @return the builder
*/
public ConfigurationOverviewBuilder setTrieLogRetentionThreshold(final long threshold) {
trieLogRetentionThreshold = threshold;
public ConfigurationOverviewBuilder setTrieLogRetentionLimit(final long limit) {
trieLogRetentionLimit = limit;
return this;
}
/**
* Sets trie log pruning limit
* Sets trie logs pruning window size
*
* @param limit the max number of blocks to load and prune trie logs for at startup
* @param size the max number of blocks to load and prune trie logs for at startup
* @return the builder
*/
public ConfigurationOverviewBuilder setTrieLogPruningLimit(final int limit) {
trieLogPruningLimit = limit;
public ConfigurationOverviewBuilder setTrieLogsPruningWindowSize(final int size) {
trieLogsPruningWindowSize = size;
return this;
}
@ -339,13 +339,13 @@ public class ConfigurationOverviewBuilder {
lines.add("Using " + worldStateUpdateMode + " worldstate update mode");
if (isTrieLogPruningEnabled) {
if (isBonsaiLimitTrieLogsEnabled) {
final StringBuilder trieLogPruningString = new StringBuilder();
trieLogPruningString
.append("Trie log pruning enabled: retention: ")
.append(trieLogRetentionThreshold);
if (trieLogPruningLimit != null) {
trieLogPruningString.append("; prune limit: ").append(trieLogPruningLimit);
.append("Limit trie logs enabled: retention: ")
.append(trieLogRetentionLimit);
if (trieLogsPruningWindowSize != null) {
trieLogPruningString.append("; prune window: ").append(trieLogsPruningWindowSize);
}
lines.add(trieLogPruningString.toString());
}

79
besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
Unescape View File

@ -17,10 +17,9 @@
package org.hyperledger.besu.cli.options.stable;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
import org.hyperledger.besu.cli.options.CLIOptions;
import org.hyperledger.besu.cli.util.CommandLineUtils;
@ -39,7 +38,8 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
private static final String DATA_STORAGE_FORMAT = "--data-storage-format";
private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD =
/** The maximum number of historical layers to load. */
public static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD =
"--bonsai-historical-block-limit";
// Use Bonsai DB
@ -54,39 +54,37 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"},
paramLabel = "<LONG>",
description =
"Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}).",
"Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using "
+ Unstable.BONSAI_LIMIT_TRIE_LOGS_ENABLED
+ " it will also be used as the number of layers of trie logs to retain.",
arity = "1")
private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
@CommandLine.ArgGroup(validate = false)
private final DataStorageOptions.Unstable unstableOptions = new Unstable();
static class Unstable {
/** The unstable options for data storage. */
public static class Unstable {
private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED =
"--Xbonsai-limit-trie-logs-enabled";
private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD =
"--Xbonsai-trie-logs-retention-threshold";
private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit";
@CommandLine.Option(
hidden = true,
names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED},
description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})")
private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
/** The bonsai trie logs pruning window size. */
public static final String BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE =
"--Xbonsai-trie-logs-pruning-window-size";
@CommandLine.Option(
hidden = true,
names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD},
names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED},
description =
"The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})")
private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
"Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})")
private boolean bonsaiLimitTrieLogsEnabled = DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
@CommandLine.Option(
hidden = true,
names = {BONSAI_TRIE_LOG_PRUNING_LIMIT},
names = {BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE},
description =
"The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;
private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
}
/**
* Create data storage options.
@ -103,21 +101,31 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
* @param commandLine the full commandLine to check all the options specified by the user
*/
public void validate(final CommandLine commandLine) {
if (unstableOptions.bonsaiTrieLogPruningEnabled) {
if (unstableOptions.bonsaiTrieLogRetentionThreshold
< MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) {
if (unstableOptions.bonsaiLimitTrieLogsEnabled) {
if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d",
MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT));
}
if (unstableOptions.bonsaiTrieLogPruningWindowSize <= 0) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
"--Xbonsai-trie-log-retention-threshold minimum value is %d",
MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD));
Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0",
unstableOptions.bonsaiTrieLogPruningWindowSize));
}
if (unstableOptions.bonsaiTrieLogPruningLimit <= 0) {
if (unstableOptions.bonsaiTrieLogPruningWindowSize <= bonsaiMaxLayersToLoad) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
"--Xbonsai-trie-log-pruning-limit=%d must be greater than 0",
unstableOptions.bonsaiTrieLogPruningLimit));
Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE
+ "=%d must be greater than "
+ BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD
+ "=%d",
unstableOptions.bonsaiTrieLogPruningWindowSize,
bonsaiMaxLayersToLoad));
}
}
}
@ -126,12 +134,10 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat();
dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad();
dataStorageOptions.unstableOptions.bonsaiTrieLogPruningEnabled =
domainObject.getUnstable().getBonsaiTrieLogPruningEnabled();
dataStorageOptions.unstableOptions.bonsaiTrieLogRetentionThreshold =
domainObject.getUnstable().getBonsaiTrieLogRetentionThreshold();
dataStorageOptions.unstableOptions.bonsaiTrieLogPruningLimit =
domainObject.getUnstable().getBonsaiTrieLogPruningLimit();
dataStorageOptions.unstableOptions.bonsaiLimitTrieLogsEnabled =
domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled();
dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize =
domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize();
return dataStorageOptions;
}
@ -143,9 +149,8 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
.bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiTrieLogPruningEnabled(unstableOptions.bonsaiTrieLogPruningEnabled)
.bonsaiTrieLogRetentionThreshold(unstableOptions.bonsaiTrieLogRetentionThreshold)
.bonsaiTrieLogPruningLimit(unstableOptions.bonsaiTrieLogPruningLimit)
.bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled)
.bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize)
.build())
.build();
}

73
besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java
Unescape View File

@ -18,6 +18,7 @@ package org.hyperledger.besu.cli.subcommands.storage;
import static com.google.common.base.Preconditions.checkArgument;
import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH;
import org.hyperledger.besu.cli.options.stable.DataStorageOptions;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
@ -44,6 +45,7 @@ import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.annotations.VisibleForTesting;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.slf4j.Logger;
@ -56,7 +58,7 @@ public class TrieLogHelper {
private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000;
private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class);
static void prune(
void prune(
final DataStorageConfiguration config,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final MutableBlockchain blockchain,
@ -66,7 +68,7 @@ public class TrieLogHelper {
validatePruneConfiguration(config);
final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold();
final long layersToRetain = config.getBonsaiMaxLayersToLoad();
final long chainHeight = blockchain.getChainHeadBlockNumber();
@ -94,7 +96,7 @@ public class TrieLogHelper {
}
}
private static void processTrieLogBatches(
private void processTrieLogBatches(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final MutableBlockchain blockchain,
final long chainHeight,
@ -122,7 +124,7 @@ public class TrieLogHelper {
}
}
private static void saveTrieLogBatches(
private void saveTrieLogBatches(
final String batchFileName,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final List<Hash> trieLogKeys) {
@ -135,7 +137,7 @@ public class TrieLogHelper {
}
}
private static void restoreTrieLogBatches(
private void restoreTrieLogBatches(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final long batchNumber,
final String batchFileNameBase) {
@ -149,7 +151,7 @@ public class TrieLogHelper {
}
}
private static void deleteFiles(final String batchFileNameBase, final long numberOfBatches) {
private void deleteFiles(final String batchFileNameBase, final long numberOfBatches) {
LOG.info("Deleting files...");
@ -161,7 +163,7 @@ public class TrieLogHelper {
}
}
private static List<Hash> getTrieLogKeysForBlocks(
private List<Hash> getTrieLogKeysForBlocks(
final MutableBlockchain blockchain,
final long firstBlockOfBatch,
final long lastBlockOfBatch) {
@ -175,11 +177,11 @@ public class TrieLogHelper {
return trieLogKeys;
}
private static long calculateNumberofBatches(final long layersToRetain) {
private long calculateNumberofBatches(final long layersToRetain) {
return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1);
}
private static boolean validPruneRequirements(
private boolean validPruneRequirements(
final MutableBlockchain blockchain,
final long chainHeight,
final long lastBlockNumberToRetainTrieLogsFor) {
@ -206,7 +208,7 @@ public class TrieLogHelper {
return true;
}
private static void recreateTrieLogs(
private void recreateTrieLogs(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final long batchNumber,
final String batchFileNameBase)
@ -222,7 +224,7 @@ public class TrieLogHelper {
}
}
private static void processTransactionChunk(
private void processTransactionChunk(
final int startIndex,
final int chunkSize,
final List<byte[]> keys,
@ -242,28 +244,33 @@ public class TrieLogHelper {
updater.getTrieLogStorageTransaction().commit();
}
private static void validatePruneConfiguration(final DataStorageConfiguration config) {
@VisibleForTesting
void validatePruneConfiguration(final DataStorageConfiguration config) {
checkArgument(
config.getUnstable().getBonsaiTrieLogRetentionThreshold()
>= config.getBonsaiMaxLayersToLoad(),
config.getBonsaiMaxLayersToLoad()
>= DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT,
String.format(
"--Xbonsai-trie-log-retention-threshold minimum value is %d",
config.getBonsaiMaxLayersToLoad()));
DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d",
DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT));
checkArgument(
config.getUnstable().getBonsaiTrieLogPruningLimit() > 0,
config.getUnstable().getBonsaiTrieLogPruningWindowSize() > 0,
String.format(
"--Xbonsai-trie-log-pruning-limit=%d must be greater than 0",
config.getUnstable().getBonsaiTrieLogPruningLimit()));
DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE
+ "=%d must be greater than 0",
config.getUnstable().getBonsaiTrieLogPruningWindowSize()));
checkArgument(
config.getUnstable().getBonsaiTrieLogPruningLimit()
> config.getUnstable().getBonsaiTrieLogRetentionThreshold(),
config.getUnstable().getBonsaiTrieLogPruningWindowSize()
> config.getBonsaiMaxLayersToLoad(),
String.format(
"--Xbonsai-trie-log-pruning-limit=%d must greater than --Xbonsai-trie-log-retention-threshold=%d",
config.getUnstable().getBonsaiTrieLogPruningLimit(),
config.getUnstable().getBonsaiTrieLogRetentionThreshold()));
DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE
+ "=%d must be greater than "
+ DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD
+ "=%d",
config.getUnstable().getBonsaiTrieLogPruningWindowSize(),
config.getBonsaiMaxLayersToLoad()));
}
private static void saveTrieLogsInFile(
private void saveTrieLogsInFile(
final List<Hash> trieLogsKeys,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final String batchFileName)
@ -285,7 +292,7 @@ public class TrieLogHelper {
}
@SuppressWarnings("unchecked")
static IdentityHashMap<byte[], byte[]> readTrieLogsFromFile(final String batchFileName) {
IdentityHashMap<byte[], byte[]> readTrieLogsFromFile(final String batchFileName) {
IdentityHashMap<byte[], byte[]> trieLogs;
try (FileInputStream fis = new FileInputStream(batchFileName);
@ -300,7 +307,7 @@ public class TrieLogHelper {
return trieLogs;
}
private static void saveTrieLogsAsRlpInFile(
private void saveTrieLogsAsRlpInFile(
final List<Hash> trieLogsKeys,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final String batchFileName) {
@ -325,7 +332,7 @@ public class TrieLogHelper {
}
}
static IdentityHashMap<byte[], byte[]> readTrieLogsAsRlpFromFile(final String batchFileName) {
IdentityHashMap<byte[], byte[]> readTrieLogsAsRlpFromFile(final String batchFileName) {
try {
final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName)));
final BytesValueRLPInput input = new BytesValueRLPInput(file, false);
@ -346,7 +353,7 @@ public class TrieLogHelper {
}
}
private static IdentityHashMap<byte[], byte[]> getTrieLogs(
private IdentityHashMap<byte[], byte[]> getTrieLogs(
final List<Hash> trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) {
IdentityHashMap<byte[], byte[]> trieLogsToRetain = new IdentityHashMap<>();
@ -359,7 +366,7 @@ public class TrieLogHelper {
return trieLogsToRetain;
}
static TrieLogCount getCount(
TrieLogCount getCount(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final int limit,
final Blockchain blockchain) {
@ -394,13 +401,13 @@ public class TrieLogHelper {
return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get());
}
static void printCount(final PrintWriter out, final TrieLogCount count) {
void printCount(final PrintWriter out, final TrieLogCount count) {
out.printf(
"trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n",
count.total, count.canonicalCount, count.forkCount, count.orphanCount);
}
static void importTrieLog(
void importTrieLog(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) {
var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString());
@ -410,7 +417,7 @@ public class TrieLogHelper {
updater.getTrieLogStorageTransaction().commit();
}
static void exportTrieLog(
void exportTrieLog(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final List<Hash> trieLogHash,
final Path directoryPath)

25
besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java
Unescape View File

@ -89,14 +89,15 @@ public class TrieLogSubCommand implements Runnable {
@Override
public void run() {
TrieLogContext context = getTrieLogContext();
final TrieLogContext context = getTrieLogContext();
final PrintWriter out = spec.commandLine().getOut();
out.println("Counting trie logs...");
TrieLogHelper.printCount(
final TrieLogHelper trieLogHelper = new TrieLogHelper();
trieLogHelper.printCount(
out,
TrieLogHelper.getCount(
trieLogHelper.getCount(
context.rootWorldStateStorage, Integer.MAX_VALUE, context.blockchain));
}
}
@ -104,7 +105,7 @@ public class TrieLogSubCommand implements Runnable {
@Command(
name = "prune",
description =
"This command prunes all trie log layers below the retention threshold, including orphaned trie logs.",
"This command prunes all trie log layers below the retention limit, including orphaned trie logs.",
mixinStandardHelpOptions = true,
versionProvider = VersionProvider.class)
static class PruneTrieLog implements Runnable {
@ -119,11 +120,12 @@ public class TrieLogSubCommand implements Runnable {
@Override
public void run() {
TrieLogContext context = getTrieLogContext();
final TrieLogContext context = getTrieLogContext();
final Path dataDirectoryPath =
Paths.get(
TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString());
TrieLogHelper.prune(
final TrieLogHelper trieLogHelper = new TrieLogHelper();
trieLogHelper.prune(
context.config(),
context.rootWorldStateStorage(),
context.blockchain(),
@ -146,6 +148,7 @@ public class TrieLogSubCommand implements Runnable {
@CommandLine.Spec
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
@SuppressWarnings("unused")
@CommandLine.Option(
names = "--trie-log-block-hash",
description =
@ -173,13 +176,15 @@ public class TrieLogSubCommand implements Runnable {
.toString());
}
TrieLogContext context = getTrieLogContext();
final TrieLogContext context = getTrieLogContext();
final List<Hash> listOfBlockHashes =
trieLogBlockHashList.stream().map(Hash::fromHexString).toList();
final TrieLogHelper trieLogHelper = new TrieLogHelper();
try {
TrieLogHelper.exportTrieLog(
trieLogHelper.exportTrieLog(
context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath);
} catch (IOException e) {
throw new RuntimeException(e);
@ -222,8 +227,8 @@ public class TrieLogSubCommand implements Runnable {
}
TrieLogContext context = getTrieLogContext();
TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath);
final TrieLogHelper trieLogHelper = new TrieLogHelper();
trieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath);
}
}

6
besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
Unescape View File

@ -782,7 +782,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
final JsonRpcMethods additionalJsonRpcMethodFactory =
createAdditionalJsonRpcMethodFactory(protocolContext);
if (dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled()
if (dataStorageConfiguration.getUnstable().getBonsaiLimitTrieLogsEnabled()
&& DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) {
final TrieLogManager trieLogManager =
((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager();
@ -831,8 +831,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
(BonsaiWorldStateKeyValueStorage) worldStateStorage,
blockchain,
scheduler::executeServiceTask,
dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(),
dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(),
dataStorageConfiguration.getBonsaiMaxLayersToLoad(),
dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningWindowSize(),
isProofOfStake);
trieLogPruner.initialize();

26
besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java
Unescape View File

@ -152,21 +152,21 @@ class ConfigurationOverviewBuilderTest {
}
@Test
void setTrieLogPruningEnabled() {
final String noTrieLogRetentionThresholdSet = builder.build();
assertThat(noTrieLogRetentionThresholdSet).doesNotContain("Trie log pruning enabled");
builder.setTrieLogPruningEnabled();
builder.setTrieLogRetentionThreshold(42);
String trieLogRetentionThresholdSet = builder.build();
assertThat(trieLogRetentionThresholdSet)
.contains("Trie log pruning enabled")
void setBonsaiLimitTrieLogsEnabled() {
final String noTrieLogRetentionLimitSet = builder.build();
assertThat(noTrieLogRetentionLimitSet).doesNotContain("Limit trie logs enabled");
builder.setLimitTrieLogsEnabled();
builder.setTrieLogRetentionLimit(42);
String trieLogRetentionLimitSet = builder.build();
assertThat(trieLogRetentionLimitSet)
.contains("Limit trie logs enabled")
.contains("retention: 42");
assertThat(trieLogRetentionThresholdSet).doesNotContain("prune limit");
assertThat(trieLogRetentionLimitSet).doesNotContain("prune window");
builder.setTrieLogPruningLimit(1000);
trieLogRetentionThresholdSet = builder.build();
assertThat(trieLogRetentionThresholdSet).contains("prune limit: 1000");
builder.setTrieLogsPruningWindowSize(1000);
trieLogRetentionLimitSet = builder.build();
assertThat(trieLogRetentionLimitSet).contains("prune window: 1000");
}
@Test

54
besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java
Unescape View File

@ -16,7 +16,7 @@
package org.hyperledger.besu.cli.options.stable;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
import org.hyperledger.besu.cli.options.AbstractCLIOptionsTest;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
@ -32,50 +32,59 @@ public class DataStorageOptionsTest
public void bonsaiTrieLogPruningLimitOption() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit())
.isEqualTo(1),
assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningWindowSize())
.isEqualTo(600),
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-pruning-limit",
"1");
"--Xbonsai-trie-logs-pruning-window-size",
"600");
}
@Test
public void bonsaiTrieLogPruningLimitShouldBePositive() {
public void bonsaiTrieLogPruningWindowSizeShouldBePositive() {
internalTestFailure(
"--Xbonsai-trie-log-pruning-limit=0 must be greater than 0",
"--Xbonsai-trie-logs-pruning-window-size=0 must be greater than 0",
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-pruning-limit",
"--Xbonsai-trie-logs-pruning-window-size",
"0");
}
@Test
public void bonsaiTrieLogRetentionThresholdOption() {
public void bonsaiTrieLogPruningWindowSizeShouldBeAboveRetentionLimit() {
internalTestFailure(
"--Xbonsai-trie-logs-pruning-window-size=512 must be greater than --bonsai-historical-block-limit=512",
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-pruning-window-size",
"512");
}
@Test
public void bonsaiTrieLogRetentionLimitOption() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold())
.isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1),
assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad())
.isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT + 1),
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-retention-threshold",
"--bonsai-historical-block-limit",
"513");
}
@Test
public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() {
public void bonsaiTrieLogRetentionLimitOption_boundaryTest() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold())
.isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD),
assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad())
.isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT),
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-retention-threshold",
"--bonsai-historical-block-limit",
"512");
}
@Test
public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() {
public void bonsaiTrieLogRetentionLimitShouldBeAboveMinimum() {
internalTestFailure(
"--Xbonsai-trie-log-retention-threshold minimum value is 512",
"--bonsai-historical-block-limit minimum value is 512",
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-retention-threshold",
"--bonsai-historical-block-limit",
"511");
}
@ -88,12 +97,11 @@ public class DataStorageOptionsTest
protected DataStorageConfiguration createCustomizedDomainObject() {
return ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(DataStorageFormat.BONSAI)
.bonsaiMaxLayersToLoad(100L)
.bonsaiMaxLayersToLoad(513L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiTrieLogPruningEnabled(true)
.bonsaiTrieLogRetentionThreshold(1000L)
.bonsaiTrieLogPruningLimit(20)
.bonsaiLimitTrieLogsEnabled(true)
.bonsaiTrieLogPruningWindowSize(514)
.build())
.build();
}

280
besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java
Unescape View File

@ -16,10 +16,9 @@
package org.hyperledger.besu.cli.subcommands.storage;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
@ -37,6 +36,7 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -58,13 +58,19 @@ class TrieLogHelperTest {
private static final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider();
private static BonsaiWorldStateKeyValueStorage inMemoryWorldState;
private TrieLogHelper trieLogHelper;
@Mock private MutableBlockchain blockchain;
private static class NonValidatingTrieLogHelper extends TrieLogHelper {
@Override
void validatePruneConfiguration(final DataStorageConfiguration config) {}
}
@Mock private MutableBlockchain blockchain;
static BlockHeader blockHeader1;
static BlockHeader blockHeader2;
static BlockHeader blockHeader3;
static BlockHeader blockHeader4;
static BlockHeader blockHeader5;
@BeforeEach
@ -99,6 +105,8 @@ class TrieLogHelperTest {
.getTrieLogStorageTransaction()
.put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5));
updater.getTrieLogStorageTransaction().commit();
trieLogHelper = new NonValidatingTrieLogHelper();
}
private static byte[] createTrieLog(final BlockHeader blockHeader) {
@ -122,12 +130,11 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(2L)
.bonsaiMaxLayersToLoad(3L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiTrieLogRetentionThreshold(3)
.build()
.withBonsaiTrieLogRetentionThreshold(3))
.bonsaiLimitTrieLogsEnabled(true)
.build())
.build();
mockBlockchainBase();
@ -136,73 +143,75 @@ class TrieLogHelperTest {
when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3));
// assert trie logs that will be pruned exist before prune call
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3));
assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get())
.isEqualTo(createTrieLog(blockHeader1));
assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get())
.isEqualTo(createTrieLog(blockHeader2));
assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get())
.isEqualTo(createTrieLog(blockHeader3));
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir);
trieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir);
// assert pruned trie logs are not in the DB
assertEquals(inMemoryWorldState.getTrieLog(blockHeader1.getHash()), Optional.empty());
assertEquals(inMemoryWorldState.getTrieLog(blockHeader2.getHash()), Optional.empty());
assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash())).isEqualTo(Optional.empty());
assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash())).isEqualTo(Optional.empty());
// assert retained trie logs are in the DB
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5));
assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get())
.isEqualTo(createTrieLog(blockHeader3));
assertThat(inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get())
.isEqualTo(createTrieLog(blockHeader4));
assertThat(inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get())
.isEqualTo(createTrieLog(blockHeader5));
}
@Test
public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) {
public void cannotPruneIfNoFinalizedIsFound() {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(2L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiTrieLogRetentionThreshold(2)
.build()
.withBonsaiTrieLogRetentionThreshold(2))
.bonsaiLimitTrieLogsEnabled(true)
.build())
.build();
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L);
when(blockchain.getFinalized()).thenReturn(Optional.empty());
assertThrows(
RuntimeException.class,
() ->
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir));
assertThatThrownBy(
() ->
trieLogHelper.prune(
dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of("")))
.isInstanceOf(RuntimeException.class)
.hasMessage("No finalized block present, can't safely run trie log prune");
}
@Test
public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDir Path dataDir) {
public void cannotPruneIfUserRetainsMoreLayersThanExistingChainLength() {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(2L)
.bonsaiMaxLayersToLoad(10L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiTrieLogRetentionThreshold(10)
.build()
.withBonsaiTrieLogRetentionThreshold(10))
.bonsaiLimitTrieLogsEnabled(true)
.build())
.build();
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L);
assertThrows(
IllegalArgumentException.class,
() ->
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir));
assertThatThrownBy(
() ->
trieLogHelper.prune(
dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of("")))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Trying to retain more trie logs than chain length (5), skipping pruning");
}
@Test
public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) {
public void cannotPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
@ -210,17 +219,84 @@ class TrieLogHelperTest {
.bonsaiMaxLayersToLoad(2L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiTrieLogRetentionThreshold(2)
.build()
.withBonsaiTrieLogRetentionThreshold(2))
.bonsaiLimitTrieLogsEnabled(true)
.build())
.build();
mockBlockchainBase();
assertThrows(
IllegalArgumentException.class,
() ->
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir));
assertThatThrownBy(
() ->
trieLogHelper.prune(
dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage(
"Trying to prune more layers than the finalized block height, skipping pruning");
}
@Test
public void trieLogRetentionLimitShouldBeAboveMinimum() {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(511L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiLimitTrieLogsEnabled(true)
.build())
.build();
TrieLogHelper helper = new TrieLogHelper();
assertThatThrownBy(
() ->
helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of("")))
.isInstanceOf(RuntimeException.class)
.hasMessage("--bonsai-historical-block-limit minimum value is 512");
}
@Test
public void trieLogPruningWindowSizeShouldBePositive() {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(512L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiLimitTrieLogsEnabled(true)
.bonsaiTrieLogPruningWindowSize(0)
.build())
.build();
TrieLogHelper helper = new TrieLogHelper();
assertThatThrownBy(
() ->
helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of("")))
.isInstanceOf(RuntimeException.class)
.hasMessage("--Xbonsai-trie-logs-pruning-window-size=0 must be greater than 0");
}
@Test
public void trieLogPruningWindowSizeShouldBeAboveRetentionLimit() {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(512L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiLimitTrieLogsEnabled(true)
.bonsaiTrieLogPruningWindowSize(512)
.build())
.build();
TrieLogHelper helper = new TrieLogHelper();
assertThatThrownBy(
() ->
helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of("")))
.isInstanceOf(RuntimeException.class)
.hasMessage(
"--Xbonsai-trie-logs-pruning-window-size=512 must be greater than --bonsai-historical-block-limit=512");
}
@Test
@ -229,77 +305,81 @@ class TrieLogHelperTest {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.bonsaiMaxLayersToLoad(2L)
.bonsaiMaxLayersToLoad(3L)
.unstable(
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiTrieLogRetentionThreshold(2)
.build()
.withBonsaiTrieLogRetentionThreshold(2))
.bonsaiLimitTrieLogsEnabled(true)
.build())
.build();
assertThrows(
RuntimeException.class,
() ->
TrieLogHelper.prune(
dataStorageConfiguration,
inMemoryWorldState,
blockchain,
dataDir.resolve("unknownPath")));
mockBlockchainBase();
when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5));
when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4));
when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3));
assertThatThrownBy(
() ->
trieLogHelper.prune(
dataStorageConfiguration,
inMemoryWorldState,
blockchain,
dataDir.resolve("unknownPath")))
.isInstanceOf(RuntimeException.class)
.hasCauseExactlyInstanceOf(FileNotFoundException.class);
// assert all trie logs are still in the DB
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5));
assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get())
.isEqualTo(createTrieLog(blockHeader1));
assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get())
.isEqualTo(createTrieLog(blockHeader2));
assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get())
.isEqualTo(createTrieLog(blockHeader3));
assertThat(inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get())
.isEqualTo(createTrieLog(blockHeader4));
assertThat(inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get())
.isEqualTo(createTrieLog(blockHeader5));
}
@Test
public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException {
TrieLogHelper.exportTrieLog(
trieLogHelper.exportTrieLog(
inMemoryWorldState,
singletonList(blockHeader1.getHash()),
dataDir.resolve("trie-log-dump"));
var trieLog =
TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString())
trieLogHelper
.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString())
.entrySet()
.stream()
.findFirst()
.get();
assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe());
assertArrayEquals(
trieLog.getValue(), inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
assertThat(trieLog.getKey()).isEqualTo(blockHeader1.getHash().toArrayUnsafe());
assertThat(trieLog.getValue())
.isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
}
@Test
public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException {
TrieLogHelper.exportTrieLog(
trieLogHelper.exportTrieLog(
inMemoryWorldState,
List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()),
dataDir.resolve("trie-log-dump"));
var trieLogs =
TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString())
trieLogHelper
.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString())
.entrySet()
.stream()
.collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue));
assertArrayEquals(
trieLogs.get(blockHeader1.getHash()),
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
assertArrayEquals(
trieLogs.get(blockHeader2.getHash()),
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get());
assertArrayEquals(
trieLogs.get(blockHeader3.getHash()),
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get());
assertThat(trieLogs.get(blockHeader1.getHash()))
.isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
assertThat(trieLogs.get(blockHeader2.getHash()))
.isEqualTo(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get());
assertThat(trieLogs.get(blockHeader3.getHash()))
.isEqualTo(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get());
}
@Test
@ -309,22 +389,21 @@ class TrieLogHelperTest {
new BonsaiWorldStateKeyValueStorage(
tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG);
TrieLogHelper.exportTrieLog(
trieLogHelper.exportTrieLog(
inMemoryWorldState,
singletonList(blockHeader1.getHash()),
dataDir.resolve("trie-log-dump"));
var trieLog =
TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString());
trieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString());
var updater = inMemoryWorldState2.updater();
trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v));
updater.getTrieLogStorageTransaction().commit();
assertArrayEquals(
inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(),
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
assertThat(inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get())
.isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
}
@Test
@ -334,27 +413,24 @@ class TrieLogHelperTest {
new BonsaiWorldStateKeyValueStorage(
tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG);
TrieLogHelper.exportTrieLog(
trieLogHelper.exportTrieLog(
inMemoryWorldState,
List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()),
dataDir.resolve("trie-log-dump"));
var trieLog =
TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString());
trieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString());
var updater = inMemoryWorldState2.updater();
trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v));
updater.getTrieLogStorageTransaction().commit();
assertArrayEquals(
inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(),
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
assertArrayEquals(
inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get(),
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get());
assertArrayEquals(
inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get(),
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get());
assertThat(inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get())
.isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
assertThat(inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get())
.isEqualTo(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get());
assertThat(inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get())
.isEqualTo(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get());
}
}

20
ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java
Unescape View File

@ -43,27 +43,21 @@ public interface DataStorageConfiguration {
@Value.Immutable
interface Unstable {
boolean DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED = false;
long DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = 512L;
long MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
int DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT = 30_000;
boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false;
long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000;
DataStorageConfiguration.Unstable DEFAULT =
ImmutableDataStorageConfiguration.Unstable.builder().build();
@Value.Default
default boolean getBonsaiTrieLogPruningEnabled() {
return DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
default boolean getBonsaiLimitTrieLogsEnabled() {
return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
}
@Value.Default
default long getBonsaiTrieLogRetentionThreshold() {
return DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
}
@Value.Default
default int getBonsaiTrieLogPruningLimit() {
return DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;
default int getBonsaiTrieLogPruningWindowSize() {
return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
}
}
}

Write Preview
Loading…
Cancel
Save