diff --git a/.circleci/config.yml b/.circleci/config.yml index 065a0280d0..0e84ab22aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -204,7 +204,7 @@ jobs: at: ~/project - run: name: ReferenceTests - no_output_timeout: 30m + no_output_timeout: 40m command: | git submodule update --init --recursive ./gradlew --no-daemon referenceTest diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f84aa88b7..57416d0785 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,32 @@ # Changelog -## 24.1.1-SNAPSHOT +## 24.1.2-SNAPSHOT + +### Breaking Changes +- The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` [#6446](https://github.com/hyperledger/besu/pull/6446) + +### Deprecations + +### Additions and Improvements +- Add `OperationTracer.tracePrepareTransaction`, where the sender account has not yet been altered[#6453](https://github.com/hyperledger/besu/pull/6453) +- Improve the high spec flag by limiting it to a few column families [#6354](https://github.com/hyperledger/besu/pull/6354) + + +### Bug fixes +- Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) +- Fix `poa-block-txs-selection-max-time` option that was inadvertently reset to its default after being configured [#6444](https://github.com/hyperledger/besu/pull/6444) + +### Download Links + +## 24.1.1 ### Breaking Changes - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) -- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. +- Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): + - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. ### Deprecations @@ -20,6 +40,8 @@ - Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) - Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414) - Options to tune the max allowed time that can be spent selecting transactions during block creation are now stable [#6423](https://github.com/hyperledger/besu/pull/6423) +- Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390) +- Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) @@ -30,7 +52,6 @@ ### Download Links - ## 24.1.0 ### Breaking Changes @@ -44,6 +65,7 @@ - Set Ethereum Classic mainnet activation block for Spiral network upgrade [#6267](https://github.com/hyperledger/besu/pull/6267) - Add custom genesis file name to config overview if specified [#6297](https://github.com/hyperledger/besu/pull/6297) - Update Gradle plugins and replace unmaintained License Gradle Plugin with the actively maintained Gradle License Report [#6275](https://github.com/hyperledger/besu/pull/6275) +- Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) ### Bug fixes - Hotfix for selfdestruct preimages on bonsai [#6359]((https://github.com/hyperledger/besu/pull/6359) @@ -87,7 +109,6 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3-hotfix/besu- ### Bug fixes - Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194) - Fix `logIndex` in `eth_getTransactionReceipt` JSON RPC method [#6206](https://github.com/hyperledger/besu/pull/6206) -- Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) ### Download Links https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3/besu-23.10.3.zip / sha256 da7ef8a6ceb88d3e327cacddcdb32218d1750b464c14165a74068f6dc6e0871a diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 7f276c704c..b9a137cbc1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -31,7 +31,6 @@ import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEF import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.VALID_APIS; import static org.hyperledger.besu.ethereum.api.jsonrpc.authentication.EngineAuthService.EPHEMERAL_JWT_FILE; -import static org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration.DEFAULT_WEBSOCKET_PORT; import static org.hyperledger.besu.metrics.BesuMetricCategory.DEFAULT_METRIC_CATEGORIES; import static org.hyperledger.besu.metrics.MetricsProtocol.PROMETHEUS; import static org.hyperledger.besu.metrics.prometheus.MetricsConfiguration.DEFAULT_METRICS_PORT; @@ -46,6 +45,7 @@ import org.hyperledger.besu.chainimport.JsonBlockImporter; import org.hyperledger.besu.chainimport.RlpBlockImporter; import org.hyperledger.besu.cli.config.EthNetworkConfig; import org.hyperledger.besu.cli.config.NetworkName; +import org.hyperledger.besu.cli.config.ProfileName; import org.hyperledger.besu.cli.converter.MetricCategoryConverter; import org.hyperledger.besu.cli.converter.PercentageConverter; import org.hyperledger.besu.cli.custom.CorsAllowedOriginsProperty; @@ -60,6 +60,7 @@ import org.hyperledger.besu.cli.options.stable.EthstatsOptions; import org.hyperledger.besu.cli.options.stable.LoggingLevelOption; import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption; import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions; +import org.hyperledger.besu.cli.options.stable.RpcWebsocketOptions; import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions; import org.hyperledger.besu.cli.options.unstable.DnsOptions; import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions; @@ -123,7 +124,6 @@ import org.hyperledger.besu.ethereum.api.tls.FileBasedPasswordProvider; import org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration; import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration; import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; @@ -224,6 +224,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.TreeMap; import java.util.function.Function; @@ -536,6 +537,13 @@ public class BesuCommand implements DefaultCommandValues, Runnable { + " (default: ${DEFAULT-VALUE})") private final NetworkName network = null; + @Option( + names = {PROFILE_OPTION_NAME}, + paramLabel = PROFILE_FORMAT_HELP, + description = + "Overwrite default settings. Possible values are ${COMPLETION-CANDIDATES}. (default: none)") + private final ProfileName profile = null; + @Option( names = {"--nat-method"}, description = @@ -798,93 +806,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { // JSON-RPC Websocket Options @CommandLine.ArgGroup(validate = false, heading = "@|bold JSON-RPC Websocket Options|@%n") - JsonRPCWebsocketOptionGroup jsonRPCWebsocketOptionGroup = new JsonRPCWebsocketOptionGroup(); - - static class JsonRPCWebsocketOptionGroup { - @Option( - names = {"--rpc-ws-authentication-jwt-algorithm"}, - description = - "Encryption algorithm used for Websockets JWT public key. Possible values are ${COMPLETION-CANDIDATES}" - + " (default: ${DEFAULT-VALUE})", - arity = "1") - private final JwtAlgorithm rpcWebsocketsAuthenticationAlgorithm = DEFAULT_JWT_ALGORITHM; - - @Option( - names = {"--rpc-ws-enabled"}, - description = "Set to start the JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") - private final Boolean isRpcWsEnabled = false; - - @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. - @Option( - names = {"--rpc-ws-host"}, - paramLabel = MANDATORY_HOST_FORMAT_HELP, - description = - "Host for JSON-RPC WebSocket service to listen on (default: ${DEFAULT-VALUE})", - arity = "1") - private String rpcWsHost; - - @Option( - names = {"--rpc-ws-port"}, - paramLabel = MANDATORY_PORT_FORMAT_HELP, - description = - "Port for JSON-RPC WebSocket service to listen on (default: ${DEFAULT-VALUE})", - arity = "1") - private final Integer rpcWsPort = DEFAULT_WEBSOCKET_PORT; - - @Option( - names = {"--rpc-ws-max-frame-size"}, - description = - "Maximum size in bytes for JSON-RPC WebSocket frames (default: ${DEFAULT-VALUE}). If this limit is exceeded, the websocket will be disconnected.", - arity = "1") - private final Integer rpcWsMaxFrameSize = DEFAULT_WS_MAX_FRAME_SIZE; - - @Option( - names = {"--rpc-ws-max-active-connections"}, - description = - "Maximum number of WebSocket connections allowed for JSON-RPC (default: ${DEFAULT-VALUE}). Once this limit is reached, incoming connections will be rejected.", - arity = "1") - private final Integer rpcWsMaxConnections = DEFAULT_WS_MAX_CONNECTIONS; - - @Option( - names = {"--rpc-ws-api", "--rpc-ws-apis"}, - paramLabel = "", - split = " {0,1}, {0,1}", - arity = "1..*", - description = - "Comma separated list of APIs to enable on JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") - private final List rpcWsApis = DEFAULT_RPC_APIS; - - @Option( - names = {"--rpc-ws-api-methods-no-auth", "--rpc-ws-api-method-no-auth"}, - paramLabel = "", - split = " {0,1}, {0,1}", - arity = "1..*", - description = - "Comma separated list of RPC methods to exclude from RPC authentication services, RPC WebSocket authentication must be enabled") - private final List rpcWsApiMethodsNoAuth = new ArrayList(); - - @Option( - names = {"--rpc-ws-authentication-enabled"}, - description = - "Require authentication for the JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") - private final Boolean isRpcWsAuthenticationEnabled = false; - - @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. - @CommandLine.Option( - names = {"--rpc-ws-authentication-credentials-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = - "Storage file for JSON-RPC WebSocket authentication credentials (default: ${DEFAULT-VALUE})", - arity = "1") - private String rpcWsAuthenticationCredentialsFile = null; - - @CommandLine.Option( - names = {"--rpc-ws-authentication-jwt-public-key-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = "JWT public key file for JSON-RPC WebSocket authentication", - arity = "1") - private final File rpcWsAuthenticationPublicKeyFile = null; - } + RpcWebsocketOptions rpcWebsocketOptions = new RpcWebsocketOptions(); // Privacy Options Group @CommandLine.ArgGroup(validate = false, heading = "@|bold Privacy Options|@%n") @@ -1259,6 +1181,12 @@ public class BesuCommand implements DefaultCommandValues, Runnable { description = "Specifies the number of last blocks to cache (default: ${DEFAULT-VALUE})") private final Integer numberOfblocksToCache = 0; + @Option( + names = {"--rpc-max-trace-filter-range"}, + description = + "Specifies the maximum number of blocks for the trace_filter method. Must be >=0. 0 specifies no limit (default: $DEFAULT-VALUE)") + private final Long maxTraceFilterRange = 1000L; + @Mixin private P2PTLSConfigOptions p2pTLSConfigOptions; @Mixin private PkiBlockCreationOptions pkiBlockCreationOptions; @@ -1806,6 +1734,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { validateDnsOptionsParams(); ensureValidPeerBoundParams(); validateRpcOptionsParams(); + validateRpcWsOptions(); validateChainDataPruningParams(); validatePostMergeCheckpointBlockRequirements(); validateTransactionPoolOptions(); @@ -1948,15 +1877,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { + invalidHttpApis.toString()); } - if (!jsonRPCWebsocketOptionGroup.rpcWsApis.stream().allMatch(configuredApis)) { - final List invalidWsApis = - new ArrayList(jsonRPCWebsocketOptionGroup.rpcWsApis); - invalidWsApis.removeAll(VALID_APIS); - throw new ParameterException( - this.commandLine, - "Invalid value for option '--rpc-ws-api': invalid entries found " + invalidWsApis); - } - final boolean validHttpApiMethods = jsonRPCHttpOptionGroup.rpcHttpApiMethodsNoAuth.stream() .allMatch(RpcMethod::rpcMethodExists); @@ -1966,16 +1886,15 @@ public class BesuCommand implements DefaultCommandValues, Runnable { this.commandLine, "Invalid value for option '--rpc-http-api-methods-no-auth', options must be valid RPC methods"); } + } - final boolean validWsApiMethods = - jsonRPCWebsocketOptionGroup.rpcWsApiMethodsNoAuth.stream() - .allMatch(RpcMethod::rpcMethodExists); - - if (!validWsApiMethods) { - throw new ParameterException( - this.commandLine, - "Invalid value for option '--rpc-ws-api-methods-no-auth', options must be valid RPC methods"); - } + private void validateRpcWsOptions() { + final Predicate configuredApis = + apiName -> + Arrays.stream(RpcApis.values()) + .anyMatch(builtInApi -> apiName.equals(builtInApi.name())) + || rpcEndpointServiceImpl.hasNamespace(apiName); + rpcWebsocketOptions.validate(logger, commandLine, configuredApis); } private void validateChainDataPruningParams() { @@ -2078,10 +1997,10 @@ public class BesuCommand implements DefaultCommandValues, Runnable { p2pTLSConfiguration = p2pTLSConfigOptions.p2pTLSConfiguration(commandLine); graphQLConfiguration = graphQLConfiguration(); webSocketConfiguration = - webSocketConfiguration( - jsonRPCWebsocketOptionGroup.rpcWsPort, - jsonRPCWebsocketOptionGroup.rpcWsApis, - hostsAllowlist); + rpcWebsocketOptions.webSocketConfiguration( + hostsAllowlist, + p2PDiscoveryOptionGroup.autoDiscoverDefaultIP().getHostAddress(), + unstableRPCOptions.getWsTimeoutSec()); jsonRpcIpcConfiguration = jsonRpcIpcConfiguration( unstableIpcOptions.isEnabled(), @@ -2450,71 +2369,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { return jsonRPCHttpOptionGroup.isRpcHttpEnabled && jsonRPCHttpOptionGroup.isRpcHttpTlsEnabled; } - private WebSocketConfiguration webSocketConfiguration( - final Integer listenPort, final List apiGroups, final List allowCallsFrom) { - - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--rpc-ws-enabled", - !jsonRPCWebsocketOptionGroup.isRpcWsEnabled, - asList( - "--rpc-ws-api", - "--rpc-ws-apis", - "--rpc-ws-api-method-no-auth", - "--rpc-ws-api-methods-no-auth", - "--rpc-ws-host", - "--rpc-ws-port", - "--rpc-ws-max-frame-size", - "--rpc-ws-max-active-connections", - "--rpc-ws-authentication-enabled", - "--rpc-ws-authentication-credentials-file", - "--rpc-ws-authentication-public-key-file", - "--rpc-ws-authentication-jwt-algorithm")); - - if (jsonRPCWebsocketOptionGroup.isRpcWsAuthenticationEnabled) { - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--rpc-ws-authentication-public-key-file", - jsonRPCWebsocketOptionGroup.rpcWsAuthenticationPublicKeyFile == null, - asList("--rpc-ws-authentication-jwt-algorithm")); - } - - if (jsonRPCWebsocketOptionGroup.isRpcWsAuthenticationEnabled - && rpcWsAuthenticationCredentialsFile() == null - && jsonRPCWebsocketOptionGroup.rpcWsAuthenticationPublicKeyFile == null) { - throw new ParameterException( - commandLine, - "Unable to authenticate JSON-RPC WebSocket endpoint without a supplied credentials file or authentication public key file"); - } - - final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); - webSocketConfiguration.setEnabled(jsonRPCWebsocketOptionGroup.isRpcWsEnabled); - webSocketConfiguration.setHost( - Strings.isNullOrEmpty(jsonRPCWebsocketOptionGroup.rpcWsHost) - ? p2PDiscoveryOptionGroup.autoDiscoverDefaultIP().getHostAddress() - : jsonRPCWebsocketOptionGroup.rpcWsHost); - webSocketConfiguration.setPort(listenPort); - webSocketConfiguration.setMaxFrameSize(jsonRPCWebsocketOptionGroup.rpcWsMaxFrameSize); - webSocketConfiguration.setMaxActiveConnections(jsonRPCWebsocketOptionGroup.rpcWsMaxConnections); - webSocketConfiguration.setRpcApis(apiGroups); - webSocketConfiguration.setRpcApisNoAuth( - jsonRPCWebsocketOptionGroup.rpcWsApiMethodsNoAuth.stream() - .distinct() - .collect(Collectors.toList())); - webSocketConfiguration.setAuthenticationEnabled( - jsonRPCWebsocketOptionGroup.isRpcWsAuthenticationEnabled); - webSocketConfiguration.setAuthenticationCredentialsFile(rpcWsAuthenticationCredentialsFile()); - webSocketConfiguration.setHostsAllowlist(allowCallsFrom); - webSocketConfiguration.setAuthenticationPublicKeyFile( - jsonRPCWebsocketOptionGroup.rpcWsAuthenticationPublicKeyFile); - webSocketConfiguration.setAuthenticationAlgorithm( - jsonRPCWebsocketOptionGroup.rpcWebsocketsAuthenticationAlgorithm); - webSocketConfiguration.setTimeoutSec(unstableRPCOptions.getWsTimeoutSec()); - return webSocketConfiguration; - } - private ApiConfiguration apiConfiguration() { checkApiOptionsDependencies(); var builder = @@ -2526,7 +2380,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable { .gasPriceMax(apiGasPriceMax) .maxLogsRange(rpcMaxLogsRange) .gasCap(rpcGasCap) - .isGasAndPriorityFeeLimitingEnabled(apiGasAndPriorityFeeLimitingEnabled); + .isGasAndPriorityFeeLimitingEnabled(apiGasAndPriorityFeeLimitingEnabled) + .maxTraceFilterRange(maxTraceFilterRange); if (apiGasAndPriorityFeeLimitingEnabled) { if (apiGasAndPriorityFeeLowerBoundCoefficient > apiGasAndPriorityFeeUpperBoundCoefficient) { throw new ParameterException( @@ -2596,7 +2451,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private Optional permissioningConfiguration() throws Exception { if (!(localPermissionsEnabled() || contractPermissionsEnabled())) { if (jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.PERM.name()) - || jsonRPCWebsocketOptionGroup.rpcWsApis.contains(RpcApis.PERM.name())) { + || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.PERM.name())) { logger.warn( "Permissions are disabled. Cannot enable PERM APIs when not using Permissions."); } @@ -2799,9 +2654,9 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private boolean anyPrivacyApiEnabled() { return jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.EEA.name()) - || jsonRPCWebsocketOptionGroup.rpcWsApis.contains(RpcApis.EEA.name()) + || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.EEA.name()) || jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.PRIV.name()) - || jsonRPCWebsocketOptionGroup.rpcWsApis.contains(RpcApis.PRIV.name()); + || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.PRIV.name()); } private PrivacyKeyValueStorageProvider privacyKeyStorageProvider(final String name) { @@ -2903,32 +2758,28 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private MiningParameters getMiningParameters() { if (miningParameters == null) { - final var miningParametersBuilder = - ImmutableMiningParameters.builder().from(miningOptions.toDomainObject()); - final var actualGenesisOptions = getActualGenesisConfigOptions(); - if (actualGenesisOptions.isPoa()) { - miningParametersBuilder.genesisBlockPeriodSeconds( - getGenesisBlockPeriodSeconds(actualGenesisOptions)); - } - miningParameters = miningParametersBuilder.build(); + miningOptions.setGenesisBlockPeriodSeconds( + getGenesisBlockPeriodSeconds(getActualGenesisConfigOptions())); + miningParameters = miningOptions.toDomainObject(); } return miningParameters; } - private int getGenesisBlockPeriodSeconds(final GenesisConfigOptions genesisConfigOptions) { + private OptionalInt getGenesisBlockPeriodSeconds( + final GenesisConfigOptions genesisConfigOptions) { if (genesisConfigOptions.isClique()) { - return genesisConfigOptions.getCliqueConfigOptions().getBlockPeriodSeconds(); + return OptionalInt.of(genesisConfigOptions.getCliqueConfigOptions().getBlockPeriodSeconds()); } if (genesisConfigOptions.isIbft2()) { - return genesisConfigOptions.getBftConfigOptions().getBlockPeriodSeconds(); + return OptionalInt.of(genesisConfigOptions.getBftConfigOptions().getBlockPeriodSeconds()); } if (genesisConfigOptions.isQbft()) { - return genesisConfigOptions.getQbftConfigOptions().getBlockPeriodSeconds(); + return OptionalInt.of(genesisConfigOptions.getQbftConfigOptions().getBlockPeriodSeconds()); } - throw new IllegalArgumentException("Should only be called for a PoA network"); + return OptionalInt.empty(); } private boolean isPruningEnabled() { @@ -3176,15 +3027,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { return filename; } - private String rpcWsAuthenticationCredentialsFile() { - final String filename = jsonRPCWebsocketOptionGroup.rpcWsAuthenticationCredentialsFile; - - if (filename != null) { - RpcAuthFileValidator.validate(commandLine, filename, "WS"); - } - return filename; - } - private String getDefaultPermissioningFilePath() { return dataDir() + System.getProperty("file.separator") @@ -3322,9 +3164,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { jsonRPCHttpOptionGroup.rpcHttpPort, jsonRPCHttpOptionGroup.isRpcHttpEnabled); addPortIfEnabled( - effectivePorts, - jsonRPCWebsocketOptionGroup.rpcWsPort, - jsonRPCWebsocketOptionGroup.isRpcWsEnabled); + effectivePorts, rpcWebsocketOptions.getRpcWsPort(), rpcWebsocketOptions.isRpcWsEnabled()); addPortIfEnabled(effectivePorts, engineRPCOptionGroup.engineRpcPort, isEngineApiEnabled()); addPortIfEnabled( effectivePorts, metricsOptionGroup.metricsPort, metricsOptionGroup.isMetricsEnabled); @@ -3402,7 +3242,12 @@ public class BesuCommand implements DefaultCommandValues, Runnable { return genesisConfigOptions.getEcCurve(); } - private GenesisConfigOptions getActualGenesisConfigOptions() { + /** + * Return the genesis config options after applying any specified config overrides + * + * @return the genesis config options after applying any specified config overrides + */ + protected GenesisConfigOptions getActualGenesisConfigOptions() { return Optional.ofNullable(genesisConfigOptions) .orElseGet( () -> @@ -3508,6 +3353,10 @@ public class BesuCommand implements DefaultCommandValues, Runnable { builder.setNetwork(network.normalize()); } + if (profile != null) { + builder.setProfile(profile.toString()); + } + builder.setHasCustomGenesis(genesisFile != null); if (genesisFile != null) { builder.setCustomGenesis(genesisFile.getAbsolutePath()); @@ -3543,12 +3392,12 @@ public class BesuCommand implements DefaultCommandValues, Runnable { builder.setHighSpecEnabled(); } - if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningEnabled()) { - builder.setTrieLogPruningEnabled(); - builder.setTrieLogRetentionThreshold( - dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogRetentionThreshold()); - builder.setTrieLogPruningLimit( - dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningLimit()); + if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiLimitTrieLogsEnabled()) { + builder.setLimitTrieLogsEnabled(); + builder.setTrieLogRetentionLimit( + dataStorageOptions.toDomainObject().getBonsaiMaxLayersToLoad()); + builder.setTrieLogsPruningWindowSize( + dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningWindowSize()); } builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index 4fd870e99c..bf03c675d8 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -41,6 +41,7 @@ public class ConfigurationOverviewBuilder { private String network; private BigInteger networkId; + private String profile; private boolean hasCustomGenesis; private String customGenesisFileName; private String dataStorage; @@ -51,9 +52,9 @@ public class ConfigurationOverviewBuilder { private Collection engineApis; private String engineJwtFilePath; private boolean isHighSpec = false; - private boolean isTrieLogPruningEnabled = false; - private long trieLogRetentionThreshold = 0; - private Integer trieLogPruningLimit = null; + private boolean isBonsaiLimitTrieLogsEnabled = false; + private long trieLogRetentionLimit = 0; + private Integer trieLogsPruningWindowSize = null; private TransactionPoolConfiguration.Implementation txPoolImplementation; private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode; private Map environment; @@ -88,6 +89,17 @@ public class ConfigurationOverviewBuilder { return this; } + /** + * Sets profile. + * + * @param profile the profile + * @return the profile + */ + public ConfigurationOverviewBuilder setProfile(final String profile) { + this.profile = profile; + return this; + } + /** * Sets whether a custom genesis has been specified. * @@ -187,34 +199,34 @@ public class ConfigurationOverviewBuilder { } /** - * Sets trie log pruning enabled + * Sets limit trie logs enabled * * @return the builder */ - public ConfigurationOverviewBuilder setTrieLogPruningEnabled() { - isTrieLogPruningEnabled = true; + public ConfigurationOverviewBuilder setLimitTrieLogsEnabled() { + isBonsaiLimitTrieLogsEnabled = true; return this; } /** - * Sets trie log retention threshold + * Sets trie log retention limit * - * @param threshold the number of blocks to retain trie logs for + * @param limit the number of blocks to retain trie logs for * @return the builder */ - public ConfigurationOverviewBuilder setTrieLogRetentionThreshold(final long threshold) { - trieLogRetentionThreshold = threshold; + public ConfigurationOverviewBuilder setTrieLogRetentionLimit(final long limit) { + trieLogRetentionLimit = limit; return this; } /** - * Sets trie log pruning limit + * Sets trie logs pruning window size * - * @param limit the max number of blocks to load and prune trie logs for at startup + * @param size the max number of blocks to load and prune trie logs for at startup * @return the builder */ - public ConfigurationOverviewBuilder setTrieLogPruningLimit(final int limit) { - trieLogPruningLimit = limit; + public ConfigurationOverviewBuilder setTrieLogsPruningWindowSize(final int size) { + trieLogsPruningWindowSize = size; return this; } @@ -290,6 +302,10 @@ public class ConfigurationOverviewBuilder { lines.add("Network Id: " + networkId); } + if (profile != null) { + lines.add("Profile: " + profile); + } + if (dataStorage != null) { lines.add("Data storage: " + dataStorage); } @@ -323,13 +339,13 @@ public class ConfigurationOverviewBuilder { lines.add("Using " + worldStateUpdateMode + " worldstate update mode"); - if (isTrieLogPruningEnabled) { + if (isBonsaiLimitTrieLogsEnabled) { final StringBuilder trieLogPruningString = new StringBuilder(); trieLogPruningString - .append("Trie log pruning enabled: retention: ") - .append(trieLogRetentionThreshold); - if (trieLogPruningLimit != null) { - trieLogPruningString.append("; prune limit: ").append(trieLogPruningLimit); + .append("Limit trie logs enabled: retention: ") + .append(trieLogRetentionLimit); + if (trieLogsPruningWindowSize != null) { + trieLogPruningString.append("; prune window: ").append(trieLogsPruningWindowSize); } lines.add(trieLogPruningString.toString()); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java b/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java index d8644c36de..3a0db47816 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java @@ -54,6 +54,10 @@ public interface DefaultCommandValues { String MANDATORY_MODE_FORMAT_HELP = ""; /** The constant MANDATORY_NETWORK_FORMAT_HELP. */ String MANDATORY_NETWORK_FORMAT_HELP = ""; + /** The constant PROFILE_OPTION_NAME. */ + String PROFILE_OPTION_NAME = "--profile"; + /** The constant PROFILE_FORMAT_HELP. */ + String PROFILE_FORMAT_HELP = ""; /** The constant MANDATORY_NODE_ID_FORMAT_HELP. */ String MANDATORY_NODE_ID_FORMAT_HELP = ""; /** The constant PERMISSIONING_CONFIG_LOCATION. */ diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java new file mode 100644 index 0000000000..823dd9f142 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java @@ -0,0 +1,41 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.config; + +/** Enum for profile names. Each profile corresponds to a configuration file. */ +public enum ProfileName { + /** The 'DEV' profile. Corresponds to the 'profiles/dev.toml' configuration file. */ + DEV("profiles/dev.toml"); + + private final String configFile; + + /** + * Constructs a new ProfileName. + * + * @param configFile the configuration file corresponding to the profile + */ + ProfileName(final String configFile) { + this.configFile = configFile; + } + + /** + * Gets the configuration file corresponding to the profile. + * + * @return the configuration file + */ + public String getConfigFile() { + return configFile; + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java index 55197b8e82..2e5a239f16 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.util.number.PositiveNumber; import java.util.List; +import java.util.OptionalInt; import org.apache.tuweni.bytes.Bytes; import org.slf4j.Logger; @@ -188,6 +189,8 @@ public class MiningOptions implements CLIOptions { DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION; } + private OptionalInt maybeGenesisBlockPeriodSeconds; + private MiningOptions() {} /** @@ -199,6 +202,16 @@ public class MiningOptions implements CLIOptions { return new MiningOptions(); } + /** + * Set the optional genesis block period per seconds + * + * @param genesisBlockPeriodSeconds if the network is PoA then the block period in seconds + * specified in the genesis file, otherwise empty. + */ + public void setGenesisBlockPeriodSeconds(final OptionalInt genesisBlockPeriodSeconds) { + maybeGenesisBlockPeriodSeconds = genesisBlockPeriodSeconds; + } + /** * Validate that there are no inconsistencies in the specified options. For example that the * options are valid for the selected implementation. @@ -285,6 +298,7 @@ public class MiningOptions implements CLIOptions { static MiningOptions fromConfig(final MiningParameters miningParameters) { final MiningOptions miningOptions = MiningOptions.create(); + miningOptions.setGenesisBlockPeriodSeconds(miningParameters.getGenesisBlockPeriodSeconds()); miningOptions.isMiningEnabled = miningParameters.isMiningEnabled(); miningOptions.iStratumMiningEnabled = miningParameters.isStratumMiningEnabled(); miningOptions.stratumNetworkInterface = miningParameters.getStratumNetworkInterface(); @@ -319,6 +333,11 @@ public class MiningOptions implements CLIOptions { @Override public MiningParameters toDomainObject() { + if (maybeGenesisBlockPeriodSeconds == null) { + throw new IllegalStateException( + "genesisBlockPeriodSeconds must be set before using this object"); + } + final var updatableInitValuesBuilder = MutableInitValues.builder() .isMiningEnabled(isMiningEnabled) @@ -334,27 +353,26 @@ public class MiningOptions implements CLIOptions { updatableInitValuesBuilder.coinbase(coinbase); } - final var miningParametersBuilder = - ImmutableMiningParameters.builder() - .mutableInitValues(updatableInitValuesBuilder.build()) - .isStratumMiningEnabled(iStratumMiningEnabled) - .stratumNetworkInterface(stratumNetworkInterface) - .stratumPort(stratumPort) - .nonPoaBlockTxsSelectionMaxTime(nonPoaBlockTxsSelectionMaxTime) - .poaBlockTxsSelectionMaxTime(poaBlockTxsSelectionMaxTime) - .unstable( - ImmutableMiningParameters.Unstable.builder() - .remoteSealersLimit(unstableOptions.remoteSealersLimit) - .remoteSealersTimeToLive(unstableOptions.remoteSealersTimeToLive) - .powJobTimeToLive(unstableOptions.powJobTimeToLive) - .maxOmmerDepth(unstableOptions.maxOmmersDepth) - .stratumExtranonce(unstableOptions.stratumExtranonce) - .posBlockCreationMaxTime(unstableOptions.posBlockCreationMaxTime) - .posBlockCreationRepetitionMinDuration( - unstableOptions.posBlockCreationRepetitionMinDuration) - .build()); - - return miningParametersBuilder.build(); + return ImmutableMiningParameters.builder() + .genesisBlockPeriodSeconds(maybeGenesisBlockPeriodSeconds) + .mutableInitValues(updatableInitValuesBuilder.build()) + .isStratumMiningEnabled(iStratumMiningEnabled) + .stratumNetworkInterface(stratumNetworkInterface) + .stratumPort(stratumPort) + .nonPoaBlockTxsSelectionMaxTime(nonPoaBlockTxsSelectionMaxTime) + .poaBlockTxsSelectionMaxTime(poaBlockTxsSelectionMaxTime) + .unstable( + ImmutableMiningParameters.Unstable.builder() + .remoteSealersLimit(unstableOptions.remoteSealersLimit) + .remoteSealersTimeToLive(unstableOptions.remoteSealersTimeToLive) + .powJobTimeToLive(unstableOptions.powJobTimeToLive) + .maxOmmerDepth(unstableOptions.maxOmmersDepth) + .stratumExtranonce(unstableOptions.stratumExtranonce) + .posBlockCreationMaxTime(unstableOptions.posBlockCreationMaxTime) + .posBlockCreationRepetitionMinDuration( + unstableOptions.posBlockCreationRepetitionMinDuration) + .build()) + .build(); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index e0b1973568..a1f4b950bb 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -17,10 +17,9 @@ package org.hyperledger.besu.cli.options.stable; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT; import org.hyperledger.besu.cli.options.CLIOptions; import org.hyperledger.besu.cli.util.CommandLineUtils; @@ -39,7 +38,8 @@ public class DataStorageOptions implements CLIOptions private static final String DATA_STORAGE_FORMAT = "--data-storage-format"; - private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = + /** The maximum number of historical layers to load. */ + public static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = "--bonsai-historical-block-limit"; // Use Bonsai DB @@ -54,34 +54,37 @@ public class DataStorageOptions implements CLIOptions names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"}, paramLabel = "", description = - "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}).", + "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using " + + Unstable.BONSAI_LIMIT_TRIE_LOGS_ENABLED + + " it will also be used as the number of layers of trie logs to retain.", arity = "1") private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; @CommandLine.ArgGroup(validate = false) private final DataStorageOptions.Unstable unstableOptions = new Unstable(); - static class Unstable { + /** The unstable options for data storage. */ + public static class Unstable { + private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = + "--Xbonsai-limit-trie-logs-enabled"; - @CommandLine.Option( - hidden = true, - names = {"--Xbonsai-trie-log-pruning-enabled"}, - description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})") - private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; + /** The bonsai trie logs pruning window size. */ + public static final String BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = + "--Xbonsai-trie-logs-pruning-window-size"; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-retention-threshold"}, + names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, description = - "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})") - private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; + "Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})") + private boolean bonsaiLimitTrieLogsEnabled = DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-limit"}, + names = {BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE}, description = "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") - private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; + private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; } /** * Create data storage options. @@ -98,21 +101,31 @@ public class DataStorageOptions implements CLIOptions * @param commandLine the full commandLine to check all the options specified by the user */ public void validate(final CommandLine commandLine) { - if (unstableOptions.bonsaiTrieLogPruningEnabled) { - if (unstableOptions.bonsaiTrieLogRetentionThreshold - < MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) { + if (unstableOptions.bonsaiLimitTrieLogsEnabled) { + if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", + MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT)); + } + if (unstableOptions.bonsaiTrieLogPruningWindowSize <= 0) { throw new CommandLine.ParameterException( commandLine, String.format( - "--Xbonsai-trie-log-retention-threshold minimum value is %d", - MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD)); + Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0", + unstableOptions.bonsaiTrieLogPruningWindowSize)); } - if (unstableOptions.bonsaiTrieLogPruningLimit <= 0) { + if (unstableOptions.bonsaiTrieLogPruningWindowSize <= bonsaiMaxLayersToLoad) { throw new CommandLine.ParameterException( commandLine, String.format( - "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", - unstableOptions.bonsaiTrieLogPruningLimit)); + Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + + "=%d must be greater than " + + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + + "=%d", + unstableOptions.bonsaiTrieLogPruningWindowSize, + bonsaiMaxLayersToLoad)); } } } @@ -121,12 +134,10 @@ public class DataStorageOptions implements CLIOptions final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat(); dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad(); - dataStorageOptions.unstableOptions.bonsaiTrieLogPruningEnabled = - domainObject.getUnstable().getBonsaiTrieLogPruningEnabled(); - dataStorageOptions.unstableOptions.bonsaiTrieLogRetentionThreshold = - domainObject.getUnstable().getBonsaiTrieLogRetentionThreshold(); - dataStorageOptions.unstableOptions.bonsaiTrieLogPruningLimit = - domainObject.getUnstable().getBonsaiTrieLogPruningLimit(); + dataStorageOptions.unstableOptions.bonsaiLimitTrieLogsEnabled = + domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled(); + dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize = + domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize(); return dataStorageOptions; } @@ -138,9 +149,8 @@ public class DataStorageOptions implements CLIOptions .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogPruningEnabled(unstableOptions.bonsaiTrieLogPruningEnabled) - .bonsaiTrieLogRetentionThreshold(unstableOptions.bonsaiTrieLogRetentionThreshold) - .bonsaiTrieLogPruningLimit(unstableOptions.bonsaiTrieLogPruningLimit) + .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled) + .bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize) .build()) .build(); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/RpcWebsocketOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/RpcWebsocketOptions.java new file mode 100644 index 0000000000..cdc77a3f65 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/RpcWebsocketOptions.java @@ -0,0 +1,266 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options.stable; + +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.VALID_APIS; +import static org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration.DEFAULT_WEBSOCKET_PORT; + +import org.hyperledger.besu.cli.DefaultCommandValues; +import org.hyperledger.besu.cli.custom.RpcAuthFileValidator; +import org.hyperledger.besu.cli.util.CommandLineUtils; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; +import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import picocli.CommandLine; + +/** This class represents the WebSocket options for the RPC. */ +public class RpcWebsocketOptions { + @CommandLine.Option( + names = {"--rpc-ws-authentication-jwt-algorithm"}, + description = + "Encryption algorithm used for Websockets JWT public key. Possible values are ${COMPLETION-CANDIDATES}" + + " (default: ${DEFAULT-VALUE})", + arity = "1") + private final JwtAlgorithm rpcWebsocketsAuthenticationAlgorithm = + DefaultCommandValues.DEFAULT_JWT_ALGORITHM; + + @CommandLine.Option( + names = {"--rpc-ws-enabled"}, + description = "Set to start the JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") + private final Boolean isRpcWsEnabled = false; + + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. + @CommandLine.Option( + names = {"--rpc-ws-host"}, + paramLabel = DefaultCommandValues.MANDATORY_HOST_FORMAT_HELP, + description = "Host for JSON-RPC WebSocket service to listen on (default: ${DEFAULT-VALUE})", + arity = "1") + private String rpcWsHost; + + @CommandLine.Option( + names = {"--rpc-ws-port"}, + paramLabel = DefaultCommandValues.MANDATORY_PORT_FORMAT_HELP, + description = "Port for JSON-RPC WebSocket service to listen on (default: ${DEFAULT-VALUE})", + arity = "1") + private final Integer rpcWsPort = DEFAULT_WEBSOCKET_PORT; + + @CommandLine.Option( + names = {"--rpc-ws-max-frame-size"}, + description = + "Maximum size in bytes for JSON-RPC WebSocket frames (default: ${DEFAULT-VALUE}). If this limit is exceeded, the websocket will be disconnected.", + arity = "1") + private final Integer rpcWsMaxFrameSize = DefaultCommandValues.DEFAULT_WS_MAX_FRAME_SIZE; + + @CommandLine.Option( + names = {"--rpc-ws-max-active-connections"}, + description = + "Maximum number of WebSocket connections allowed for JSON-RPC (default: ${DEFAULT-VALUE}). Once this limit is reached, incoming connections will be rejected.", + arity = "1") + private final Integer rpcWsMaxConnections = DefaultCommandValues.DEFAULT_WS_MAX_CONNECTIONS; + + @CommandLine.Option( + names = {"--rpc-ws-api", "--rpc-ws-apis"}, + paramLabel = "", + split = " {0,1}, {0,1}", + arity = "1..*", + description = + "Comma separated list of APIs to enable on JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") + private final List rpcWsApis = DEFAULT_RPC_APIS; + + @CommandLine.Option( + names = {"--rpc-ws-api-methods-no-auth", "--rpc-ws-api-method-no-auth"}, + paramLabel = "", + split = " {0,1}, {0,1}", + arity = "1..*", + description = + "Comma separated list of RPC methods to exclude from RPC authentication services, RPC WebSocket authentication must be enabled") + private final List rpcWsApiMethodsNoAuth = new ArrayList(); + + @CommandLine.Option( + names = {"--rpc-ws-authentication-enabled"}, + description = + "Require authentication for the JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") + private final Boolean isRpcWsAuthenticationEnabled = false; + + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. + @CommandLine.Option( + names = {"--rpc-ws-authentication-credentials-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = + "Storage file for JSON-RPC WebSocket authentication credentials (default: ${DEFAULT-VALUE})", + arity = "1") + private String rpcWsAuthenticationCredentialsFile = null; + + @CommandLine.Option( + names = {"--rpc-ws-authentication-jwt-public-key-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = "JWT public key file for JSON-RPC WebSocket authentication", + arity = "1") + private final File rpcWsAuthenticationPublicKeyFile = null; + + /** + * Validates the WebSocket options. + * + * @param logger Logger instance + * @param commandLine CommandLine instance + * @param configuredApis Predicate for configured APIs + */ + public void validate( + final Logger logger, final CommandLine commandLine, final Predicate configuredApis) { + checkOptionDependencies(logger, commandLine); + + if (!rpcWsApis.stream().allMatch(configuredApis)) { + final List invalidWsApis = new ArrayList<>(rpcWsApis); + invalidWsApis.removeAll(VALID_APIS); + throw new CommandLine.ParameterException( + commandLine, + "Invalid value for option '--rpc-ws-api': invalid entries found " + invalidWsApis); + } + + final boolean validWsApiMethods = + rpcWsApiMethodsNoAuth.stream().allMatch(RpcMethod::rpcMethodExists); + + if (!validWsApiMethods) { + throw new CommandLine.ParameterException( + commandLine, + "Invalid value for option '--rpc-ws-api-methods-no-auth', options must be valid RPC methods"); + } + + if (isRpcWsAuthenticationEnabled + && rpcWsAuthenticationCredentialsFile(commandLine) == null + && rpcWsAuthenticationPublicKeyFile == null) { + throw new CommandLine.ParameterException( + commandLine, + "Unable to authenticate JSON-RPC WebSocket endpoint without a supplied credentials file or authentication public key file"); + } + } + + /** + * Checks the dependencies of the WebSocket options. + * + * @param logger Logger instance + * @param commandLine CommandLine instance + */ + private void checkOptionDependencies(final Logger logger, final CommandLine commandLine) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-enabled", + !isRpcWsEnabled, + List.of( + "--rpc-ws-api", + "--rpc-ws-apis", + "--rpc-ws-api-method-no-auth", + "--rpc-ws-api-methods-no-auth", + "--rpc-ws-host", + "--rpc-ws-port", + "--rpc-ws-max-frame-size", + "--rpc-ws-max-active-connections", + "--rpc-ws-authentication-enabled", + "--rpc-ws-authentication-credentials-file", + "--rpc-ws-authentication-public-key-file", + "--rpc-ws-authentication-jwt-algorithm")); + + if (isRpcWsAuthenticationEnabled) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-authentication-public-key-file", + rpcWsAuthenticationPublicKeyFile == null, + List.of("--rpc-ws-authentication-jwt-algorithm")); + } + } + + /** + * Creates a WebSocket configuration based on the WebSocket options. + * + * @param hostsAllowlist List of allowed hosts + * @param defaultHostAddress Default host address + * @param wsTimoutSec WebSocket timeout in seconds + * @return WebSocketConfiguration instance + */ + public WebSocketConfiguration webSocketConfiguration( + final List hostsAllowlist, final String defaultHostAddress, final Long wsTimoutSec) { + final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); + webSocketConfiguration.setEnabled(isRpcWsEnabled); + webSocketConfiguration.setHost( + Strings.isNullOrEmpty(rpcWsHost) ? defaultHostAddress : rpcWsHost); + webSocketConfiguration.setPort(rpcWsPort); + webSocketConfiguration.setMaxFrameSize(rpcWsMaxFrameSize); + webSocketConfiguration.setMaxActiveConnections(rpcWsMaxConnections); + webSocketConfiguration.setRpcApis(rpcWsApis); + webSocketConfiguration.setRpcApisNoAuth( + rpcWsApiMethodsNoAuth.stream().distinct().collect(Collectors.toList())); + webSocketConfiguration.setAuthenticationEnabled(isRpcWsAuthenticationEnabled); + webSocketConfiguration.setAuthenticationCredentialsFile(rpcWsAuthenticationCredentialsFile); + webSocketConfiguration.setHostsAllowlist(hostsAllowlist); + webSocketConfiguration.setAuthenticationPublicKeyFile(rpcWsAuthenticationPublicKeyFile); + webSocketConfiguration.setAuthenticationAlgorithm(rpcWebsocketsAuthenticationAlgorithm); + webSocketConfiguration.setTimeoutSec(wsTimoutSec); + return webSocketConfiguration; + } + + /** + * Validates the authentication credentials file for the WebSocket. + * + * @param commandLine CommandLine instance + * @return Filename of the authentication credentials file + */ + private String rpcWsAuthenticationCredentialsFile(final CommandLine commandLine) { + final String filename = rpcWsAuthenticationCredentialsFile; + + if (filename != null) { + RpcAuthFileValidator.validate(commandLine, filename, "WS"); + } + return filename; + } + + /** + * Returns the list of APIs for the WebSocket. + * + * @return List of APIs + */ + public List getRpcWsApis() { + return rpcWsApis; + } + + /** + * Checks if the WebSocket service is enabled. + * + * @return Boolean indicating if the WebSocket service is enabled + */ + public Boolean isRpcWsEnabled() { + return isRpcWsEnabled; + } + + /** + * Returns the port for the WebSocket service. + * + * @return Port number + */ + public Integer getRpcWsPort() { + return rpcWsPort; + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java index 0ad68e3627..69e62edfc0 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java @@ -37,7 +37,7 @@ public class NetworkingOptions implements CLIOptions { private final String DNS_DISCOVERY_SERVER_OVERRIDE_FLAG = "--Xp2p-dns-discovery-server"; private final String DISCOVERY_PROTOCOL_V5_ENABLED = "--Xv5-discovery-enabled"; /** The constant FILTER_ON_ENR_FORK_ID. */ - public static final String FILTER_ON_ENR_FORK_ID = "--Xfilter-on-enr-fork-id"; + public static final String FILTER_ON_ENR_FORK_ID = "--filter-on-enr-fork-id"; @CommandLine.Option( names = INITIATE_CONNECTIONS_FREQUENCY_FLAG, @@ -76,9 +76,9 @@ public class NetworkingOptions implements CLIOptions { @CommandLine.Option( names = FILTER_ON_ENR_FORK_ID, hidden = true, - defaultValue = "false", + defaultValue = "true", description = "Whether to enable filtering of peers based on the ENR field ForkId)") - private final Boolean filterOnEnrForkId = false; + private final Boolean filterOnEnrForkId = NetworkingConfiguration.DEFAULT_FILTER_ON_ENR_FORK_ID; @CommandLine.Option( hidden = true, diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java index f34ac97470..f17f1a09fc 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java @@ -19,7 +19,7 @@ import static org.hyperledger.besu.cli.subcommands.ValidateConfigSubCommand.COMM import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.cli.DefaultCommandValues; -import org.hyperledger.besu.cli.util.TomlConfigFileDefaultProvider; +import org.hyperledger.besu.cli.util.TomlConfigurationDefaultProvider; import org.hyperledger.besu.cli.util.VersionProvider; import java.io.PrintWriter; @@ -69,7 +69,8 @@ public class ValidateConfigSubCommand implements Runnable { public void run() { checkNotNull(parentCommand); try { - new TomlConfigFileDefaultProvider(commandLine, dataPath.toFile()).loadConfigurationFromFile(); + TomlConfigurationDefaultProvider.fromFile(commandLine, dataPath.toFile()) + .loadConfigurationFromFile(); } catch (Exception e) { this.out.println(e); return; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 52dbe55903..22efd97c86 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -16,13 +16,20 @@ package org.hyperledger.besu.cli.subcommands.storage; import static com.google.common.base.Preconditions.checkArgument; +import static org.hyperledger.besu.cli.options.stable.DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; +import org.hyperledger.besu.cli.options.stable.DataStorageOptions; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.File; @@ -32,6 +39,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintWriter; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.IdentityHashMap; @@ -39,6 +47,8 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.annotations.VisibleForTesting; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +60,7 @@ public class TrieLogHelper { private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000; private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); - static void prune( + void prune( final DataStorageConfiguration config, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, @@ -60,17 +70,22 @@ public class TrieLogHelper { validatePruneConfiguration(config); - final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); + final long layersToRetain = config.getBonsaiMaxLayersToLoad(); final long chainHeight = blockchain.getChainHeadBlockNumber(); final long lastBlockNumberToRetainTrieLogsFor = chainHeight - layersToRetain + 1; - if (!validPruneRequirements(blockchain, chainHeight, lastBlockNumberToRetainTrieLogsFor)) { + if (!validatePruneRequirements( + blockchain, + chainHeight, + lastBlockNumberToRetainTrieLogsFor, + rootWorldStateStorage, + layersToRetain)) { return; } - final long numberOfBatches = calculateNumberofBatches(layersToRetain); + final long numberOfBatches = calculateNumberOfBatches(layersToRetain); processTrieLogBatches( rootWorldStateStorage, @@ -80,15 +95,27 @@ public class TrieLogHelper { numberOfBatches, batchFileNameBase); - if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { - deleteFiles(batchFileNameBase, numberOfBatches); - LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); + // Should only be layersToRetain left but loading extra just in case of an unforeseen bug + final long countAfterPrune = + rootWorldStateStorage + .streamTrieLogKeys(layersToRetain + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE) + .count(); + if (countAfterPrune == layersToRetain) { + if (deleteFiles(batchFileNameBase, numberOfBatches)) { + LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); + } else { + throw new IllegalStateException( + "There was an error deleting the trie log backup files. Please ensure besu is working before deleting them manually."); + } } else { - LOG.error("Prune failed. Re-run the subcommand to load the trie logs from file."); + throw new IllegalStateException( + String.format( + "Remaining trie logs (%d) did not match %s (%d). Trie logs backup files have not been deleted, it is safe to rerun the subcommand.", + countAfterPrune, BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, layersToRetain)); } } - private static void processTrieLogBatches( + private void processTrieLogBatches( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, final long chainHeight, @@ -97,16 +124,15 @@ public class TrieLogHelper { final String batchFileNameBase) { for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - + final String batchFileName = batchFileNameBase + "-" + batchNumber; final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); - final long lastBlockOfBatch = Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); - final List trieLogKeys = getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch); - saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys); + LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); + saveTrieLogBatches(batchFileName, rootWorldStateStorage, trieLogKeys); } LOG.info("Clear trie logs..."); @@ -117,23 +143,20 @@ public class TrieLogHelper { } } - private static void saveTrieLogBatches( - final String batchFileNameBase, + private void saveTrieLogBatches( + final String batchFileName, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, final List trieLogKeys) { - LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); - try { - saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); + saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchFileName); } catch (IOException e) { LOG.error("Error saving trie logs to file: {}", e.getMessage()); throw new RuntimeException(e); } } - private static void restoreTrieLogBatches( + private void restoreTrieLogBatches( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) { @@ -147,19 +170,25 @@ public class TrieLogHelper { } } - private static void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { + private boolean deleteFiles(final String batchFileNameBase, final long numberOfBatches) { LOG.info("Deleting files..."); - for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - File file = new File(batchFileNameBase + "-" + batchNumber); - if (file.exists()) { - file.delete(); + try { + for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { + File file = new File(batchFileNameBase + "-" + batchNumber); + if (file.exists()) { + file.delete(); + } } + return true; + } catch (Exception e) { + LOG.error("Error deleting files", e); + return false; } } - private static List getTrieLogKeysForBlocks( + private List getTrieLogKeysForBlocks( final MutableBlockchain blockchain, final long firstBlockOfBatch, final long lastBlockOfBatch) { @@ -173,14 +202,17 @@ public class TrieLogHelper { return trieLogKeys; } - private static long calculateNumberofBatches(final long layersToRetain) { + private long calculateNumberOfBatches(final long layersToRetain) { return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1); } - private static boolean validPruneRequirements( + private boolean validatePruneRequirements( final MutableBlockchain blockchain, final long chainHeight, - final long lastBlockNumberToRetainTrieLogsFor) { + final long lastBlockNumberToRetainTrieLogsFor, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final long layersToRetain) { + if (lastBlockNumberToRetainTrieLogsFor < 0) { throw new IllegalArgumentException( "Trying to retain more trie logs than chain length (" @@ -188,6 +220,19 @@ public class TrieLogHelper { + "), skipping pruning"); } + // Need to ensure we're loading at least layersToRetain if they exist + // plus extra threshold to account forks and orphans + final long clampedCountBeforePruning = + rootWorldStateStorage + .streamTrieLogKeys(layersToRetain + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE) + .count(); + if (clampedCountBeforePruning < layersToRetain) { + throw new IllegalArgumentException( + String.format( + "Trie log count (%d) is less than retention limit (%d), skipping pruning", + clampedCountBeforePruning, layersToRetain)); + } + final Optional finalizedBlockHash = blockchain.getFinalized(); if (finalizedBlockHash.isEmpty()) { @@ -204,15 +249,14 @@ public class TrieLogHelper { return true; } - private static void recreateTrieLogs( + private void recreateTrieLogs( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) throws IOException { // process in chunk to avoid OOM - - IdentityHashMap trieLogsToRetain = - readTrieLogsFromFile(batchFileNameBase, batchNumber); + final String batchFileName = batchFileNameBase + "-" + batchNumber; + IdentityHashMap trieLogsToRetain = readTrieLogsFromFile(batchFileName); final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; List keys = new ArrayList<>(trieLogsToRetain.keySet()); @@ -221,7 +265,7 @@ public class TrieLogHelper { } } - private static void processTransactionChunk( + private void processTransactionChunk( final int startIndex, final int chunkSize, final List keys, @@ -241,35 +285,39 @@ public class TrieLogHelper { updater.getTrieLogStorageTransaction().commit(); } - private static void validatePruneConfiguration(final DataStorageConfiguration config) { + @VisibleForTesting + void validatePruneConfiguration(final DataStorageConfiguration config) { checkArgument( - config.getUnstable().getBonsaiTrieLogRetentionThreshold() - >= config.getBonsaiMaxLayersToLoad(), + config.getBonsaiMaxLayersToLoad() + >= DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT, String.format( - "--Xbonsai-trie-log-retention-threshold minimum value is %d", - config.getBonsaiMaxLayersToLoad())); + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", + DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT)); checkArgument( - config.getUnstable().getBonsaiTrieLogPruningLimit() > 0, + config.getUnstable().getBonsaiTrieLogPruningWindowSize() > 0, String.format( - "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", - config.getUnstable().getBonsaiTrieLogPruningLimit())); + DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + + "=%d must be greater than 0", + config.getUnstable().getBonsaiTrieLogPruningWindowSize())); checkArgument( - config.getUnstable().getBonsaiTrieLogPruningLimit() - > config.getUnstable().getBonsaiTrieLogRetentionThreshold(), + config.getUnstable().getBonsaiTrieLogPruningWindowSize() + > config.getBonsaiMaxLayersToLoad(), String.format( - "--Xbonsai-trie-log-pruning-limit=%d must greater than --Xbonsai-trie-log-retention-threshold=%d", - config.getUnstable().getBonsaiTrieLogPruningLimit(), - config.getUnstable().getBonsaiTrieLogRetentionThreshold())); + DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + + "=%d must be greater than " + + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + + "=%d", + config.getUnstable().getBonsaiTrieLogPruningWindowSize(), + config.getBonsaiMaxLayersToLoad())); } - private static void saveTrieLogsInFile( + private void saveTrieLogsInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, - final String batchFileNameBase) + final String batchFileName) throws IOException { - File file = new File(batchFileNameBase + "-" + batchNumber); + File file = new File(batchFileName); if (file.exists()) { LOG.error("File already exists, skipping file creation"); return; @@ -285,17 +333,14 @@ public class TrieLogHelper { } @SuppressWarnings("unchecked") - private static IdentityHashMap readTrieLogsFromFile( - final String batchFileNameBase, final long batchNumber) { + IdentityHashMap readTrieLogsFromFile(final String batchFileName) { IdentityHashMap trieLogs; - try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); + try (FileInputStream fis = new FileInputStream(batchFileName); ObjectInputStream ois = new ObjectInputStream(fis)) { trieLogs = (IdentityHashMap) ois.readObject(); - } catch (IOException | ClassNotFoundException e) { - LOG.error(e.getMessage()); throw new RuntimeException(e); } @@ -303,7 +348,53 @@ public class TrieLogHelper { return trieLogs; } - private static IdentityHashMap getTrieLogs( + private void saveTrieLogsAsRlpInFile( + final List trieLogsKeys, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final String batchFileName) { + File file = new File(batchFileName); + if (file.exists()) { + LOG.error("File already exists, skipping file creation"); + return; + } + + final IdentityHashMap trieLogs = + getTrieLogs(trieLogsKeys, rootWorldStateStorage); + final Bytes rlp = + RLP.encode( + o -> + o.writeList( + trieLogs.entrySet(), (val, out) -> out.writeRaw(Bytes.wrap(val.getValue())))); + try { + Files.write(file.toPath(), rlp.toArrayUnsafe()); + } catch (IOException e) { + LOG.error(e.getMessage()); + throw new RuntimeException(e); + } + } + + IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { + try { + final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName))); + final BytesValueRLPInput input = new BytesValueRLPInput(file, false); + + input.enterList(); + final IdentityHashMap trieLogs = new IdentityHashMap<>(); + while (!input.isEndOfCurrentList()) { + final Bytes trieLogBytes = input.currentListAsBytes(); + TrieLogLayer trieLogLayer = + TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(trieLogBytes), false)); + trieLogs.put(trieLogLayer.getBlockHash().toArrayUnsafe(), trieLogBytes.toArrayUnsafe()); + } + input.leaveList(); + + return trieLogs; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private IdentityHashMap getTrieLogs( final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); @@ -316,7 +407,7 @@ public class TrieLogHelper { return trieLogsToRetain; } - static TrieLogCount getCount( + TrieLogCount getCount( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final int limit, final Blockchain blockchain) { @@ -351,11 +442,31 @@ public class TrieLogHelper { return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); } - static void printCount(final PrintWriter out, final TrieLogCount count) { + void printCount(final PrintWriter out, final TrieLogCount count) { out.printf( "trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", count.total, count.canonicalCount, count.forkCount, count.orphanCount); } + void importTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { + + var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString()); + + var updater = rootWorldStateStorage.updater(); + trieLog.forEach((key, value) -> updater.getTrieLogStorageTransaction().put(key, value)); + updater.getTrieLogStorageTransaction().commit(); + } + + void exportTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final List trieLogHash, + final Path directoryPath) + throws IOException { + final String trieLogFile = directoryPath.toString(); + + saveTrieLogsAsRlpInFile(trieLogHash, rootWorldStateStorage, trieLogFile); + } + record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 74e00197f5..a4a38737f9 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; @@ -26,9 +27,11 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -43,7 +46,12 @@ import picocli.CommandLine.ParentCommand; description = "Manipulate trie logs", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, - subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class}) + subcommands = { + TrieLogSubCommand.CountTrieLog.class, + TrieLogSubCommand.PruneTrieLog.class, + TrieLogSubCommand.ExportTrieLog.class, + TrieLogSubCommand.ImportTrieLog.class + }) public class TrieLogSubCommand implements Runnable { @SuppressWarnings("UnusedVariable") @@ -81,14 +89,15 @@ public class TrieLogSubCommand implements Runnable { @Override public void run() { - TrieLogContext context = getTrieLogContext(); + final TrieLogContext context = getTrieLogContext(); final PrintWriter out = spec.commandLine().getOut(); out.println("Counting trie logs..."); - TrieLogHelper.printCount( + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.printCount( out, - TrieLogHelper.getCount( + trieLogHelper.getCount( context.rootWorldStateStorage, Integer.MAX_VALUE, context.blockchain)); } } @@ -96,7 +105,7 @@ public class TrieLogSubCommand implements Runnable { @Command( name = "prune", description = - "This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", + "This command prunes all trie log layers below the retention limit, including orphaned trie logs.", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) static class PruneTrieLog implements Runnable { @@ -111,11 +120,12 @@ public class TrieLogSubCommand implements Runnable { @Override public void run() { - TrieLogContext context = getTrieLogContext(); + final TrieLogContext context = getTrieLogContext(); final Path dataDirectoryPath = Paths.get( TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); - TrieLogHelper.prune( + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.prune( context.config(), context.rootWorldStateStorage(), context.blockchain(), @@ -123,6 +133,105 @@ public class TrieLogSubCommand implements Runnable { } } + @Command( + name = "export", + description = "This command exports the trie log of a determined block to a binary file", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ExportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @SuppressWarnings("unused") + @CommandLine.Option( + names = "--trie-log-block-hash", + description = + "Comma separated list of hashes from the blocks you want to export the trie logs of", + split = " {0,1}, {0,1}", + arity = "1..*") + private List trieLogBlockHashList; + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The file you want to export the trie logs to", + arity = "1..1") + private Path trieLogFilePath = null; + + @Override + public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + + final TrieLogContext context = getTrieLogContext(); + + final List listOfBlockHashes = + trieLogBlockHashList.stream().map(Hash::fromHexString).toList(); + + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + + try { + trieLogHelper.exportTrieLog( + context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Command( + name = "import", + description = "This command imports a trie log exported by another besu node", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ImportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The file you want to import the trie logs from", + arity = "1..1") + private Path trieLogFilePath = null; + + @Override + public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + + TrieLogContext context = getTrieLogContext(); + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); + } + } + record TrieLogContext( DataStorageConfiguration config, BonsaiWorldStateKeyValueStorage rootWorldStateStorage, diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/AbstractConfigurationFinder.java b/besu/src/main/java/org/hyperledger/besu/cli/util/AbstractConfigurationFinder.java new file mode 100644 index 0000000000..69e0288777 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/AbstractConfigurationFinder.java @@ -0,0 +1,125 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.util; + +import java.util.Map; +import java.util.Optional; + +import picocli.CommandLine; + +/** + * Abstract class for finding configuration resources. This class provides a common structure for + * classes that need to find configuration resources based on command line options and environment + * variables. + * + * @param the type of configuration resource this finder will return + */ +public abstract class AbstractConfigurationFinder { + + /** + * Returns the name of the configuration option. + * + * @return the name of the configuration option + */ + protected abstract String getConfigOptionName(); + + /** + * Returns the name of the environment variable for the configuration. + * + * @return the name of the environment variable for the configuration + */ + protected abstract String getConfigEnvName(); + + /** + * Finds the configuration resource based on command line options and environment variables. + * + * @param environment the environment variables + * @param parseResult the command line parse result + * @return an Optional containing the configuration resource, or an empty Optional if no + * configuration resource was found + */ + public Optional findConfiguration( + final Map environment, final CommandLine.ParseResult parseResult) { + final CommandLine commandLine = parseResult.commandSpec().commandLine(); + if (isConfigSpecifiedInBothSources(environment, parseResult)) { + throwExceptionForBothSourcesSpecified(environment, parseResult, commandLine); + } + if (parseResult.hasMatchedOption(getConfigOptionName())) { + return getFromOption(parseResult, commandLine); + } + if (environment.containsKey(getConfigEnvName())) { + return getFromEnvironment(environment, commandLine); + } + return Optional.empty(); + } + + /** + * Gets the configuration resource from the command line option. + * + * @param parseResult the command line parse result + * @param commandLine the command line + * @return an Optional containing the configuration resource, or an empty Optional if the + * configuration resource was not specified in the command line option + */ + protected abstract Optional getFromOption( + final CommandLine.ParseResult parseResult, final CommandLine commandLine); + + /** + * Gets the configuration resource from the environment variable. + * + * @param environment the environment variables + * @param commandLine the command line + * @return an Optional containing the configuration resource, or an empty Optional if the + * configuration resource was not specified in the environment variable + */ + protected abstract Optional getFromEnvironment( + final Map environment, final CommandLine commandLine); + + /** + * Checks if the configuration resource is specified in both command line options and environment + * variables. + * + * @param environment the environment variables + * @param parseResult the command line parse result + * @return true if the configuration resource is specified in both places, false otherwise + */ + public boolean isConfigSpecifiedInBothSources( + final Map environment, final CommandLine.ParseResult parseResult) { + return parseResult.hasMatchedOption(getConfigOptionName()) + && environment.containsKey(getConfigEnvName()); + } + + /** + * Throws an exception if the configuration resource is specified in both command line options and + * environment variables. + * + * @param environment the environment variables + * @param parseResult the command line parse result + * @param commandLine the command line + */ + public void throwExceptionForBothSourcesSpecified( + final Map environment, + final CommandLine.ParseResult parseResult, + final CommandLine commandLine) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + "Both %s=%s and %s %s specified. Please specify only one.", + getConfigEnvName(), + getConfigOptionName(), + environment.get(getConfigEnvName()), + parseResult.matchedOption(getConfigOptionName()).stringValues())); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/CascadingDefaultProvider.java b/besu/src/main/java/org/hyperledger/besu/cli/util/CascadingDefaultProvider.java index 489d61af95..1e5d0bec6e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/CascadingDefaultProvider.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/CascadingDefaultProvider.java @@ -14,8 +14,6 @@ */ package org.hyperledger.besu.cli.util; -import static java.util.Arrays.asList; - import java.util.List; import picocli.CommandLine.IDefaultValueProvider; @@ -34,8 +32,8 @@ public class CascadingDefaultProvider implements IDefaultValueProvider { * * @param defaultValueProviders List of default value providers */ - public CascadingDefaultProvider(final IDefaultValueProvider... defaultValueProviders) { - this.defaultValueProviders = asList(defaultValueProviders); + public CascadingDefaultProvider(final List defaultValueProviders) { + this.defaultValueProviders = defaultValueProviders; } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigFileFinder.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigFileFinder.java new file mode 100644 index 0000000000..a17ef0b1a6 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigFileFinder.java @@ -0,0 +1,100 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.util; + +import static org.hyperledger.besu.cli.DefaultCommandValues.CONFIG_FILE_OPTION_NAME; + +import java.io.File; +import java.util.Map; +import java.util.Optional; + +import picocli.CommandLine; + +/** + * Class for finding configuration files. This class extends the AbstractConfigurationFinder and + * provides methods for finding configuration files based on command line options and environment + * variables. + */ +public class ConfigFileFinder extends AbstractConfigurationFinder { + private static final String CONFIG_FILE_ENV_NAME = "BESU_CONFIG_FILE"; + + /** + * Returns the name of the configuration option. + * + * @return the name of the configuration option + */ + @Override + protected String getConfigOptionName() { + return CONFIG_FILE_OPTION_NAME; + } + + /** + * Returns the name of the environment variable for the configuration. + * + * @return the name of the environment variable for the configuration + */ + @Override + protected String getConfigEnvName() { + return CONFIG_FILE_ENV_NAME; + } + + /** + * Gets the configuration file from the command line option. + * + * @param parseResult the command line parse result + * @param commandLine the command line + * @return an Optional containing the configuration file, or an empty Optional if the + * configuration file was not specified in the command line option + */ + @Override + public Optional getFromOption( + final CommandLine.ParseResult parseResult, final CommandLine commandLine) { + final CommandLine.Model.OptionSpec configFileOption = + parseResult.matchedOption(CONFIG_FILE_OPTION_NAME); + try { + File file = configFileOption.getter().get(); + if (!file.exists()) { + throw new CommandLine.ParameterException( + commandLine, + String.format("Unable to read TOML configuration, file not found: %s", file)); + } + return Optional.of(file); + } catch (final Exception e) { + throw new CommandLine.ParameterException(commandLine, e.getMessage(), e); + } + } + + /** + * Gets the configuration file from the environment variable. + * + * @param environment the environment variables + * @param commandLine the command line + * @return an Optional containing the configuration file, or an empty Optional if the + * configuration file was not specified in the environment variable + */ + @Override + public Optional getFromEnvironment( + final Map environment, final CommandLine commandLine) { + final File toml = new File(environment.get(CONFIG_FILE_ENV_NAME)); + if (!toml.exists()) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + "TOML file %s specified in environment variable %s not found", + CONFIG_FILE_ENV_NAME, environment.get(CONFIG_FILE_ENV_NAME))); + } + return Optional.of(toml); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java index 5866d43cde..52b6765282 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.util; import java.io.File; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -25,7 +26,6 @@ import picocli.CommandLine; import picocli.CommandLine.IDefaultValueProvider; import picocli.CommandLine.IExecutionStrategy; import picocli.CommandLine.IParameterExceptionHandler; -import picocli.CommandLine.Model.OptionSpec; import picocli.CommandLine.ParameterException; import picocli.CommandLine.ParseResult; @@ -54,8 +54,13 @@ public class ConfigOptionSearchAndRunHandler extends CommandLine.RunLast { @Override public List handle(final ParseResult parseResult) throws ParameterException { final CommandLine commandLine = parseResult.commandSpec().commandLine(); - final Optional configFile = findConfigFile(parseResult, commandLine); - commandLine.setDefaultValueProvider(createDefaultValueProvider(commandLine, configFile)); + + commandLine.setDefaultValueProvider( + createDefaultValueProvider( + commandLine, + new ConfigFileFinder().findConfiguration(environment, parseResult), + new ProfileFinder().findConfiguration(environment, parseResult))); + commandLine.setExecutionStrategy(resultHandler); commandLine.setParameterExceptionHandler(parameterExceptionHandler); commandLine.execute(parseResult.originalArgs().toArray(new String[0])); @@ -63,38 +68,6 @@ public class ConfigOptionSearchAndRunHandler extends CommandLine.RunLast { return new ArrayList<>(); } - private Optional findConfigFile( - final ParseResult parseResult, final CommandLine commandLine) { - if (parseResult.hasMatchedOption("--config-file") - && environment.containsKey("BESU_CONFIG_FILE")) { - throw new ParameterException( - commandLine, - String.format( - "TOML file specified using BESU_CONFIG_FILE=%s and --config-file %s", - environment.get("BESU_CONFIG_FILE"), - parseResult.matchedOption("--config-file").stringValues())); - } else if (parseResult.hasMatchedOption("--config-file")) { - final OptionSpec configFileOption = parseResult.matchedOption("--config-file"); - try { - return Optional.of(configFileOption.getter().get()); - } catch (final Exception e) { - throw new ParameterException(commandLine, e.getMessage(), e); - } - } else if (environment.containsKey("BESU_CONFIG_FILE")) { - final File toml = new File(environment.get("BESU_CONFIG_FILE")); - if (!toml.exists()) { - throw new ParameterException( - commandLine, - String.format( - "TOML file %s specified in environment variable BESU_CONFIG_FILE not found", - environment.get("BESU_CONFIG_FILE"))); - } - return Optional.of(toml); - } - - return Optional.empty(); - } - /** * Create default value provider default value provider. * @@ -104,14 +77,22 @@ public class ConfigOptionSearchAndRunHandler extends CommandLine.RunLast { */ @VisibleForTesting IDefaultValueProvider createDefaultValueProvider( - final CommandLine commandLine, final Optional configFile) { - if (configFile.isPresent()) { - return new CascadingDefaultProvider( - new EnvironmentVariableDefaultProvider(environment), - new TomlConfigFileDefaultProvider(commandLine, configFile.get())); - } else { - return new EnvironmentVariableDefaultProvider(environment); - } + final CommandLine commandLine, + final Optional configFile, + final Optional profile) { + List providers = new ArrayList<>(); + providers.add(new EnvironmentVariableDefaultProvider(environment)); + + configFile.ifPresent( + config -> { + if (config.exists()) { + providers.add(TomlConfigurationDefaultProvider.fromFile(commandLine, config)); + } + }); + + profile.ifPresent( + p -> providers.add(TomlConfigurationDefaultProvider.fromInputStream(commandLine, p))); + return new CascadingDefaultProvider(providers); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java new file mode 100644 index 0000000000..8df7382db7 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java @@ -0,0 +1,76 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.util; + +import static org.hyperledger.besu.cli.DefaultCommandValues.PROFILE_OPTION_NAME; + +import org.hyperledger.besu.cli.config.ProfileName; + +import java.io.InputStream; +import java.util.Map; +import java.util.Optional; + +import picocli.CommandLine; + +/** + * Class for finding profile configurations. This class extends the AbstractConfigurationFinder and + * provides methods for finding profile configurations based on command line options and environment + * variables. Each profile corresponds to a TOML configuration file that contains settings for + * various options. The profile to use can be specified with the '--profile' command line option or + * the 'BESU_PROFILE' environment variable. + */ +public class ProfileFinder extends AbstractConfigurationFinder { + private static final String PROFILE_ENV_NAME = "BESU_PROFILE"; + + @Override + protected String getConfigOptionName() { + return PROFILE_OPTION_NAME; + } + + @Override + protected String getConfigEnvName() { + return PROFILE_ENV_NAME; + } + + @Override + public Optional getFromOption( + final CommandLine.ParseResult parseResult, final CommandLine commandLine) { + try { + return getProfile(parseResult.matchedOption(PROFILE_OPTION_NAME).getter().get(), commandLine); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public Optional getFromEnvironment( + final Map environment, final CommandLine commandLine) { + return getProfile(ProfileName.valueOf(environment.get(PROFILE_ENV_NAME)), commandLine); + } + + private static Optional getProfile( + final ProfileName profileName, final CommandLine commandLine) { + return Optional.of(getTomlFile(commandLine, profileName.getConfigFile())); + } + + private static InputStream getTomlFile(final CommandLine commandLine, final String file) { + InputStream resourceUrl = ProfileFinder.class.getClassLoader().getResourceAsStream(file); + if (resourceUrl == null) { + throw new CommandLine.ParameterException( + commandLine, String.format("TOML file %s not found", file)); + } + return resourceUrl; + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigFileDefaultProvider.java b/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java similarity index 75% rename from besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigFileDefaultProvider.java rename to besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java index c765b7f5df..a346e260d1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigFileDefaultProvider.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java @@ -17,9 +17,13 @@ package org.hyperledger.besu.cli.util; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.util.number.Fraction; import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; @@ -41,21 +45,52 @@ import picocli.CommandLine.Model.OptionSpec; import picocli.CommandLine.ParameterException; /** The Toml config file default value provider used by PicoCli. */ -public class TomlConfigFileDefaultProvider implements IDefaultValueProvider { +public class TomlConfigurationDefaultProvider implements IDefaultValueProvider { private final CommandLine commandLine; - private final File configFile; + private final InputStream configurationInputStream; private TomlParseResult result; /** * Instantiates a new Toml config file default value provider. * * @param commandLine the command line - * @param configFile the config file + * @param configurationInputStream the input stream */ - public TomlConfigFileDefaultProvider(final CommandLine commandLine, final File configFile) { + private TomlConfigurationDefaultProvider( + final CommandLine commandLine, final InputStream configurationInputStream) { this.commandLine = commandLine; - this.configFile = configFile; + this.configurationInputStream = configurationInputStream; + } + + /** + * Creates a new TomlConfigurationDefaultProvider from a file. + * + * @param commandLine the command line + * @param configFile the configuration file + * @return a new TomlConfigurationDefaultProvider + * @throws ParameterException if the configuration file is not found + */ + public static TomlConfigurationDefaultProvider fromFile( + final CommandLine commandLine, final File configFile) { + try { + return new TomlConfigurationDefaultProvider(commandLine, new FileInputStream(configFile)); + } catch (final FileNotFoundException e) { + throw new ParameterException( + commandLine, "Unable to read TOML configuration, file not found."); + } + } + + /** + * Creates a new TomlConfigurationDefaultProvider from an input stream. + * + * @param commandLine the command line + * @param inputStream the input stream + * @return a new TomlConfigurationDefaultProvider + */ + public static TomlConfigurationDefaultProvider fromInputStream( + final CommandLine commandLine, final InputStream inputStream) { + return new TomlConfigurationDefaultProvider(commandLine, inputStream); } @Override @@ -70,35 +105,33 @@ public class TomlConfigFileDefaultProvider implements IDefaultValueProvider { private String getConfigurationValue(final OptionSpec optionSpec) { // NOTE: This temporary fix is necessary to make certain options be treated as a multi-value. // This can be done automatically by picocli if the object implements Collection. - final boolean isArray = - getKeyName(optionSpec).map(keyName -> result.isArray(keyName)).orElse(false); - final String defaultValue; + final boolean isArray = getKeyName(optionSpec).map(result::isArray).orElse(false); - // Convert config values to the right string representation for default string value if (optionSpec.type().equals(Boolean.class) || optionSpec.type().equals(boolean.class)) { - defaultValue = getBooleanEntryAsString(optionSpec); + return getBooleanEntryAsString(optionSpec); } else if (optionSpec.isMultiValue() || isArray) { - defaultValue = getListEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Integer.class) || optionSpec.type().equals(int.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Long.class) || optionSpec.type().equals(long.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Wei.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(BigInteger.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Double.class) || optionSpec.type().equals(double.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Float.class) || optionSpec.type().equals(float.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Percentage.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Fraction.class)) { - defaultValue = getNumericEntryAsString(optionSpec); + return getListEntryAsString(optionSpec); + } else if (isNumericType(optionSpec.type())) { + return getNumericEntryAsString(optionSpec); } else { // else will be treated as String - defaultValue = getEntryAsString(optionSpec); + return getEntryAsString(optionSpec); } - return defaultValue; + } + + private boolean isNumericType(final Class type) { + return type.equals(Integer.class) + || type.equals(int.class) + || type.equals(Long.class) + || type.equals(long.class) + || type.equals(Wei.class) + || type.equals(BigInteger.class) + || type.equals(Double.class) + || type.equals(double.class) + || type.equals(Float.class) + || type.equals(float.class) + || type.equals(Percentage.class) + || type.equals(Fraction.class) + || type.equals(PositiveNumber.class); } private String getEntryAsString(final OptionSpec spec) { @@ -195,7 +228,8 @@ public class TomlConfigFileDefaultProvider implements IDefaultValueProvider { private void checkConfigurationValidity() { if (result == null || result.isEmpty()) throw new ParameterException( - commandLine, String.format("Unable to read TOML configuration file %s", configFile)); + commandLine, + String.format("Unable to read TOML configuration file %s", configurationInputStream)); } /** Load configuration from file. */ @@ -203,7 +237,7 @@ public class TomlConfigFileDefaultProvider implements IDefaultValueProvider { if (result == null) { try { - final TomlParseResult result = Toml.parse(configFile.toPath()); + final TomlParseResult result = Toml.parse(configurationInputStream); if (result.hasErrors()) { final String errors = @@ -224,7 +258,6 @@ public class TomlConfigFileDefaultProvider implements IDefaultValueProvider { commandLine, "Unable to read TOML configuration, file not found."); } } - checkConfigurationValidity(); } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 543de6b2fc..6310de521b 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -83,6 +83,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner; @@ -91,6 +92,7 @@ import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -781,6 +783,17 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides final JsonRpcMethods additionalJsonRpcMethodFactory = createAdditionalJsonRpcMethodFactory(protocolContext); + if (dataStorageConfiguration.getUnstable().getBonsaiLimitTrieLogsEnabled() + && DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) { + final TrieLogManager trieLogManager = + ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager(); + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = + worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class); + final TrieLogPruner trieLogPruner = + createTrieLogPruner(worldStateKeyValueStorage, blockchain, scheduler); + trieLogManager.subscribe(trieLogPruner); + } + final List closeables = new ArrayList<>(); closeables.add(protocolContext.getWorldStateArchive()); closeables.add(storageProvider); @@ -809,6 +822,26 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides dataStorageConfiguration); } + private TrieLogPruner createTrieLogPruner( + final WorldStateKeyValueStorage worldStateStorage, + final Blockchain blockchain, + final EthScheduler scheduler) { + final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get(); + final boolean isProofOfStake = genesisConfigOptions.getTerminalTotalDifficulty().isPresent(); + + final TrieLogPruner trieLogPruner = + new TrieLogPruner( + (BonsaiWorldStateKeyValueStorage) worldStateStorage, + blockchain, + scheduler::executeServiceTask, + dataStorageConfiguration.getBonsaiMaxLayersToLoad(), + dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningWindowSize(), + isProofOfStake); + trieLogPruner.initialize(); + + return trieLogPruner; + } + /** * Create synchronizer synchronizer. * @@ -1070,29 +1103,15 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides final CachedMerkleTrieLoader cachedMerkleTrieLoader) { return switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI -> { - final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get(); - final boolean isProofOfStake = - genesisConfigOptions.getTerminalTotalDifficulty().isPresent(); final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class); - final TrieLogPruner trieLogPruner = - dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled() - ? new TrieLogPruner( - worldStateKeyValueStorage, - blockchain, - dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(), - dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(), - isProofOfStake) - : TrieLogPruner.noOpTrieLogPruner(); - trieLogPruner.initialize(); yield new BonsaiWorldStateProvider( worldStateKeyValueStorage, blockchain, Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), cachedMerkleTrieLoader, besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), - evmConfiguration, - trieLogPruner); + evmConfiguration); } case FOREST -> { final WorldStatePreimageStorage preimageStorage = diff --git a/besu/src/main/resources/log4j2.xml b/besu/src/main/resources/log4j2.xml index 6bf83a973f..cb412bca71 100644 --- a/besu/src/main/resources/log4j2.xml +++ b/besu/src/main/resources/log4j2.xml @@ -42,6 +42,9 @@ + + + diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 04d9f01553..938c25af9d 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.cli; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; @@ -28,12 +27,10 @@ import static org.hyperledger.besu.cli.config.NetworkName.HOLESKY; import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; import static org.hyperledger.besu.cli.config.NetworkName.MORDOR; import static org.hyperledger.besu.cli.config.NetworkName.SEPOLIA; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ENGINE; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.PERM; -import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.WEB3; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.GOERLI_BOOTSTRAP_NODES; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.GOERLI_DISCOVERY_URL; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES; @@ -71,8 +68,6 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; @@ -123,7 +118,6 @@ import com.google.common.collect.Lists; import com.google.common.io.Resources; import io.vertx.core.json.JsonObject; import org.apache.commons.io.FileUtils; -import org.apache.commons.text.StringEscapeUtils; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.toml.Toml; import org.apache.tuweni.toml.TomlParseResult; @@ -306,17 +300,18 @@ public class BesuCommandTest extends CommandTestAbstract { final Path tempConfigFilePath = createTempFile("an-invalid-file-name-without-extension", ""); parseCommand("--config-file", tempConfigFilePath.toString()); - final String expectedOutputStart = - "Unable to read TOML configuration file " + tempConfigFilePath; + final String expectedOutputStart = "Unable to read TOML configuration file"; assertThat(commandErrorOutput.toString(UTF_8)).startsWith(expectedOutputStart); assertThat(commandOutput.toString(UTF_8)).isEmpty(); } @Test public void callingWithConfigOptionButTomlFileNotFoundShouldDisplayHelp() { - parseCommand("--config-file", "./an-invalid-file-name-sdsd87sjhqoi34io23.toml"); + String invalidFile = "./an-invalid-file-name-sdsd87sjhqoi34io23.toml"; + parseCommand("--config-file", invalidFile); - final String expectedOutputStart = "Unable to read TOML configuration, file not found."; + final String expectedOutputStart = + String.format("Unable to read TOML configuration, file not found: %s", invalidFile); assertThat(commandErrorOutput.toString(UTF_8)).startsWith(expectedOutputStart); assertThat(commandOutput.toString(UTF_8)).isEmpty(); } @@ -364,81 +359,6 @@ public class BesuCommandTest extends CommandTestAbstract { assertThat(commandOutput.toString(UTF_8)).isEmpty(); } - @Test - public void overrideDefaultValuesIfKeyIsPresentInConfigFile(final @TempDir File dataFolder) - throws IOException { - final URL configFile = this.getClass().getResource("/complete_config.toml"); - final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON); - final String updatedConfig = - Resources.toString(configFile, UTF_8) - .replace("/opt/besu/genesis.json", escapeTomlString(genesisFile.toString())) - .replace( - "data-path=\"/opt/besu\"", - "data-path=\"" + escapeTomlString(dataFolder.getPath()) + "\""); - - final Path toml = createTempFile("toml", updatedConfig.getBytes(UTF_8)); - - final List expectedApis = asList(ETH.name(), WEB3.name()); - - final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); - jsonRpcConfiguration.setEnabled(false); - jsonRpcConfiguration.setHost("5.6.7.8"); - jsonRpcConfiguration.setPort(5678); - jsonRpcConfiguration.setCorsAllowedDomains(Collections.emptyList()); - jsonRpcConfiguration.setRpcApis(expectedApis); - jsonRpcConfiguration.setMaxActiveConnections(1000); - - final GraphQLConfiguration graphQLConfiguration = GraphQLConfiguration.createDefault(); - graphQLConfiguration.setEnabled(false); - graphQLConfiguration.setHost("6.7.8.9"); - graphQLConfiguration.setPort(6789); - - final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); - webSocketConfiguration.setEnabled(false); - webSocketConfiguration.setHost("9.10.11.12"); - webSocketConfiguration.setPort(9101); - webSocketConfiguration.setRpcApis(expectedApis); - - final MetricsConfiguration metricsConfiguration = - MetricsConfiguration.builder().enabled(false).host("8.6.7.5").port(309).build(); - - parseCommand("--config-file", toml.toString()); - - verify(mockRunnerBuilder).discovery(eq(false)); - verify(mockRunnerBuilder).ethNetworkConfig(ethNetworkConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).p2pAdvertisedHost(eq("1.2.3.4")); - verify(mockRunnerBuilder).p2pListenPort(eq(1234)); - verify(mockRunnerBuilder).jsonRpcConfiguration(eq(jsonRpcConfiguration)); - verify(mockRunnerBuilder).graphQLConfiguration(eq(graphQLConfiguration)); - verify(mockRunnerBuilder).webSocketConfiguration(eq(webSocketConfiguration)); - verify(mockRunnerBuilder).metricsConfiguration(eq(metricsConfiguration)); - verify(mockRunnerBuilder).build(); - - final List nodes = - asList( - EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567"), - EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567"), - EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567")); - assertThat(ethNetworkConfigArgumentCaptor.getValue().getBootNodes()).isEqualTo(nodes); - - final EthNetworkConfig networkConfig = - new EthNetworkConfig.Builder(EthNetworkConfig.getNetworkConfig(MAINNET)) - .setNetworkId(BigInteger.valueOf(42)) - .setGenesisConfig(encodeJsonGenesis(GENESIS_VALID_JSON)) - .setBootNodes(nodes) - .setDnsDiscoveryUrl(null) - .build(); - verify(mockControllerBuilder).dataDirectory(eq(dataFolder.toPath())); - verify(mockControllerBuilderFactory).fromEthNetworkConfig(eq(networkConfig), any(), any()); - verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); - - assertThat(syncConfigurationCaptor.getValue().getSyncMode()).isEqualTo(SyncMode.FAST); - assertThat(syncConfigurationCaptor.getValue().getFastSyncMinimumPeerCount()).isEqualTo(13); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - @Test public void nodePermissionsSmartContractWithoutOptionMustError() { parseCommand("--permissions-nodes-contract-address"); @@ -861,81 +781,6 @@ public class BesuCommandTest extends CommandTestAbstract { .isEmpty(); } - @Test - public void noOverrideDefaultValuesIfKeyIsNotPresentInConfigFile() { - final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); - - parseCommand("--config-file", configFile); - final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); - - final GraphQLConfiguration graphQLConfiguration = GraphQLConfiguration.createDefault(); - - final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); - - final MetricsConfiguration metricsConfiguration = MetricsConfiguration.builder().build(); - - verify(mockRunnerBuilder).discovery(eq(true)); - verify(mockRunnerBuilder) - .ethNetworkConfig( - new EthNetworkConfig( - EthNetworkConfig.jsonConfig(MAINNET), - MAINNET.getNetworkId(), - MAINNET_BOOTSTRAP_NODES, - MAINNET_DISCOVERY_URL)); - verify(mockRunnerBuilder).p2pAdvertisedHost(eq("127.0.0.1")); - verify(mockRunnerBuilder).p2pListenPort(eq(30303)); - verify(mockRunnerBuilder).jsonRpcConfiguration(eq(jsonRpcConfiguration)); - verify(mockRunnerBuilder).graphQLConfiguration(eq(graphQLConfiguration)); - verify(mockRunnerBuilder).webSocketConfiguration(eq(webSocketConfiguration)); - verify(mockRunnerBuilder).metricsConfiguration(eq(metricsConfiguration)); - verify(mockRunnerBuilder).build(); - verify(mockControllerBuilder).build(); - verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); - - final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue(); - assertThat(syncConfig.getSyncMode()).isEqualTo(SyncMode.FAST); - assertThat(syncConfig.getFastSyncMinimumPeerCount()).isEqualTo(5); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void envVariableOverridesValueFromConfigFile() { - final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); - final String expectedCoinbase = "0x0000000000000000000000000000000000000004"; - setEnvironmentVariable("BESU_MINER_COINBASE", expectedCoinbase); - parseCommand("--config-file", configFile); - - verify(mockControllerBuilder) - .miningParameters( - ImmutableMiningParameters.builder() - .mutableInitValues( - MutableInitValues.builder() - .coinbase(Address.fromHexString(expectedCoinbase)) - .build()) - .build()); - } - - @Test - public void cliOptionOverridesEnvVariableAndConfig() { - final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); - final String expectedCoinbase = "0x0000000000000000000000000000000000000006"; - setEnvironmentVariable("BESU_MINER_COINBASE", "0x0000000000000000000000000000000000000004"); - parseCommand("--config-file", configFile, "--miner-coinbase", expectedCoinbase); - - verify(mockControllerBuilder) - .miningParameters( - ImmutableMiningParameters.builder() - .mutableInitValues( - MutableInitValues.builder() - .coinbase(Address.fromHexString(expectedCoinbase)) - .build()) - .build()); - } - @Test public void nodekeyOptionMustBeUsed() throws Exception { final File file = new File("./specific/enclavePrivateKey"); @@ -2298,18 +2143,6 @@ public class BesuCommandTest extends CommandTestAbstract { "Invalid value for option '--rpc-http-api-methods-no-auth', options must be valid RPC methods"); } - @Test - public void rpcWsNoAuthApiMethodsCannotBeInvalid() { - parseCommand("--rpc-ws-enabled", "--rpc-ws-api-methods-no-auth", "invalid"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "Invalid value for option '--rpc-ws-api-methods-no-auth', options must be valid RPC methods"); - } - @Test public void rpcHttpOptionsRequiresServiceToBeEnabled() { parseCommand( @@ -2455,18 +2288,6 @@ public class BesuCommandTest extends CommandTestAbstract { .contains("Invalid value for option '--rpc-http-api': invalid entries found [BOB]"); } - @Test - public void rpcWsApisPropertyWithInvalidEntryMustDisplayError() { - parseCommand("--rpc-ws-api", "ETH,BOB,TEST"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - - assertThat(commandErrorOutput.toString(UTF_8).trim()) - .contains("Invalid value for option '--rpc-ws-api': invalid entries found [BOB, TEST]"); - } - @Test public void rpcApisPropertyWithPluginNamespaceAreValid() { @@ -2548,35 +2369,6 @@ public class BesuCommandTest extends CommandTestAbstract { assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } - @Test - public void rpcWsMaxFrameSizePropertyMustBeUsed() { - final int maxFrameSize = 65535; - parseCommand("--rpc-ws-max-frame-size", String.valueOf(maxFrameSize)); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getMaxFrameSize()).isEqualTo(maxFrameSize); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsMaxActiveConnectionsPropertyMustBeUsed() { - final int maxConnections = 99; - parseCommand("--rpc-ws-max-active-connections", String.valueOf(maxConnections)); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getMaxActiveConnections()) - .isEqualTo(maxConnections); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - @Test public void rpcHttpTlsRequiresRpcHttpEnabled() { parseCommand("--rpc-http-tls-enabled"); @@ -3388,129 +3180,6 @@ public class BesuCommandTest extends CommandTestAbstract { assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } - @Test - public void rpcWsRpcEnabledPropertyMustBeUsed() { - parseCommand("--rpc-ws-enabled"); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().isEnabled()).isTrue(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsOptionsRequiresServiceToBeEnabled() { - parseCommand( - "--rpc-ws-api", - "ETH,NET", - "--rpc-ws-host", - "0.0.0.0", - "--rpc-ws-port", - "1234", - "--rpc-ws-max-active-connections", - "77", - "--rpc-ws-max-frame-size", - "65535"); - - verifyOptionsConstraintLoggerCall( - "--rpc-ws-enabled", - "--rpc-ws-host", - "--rpc-ws-port", - "--rpc-ws-api", - "--rpc-ws-max-active-connections", - "--rpc-ws-max-frame-size"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsOptionsRequiresServiceToBeEnabledToml() throws IOException { - final Path toml = - createTempFile( - "toml", - "rpc-ws-api=[\"ETH\", \"NET\"]\n" - + "rpc-ws-host=\"0.0.0.0\"\n" - + "rpc-ws-port=1234\n" - + "rpc-ws-max-active-connections=77\n" - + "rpc-ws-max-frame-size=65535\n"); - - parseCommand("--config-file", toml.toString()); - - verifyOptionsConstraintLoggerCall( - "--rpc-ws-enabled", - "--rpc-ws-host", - "--rpc-ws-port", - "--rpc-ws-api", - "--rpc-ws-max-active-connections", - "--rpc-ws-max-frame-size"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsApiPropertyMustBeUsed() { - final TestBesuCommand command = parseCommand("--rpc-ws-enabled", "--rpc-ws-api", "ETH, NET"); - - assertThat(command).isNotNull(); - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getRpcApis()) - .containsExactlyInAnyOrder(ETH.name(), NET.name()); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsHostAndPortOptionMustBeUsed() { - final String host = "1.2.3.4"; - final int port = 1234; - parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host, "--rpc-ws-port", String.valueOf(port)); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(wsRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsHostAndMayBeLocalhost() { - final String host = "localhost"; - parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsHostAndMayBeIPv6() { - final String host = "2600:DB8::8545"; - parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - @Test public void metricsEnabledPropertyDefaultIsFalse() { parseCommand(); @@ -4300,35 +3969,6 @@ public class BesuCommandTest extends CommandTestAbstract { "No Payload Provider has been provided. You must register one when enabling privacy plugin!"); } - private static String escapeTomlString(final String s) { - return StringEscapeUtils.escapeJava(s); - } - - /** - * Check logger calls - * - *

Here we check the calls to logger and not the result of the log line as we don't test the - * logger itself but the fact that we call it. - * - * @param dependentOptions the string representing the list of dependent options names - * @param mainOption the main option name - */ - private void verifyOptionsConstraintLoggerCall( - final String mainOption, final String... dependentOptions) { - verify(mockLogger, atLeast(1)) - .warn( - stringArgumentCaptor.capture(), - stringArgumentCaptor.capture(), - stringArgumentCaptor.capture()); - assertThat(stringArgumentCaptor.getAllValues().get(0)).isEqualTo(DEPENDENCY_WARNING_MSG); - - for (final String option : dependentOptions) { - assertThat(stringArgumentCaptor.getAllValues().get(1)).contains(option); - } - - assertThat(stringArgumentCaptor.getAllValues().get(2)).isEqualTo(mainOption); - } - /** * Check logger calls * @@ -4518,17 +4158,6 @@ public class BesuCommandTest extends CommandTestAbstract { .isEqualTo(JwtAlgorithm.ES256); } - @Test - public void webSocketAuthenticationAlgorithIsConfigured() { - parseCommand("--rpc-ws-authentication-jwt-algorithm", "ES256"); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getAuthenticationAlgorithm()) - .isEqualTo(JwtAlgorithm.ES256); - } - @Test public void httpAuthenticationPublicKeyIsConfigured() throws IOException { final Path publicKey = Files.createTempFile("public_key", ""); @@ -4552,29 +4181,6 @@ public class BesuCommandTest extends CommandTestAbstract { "Unable to authenticate JSON-RPC HTTP endpoint without a supplied credentials file or authentication public key file"); } - @Test - public void wsAuthenticationPublicKeyIsConfigured() throws IOException { - final Path publicKey = Files.createTempFile("public_key", ""); - parseCommand("--rpc-ws-authentication-jwt-public-key-file", publicKey.toString()); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getAuthenticationPublicKeyFile().getPath()) - .isEqualTo(publicKey.toString()); - } - - @Test - public void wsAuthenticationWithoutRequiredConfiguredOptionsMustFail() { - parseCommand("--rpc-ws-enabled", "--rpc-ws-authentication-enabled"); - - verifyNoInteractions(mockRunnerBuilder); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "Unable to authenticate JSON-RPC WebSocket endpoint without a supplied credentials file or authentication public key file"); - } - @Test public void privHttpApisWithPrivacyDisabledLogsWarning() { parseCommand("--privacy-enabled=false", "--rpc-http-api", "PRIV", "--rpc-http-enabled"); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java new file mode 100644 index 0000000000..e9707f1dfd --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java @@ -0,0 +1,306 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.cli.config.NetworkName.DEV; +import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.WEB3; +import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES; +import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_DISCOVERY_URL; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; + +import org.hyperledger.besu.cli.config.EthNetworkConfig; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; +import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration; +import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; +import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; +import org.hyperledger.besu.ethereum.eth.sync.SyncMode; +import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; +import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; +import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; +import org.hyperledger.besu.plugin.data.EnodeURL; + +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.net.URL; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +import com.google.common.io.Resources; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +public class CascadingDefaultProviderTest extends CommandTestAbstract { + private static final int GENESIS_CONFIG_TEST_CHAINID = 3141592; + private static final String VALID_NODE_ID = + "6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0"; + private static final JsonObject GENESIS_VALID_JSON = + (new JsonObject()) + .put("config", (new JsonObject()).put("chainId", GENESIS_CONFIG_TEST_CHAINID)); + + /** + * Test if the default values are overridden if the key is present in the configuration file. The + * test checks if the configuration file correctly overrides the default values for various + * settings, such as the JSON-RPC configuration, GraphQL configuration, WebSocket configuration, + * and metrics configuration. + */ + @Test + public void overrideDefaultValuesIfKeyIsPresentInConfigFile(final @TempDir File dataFolder) + throws IOException { + final URL configFile = this.getClass().getResource("/complete_config.toml"); + final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON); + final String updatedConfig = + Resources.toString(configFile, UTF_8) + .replace("/opt/besu/genesis.json", escapeTomlString(genesisFile.toString())) + .replace( + "data-path=\"/opt/besu\"", + "data-path=\"" + escapeTomlString(dataFolder.getPath()) + "\""); + + final Path toml = createTempFile("toml", updatedConfig.getBytes(UTF_8)); + + final List expectedApis = asList(ETH.name(), WEB3.name()); + + final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); + jsonRpcConfiguration.setEnabled(false); + jsonRpcConfiguration.setHost("5.6.7.8"); + jsonRpcConfiguration.setPort(5678); + jsonRpcConfiguration.setCorsAllowedDomains(Collections.emptyList()); + jsonRpcConfiguration.setRpcApis(expectedApis); + jsonRpcConfiguration.setMaxActiveConnections(1000); + + final GraphQLConfiguration graphQLConfiguration = GraphQLConfiguration.createDefault(); + graphQLConfiguration.setEnabled(false); + graphQLConfiguration.setHost("6.7.8.9"); + graphQLConfiguration.setPort(6789); + + final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); + webSocketConfiguration.setEnabled(false); + webSocketConfiguration.setHost("9.10.11.12"); + webSocketConfiguration.setPort(9101); + webSocketConfiguration.setRpcApis(expectedApis); + + final MetricsConfiguration metricsConfiguration = + MetricsConfiguration.builder().enabled(false).host("8.6.7.5").port(309).build(); + + parseCommand("--config-file", toml.toString()); + + verify(mockRunnerBuilder).discovery(eq(false)); + verify(mockRunnerBuilder).ethNetworkConfig(ethNetworkConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).p2pAdvertisedHost(eq("1.2.3.4")); + verify(mockRunnerBuilder).p2pListenPort(eq(1234)); + verify(mockRunnerBuilder).jsonRpcConfiguration(eq(jsonRpcConfiguration)); + verify(mockRunnerBuilder).graphQLConfiguration(eq(graphQLConfiguration)); + verify(mockRunnerBuilder).webSocketConfiguration(eq(webSocketConfiguration)); + verify(mockRunnerBuilder).metricsConfiguration(eq(metricsConfiguration)); + verify(mockRunnerBuilder).build(); + + final List nodes = + asList( + EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567"), + EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567"), + EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567")); + assertThat(ethNetworkConfigArgumentCaptor.getValue().getBootNodes()).isEqualTo(nodes); + + final EthNetworkConfig networkConfig = + new EthNetworkConfig.Builder(EthNetworkConfig.getNetworkConfig(MAINNET)) + .setNetworkId(BigInteger.valueOf(42)) + .setGenesisConfig(encodeJsonGenesis(GENESIS_VALID_JSON)) + .setBootNodes(nodes) + .setDnsDiscoveryUrl(null) + .build(); + verify(mockControllerBuilder).dataDirectory(eq(dataFolder.toPath())); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(eq(networkConfig), any(), any()); + verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); + + assertThat(syncConfigurationCaptor.getValue().getSyncMode()).isEqualTo(SyncMode.FAST); + assertThat(syncConfigurationCaptor.getValue().getFastSyncMinimumPeerCount()).isEqualTo(13); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + /** + * Test if the default values are not overridden if the key is not present in the configuration + * file. The test checks if the default values for various settings remain unchanged when the + * corresponding keys are not present in the configuration file. + */ + @Test + public void noOverrideDefaultValuesIfKeyIsNotPresentInConfigFile() { + final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); + + parseCommand("--config-file", configFile); + final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); + + final GraphQLConfiguration graphQLConfiguration = GraphQLConfiguration.createDefault(); + + final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); + + final MetricsConfiguration metricsConfiguration = MetricsConfiguration.builder().build(); + + verify(mockRunnerBuilder).discovery(eq(true)); + verify(mockRunnerBuilder) + .ethNetworkConfig( + new EthNetworkConfig( + EthNetworkConfig.jsonConfig(MAINNET), + MAINNET.getNetworkId(), + MAINNET_BOOTSTRAP_NODES, + MAINNET_DISCOVERY_URL)); + verify(mockRunnerBuilder).p2pAdvertisedHost(eq("127.0.0.1")); + verify(mockRunnerBuilder).p2pListenPort(eq(30303)); + verify(mockRunnerBuilder).jsonRpcConfiguration(eq(jsonRpcConfiguration)); + verify(mockRunnerBuilder).graphQLConfiguration(eq(graphQLConfiguration)); + verify(mockRunnerBuilder).webSocketConfiguration(eq(webSocketConfiguration)); + verify(mockRunnerBuilder).metricsConfiguration(eq(metricsConfiguration)); + verify(mockRunnerBuilder).build(); + verify(mockControllerBuilder).build(); + verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); + + final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue(); + assertThat(syncConfig.getSyncMode()).isEqualTo(SyncMode.FAST); + assertThat(syncConfig.getFastSyncMinimumPeerCount()).isEqualTo(5); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + /** + * Test if the environment variable overrides the value from the configuration file. The test + * checks if the value of the miner's coinbase address set through an environment variable + * correctly overrides the value specified in the configuration file. + */ + @Test + public void envVariableOverridesValueFromConfigFile() { + final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); + final String expectedCoinbase = "0x0000000000000000000000000000000000000004"; + setEnvironmentVariable("BESU_MINER_COINBASE", expectedCoinbase); + parseCommand("--config-file", configFile); + + verify(mockControllerBuilder) + .miningParameters( + ImmutableMiningParameters.builder() + .mutableInitValues( + ImmutableMiningParameters.MutableInitValues.builder() + .coinbase(Address.fromHexString(expectedCoinbase)) + .build()) + .build()); + } + + /** + * Test if the command line option overrides the environment variable and configuration. The test + * checks if the value of the miner's coinbase address set through a command line option correctly + * overrides the value specified in the environment variable and the configuration file. + */ + @Test + public void cliOptionOverridesEnvVariableAndConfig() { + final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); + final String expectedCoinbase = "0x0000000000000000000000000000000000000006"; + setEnvironmentVariable("BESU_MINER_COINBASE", "0x0000000000000000000000000000000000000004"); + parseCommand("--config-file", configFile, "--miner-coinbase", expectedCoinbase); + + verify(mockControllerBuilder) + .miningParameters( + ImmutableMiningParameters.builder() + .mutableInitValues( + ImmutableMiningParameters.MutableInitValues.builder() + .coinbase(Address.fromHexString(expectedCoinbase)) + .build()) + .build()); + } + + /** + * Test if the profile option sets the correct defaults. The test checks if the 'dev' profile + * correctly sets the network ID to the expected value. + */ + @Test + public void profileOptionShouldSetCorrectDefaults() { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + parseCommand("--profile", "dev"); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); + verify(mockControllerBuilder).build(); + + final EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getNetworkId()).isEqualTo(DEV.getNetworkId()); + } + + /** + * Test if the command line option overrides the profile configuration. The test checks if the + * network ID set through a command line option correctly overrides the value specified in the + * 'dev' profile. + */ + @Test + public void cliOptionOverridesProfileConfiguration() { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + parseCommand("--profile", "dev", "--network", "MAINNET"); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); + verify(mockControllerBuilder).build(); + + final EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getNetworkId()).isEqualTo(MAINNET.getNetworkId()); + } + + /** + * Test if the configuration file overrides the profile configuration. The test checks if the + * network ID specified in the configuration file correctly overrides the value specified in the + * 'dev' profile. + */ + @Test + public void configFileOverridesProfileConfiguration() { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); + parseCommand("--profile", "dev", "--config-file", configFile); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); + verify(mockControllerBuilder).build(); + + final EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getNetworkId()).isEqualTo(MAINNET.getNetworkId()); + } + + /** + * Test if the environment variable overrides the profile configuration. The test checks if the + * network ID set through an environment variable correctly overrides the value specified in the + * 'dev' profile. + */ + @Test + public void environmentVariableOverridesProfileConfiguration() { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + setEnvironmentVariable("BESU_NETWORK", "MAINNET"); + parseCommand("--profile", "dev"); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); + verify(mockControllerBuilder).build(); + + final EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getNetworkId()).isEqualTo(MAINNET.getNetworkId()); + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandLineUtilsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandLineUtilsTest.java index 726bf38965..d8e8912eef 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandLineUtilsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandLineUtilsTest.java @@ -22,7 +22,7 @@ import static picocli.CommandLine.defaultExceptionHandler; import org.hyperledger.besu.cli.util.CommandLineUtils; import org.hyperledger.besu.cli.util.EnvironmentVariableDefaultProvider; -import org.hyperledger.besu.cli.util.TomlConfigFileDefaultProvider; +import org.hyperledger.besu.cli.util.TomlConfigurationDefaultProvider; import org.hyperledger.besu.util.StringUtils; import java.io.IOException; @@ -252,7 +252,7 @@ public class CommandLineUtilsTest { final AbstractTestCommand testCommand = new TestMultiCommandWithDeps(mockLogger); testCommand.commandLine.setDefaultValueProvider( - new TomlConfigFileDefaultProvider(testCommand.commandLine, toml.toFile())); + TomlConfigurationDefaultProvider.fromFile(testCommand.commandLine, toml.toFile())); testCommand.commandLine.parseWithHandlers(new RunLast(), defaultExceptionHandler()); verifyMultiOptionsConstraintLoggerCall( diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index e9d8d55964..1609758b96 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -15,14 +15,18 @@ package org.hyperledger.besu.cli; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.Runner; @@ -40,6 +44,7 @@ import org.hyperledger.besu.cli.options.unstable.MetricsCLIOptions; import org.hyperledger.besu.cli.options.unstable.NetworkingOptions; import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions; import org.hyperledger.besu.components.BesuComponent; +import org.hyperledger.besu.config.GenesisConfigOptions; import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfiguration; import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfigurationProvider; import org.hyperledger.besu.controller.BesuController; @@ -109,6 +114,7 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.core.json.JsonObject; +import org.apache.commons.text.StringEscapeUtils; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.awaitility.Awaitility; @@ -128,20 +134,37 @@ import picocli.CommandLine.RunLast; @ExtendWith(MockitoExtension.class) public abstract class CommandTestAbstract { private static final Logger TEST_LOGGER = LoggerFactory.getLogger(CommandTestAbstract.class); + + protected static final int POA_BLOCK_PERIOD_SECONDS = 5; protected static final JsonObject VALID_GENESIS_QBFT_POST_LONDON = (new JsonObject()) .put( "config", new JsonObject() .put("londonBlock", 0) - .put("qbft", new JsonObject().put("blockperiodseconds", 5))); + .put( + "qbft", + new JsonObject().put("blockperiodseconds", POA_BLOCK_PERIOD_SECONDS))); protected static final JsonObject VALID_GENESIS_IBFT2_POST_LONDON = (new JsonObject()) .put( "config", new JsonObject() .put("londonBlock", 0) - .put("ibft2", new JsonObject().put("blockperiodseconds", 5))); + .put( + "ibft2", + new JsonObject().put("blockperiodseconds", POA_BLOCK_PERIOD_SECONDS))); + + protected static final JsonObject VALID_GENESIS_CLIQUE_POST_LONDON = + (new JsonObject()) + .put( + "config", + new JsonObject() + .put("londonBlock", 0) + .put( + "clique", + new JsonObject().put("blockperiodseconds", POA_BLOCK_PERIOD_SECONDS))); + protected final PrintStream originalOut = System.out; protected final PrintStream originalErr = System.err; protected final ByteArrayOutputStream commandOutput = new ByteArrayOutputStream(); @@ -556,6 +579,11 @@ public abstract class CommandTestAbstract { return vertx; } + @Override + public GenesisConfigOptions getActualGenesisConfigOptions() { + return super.getActualGenesisConfigOptions(); + } + public CommandSpec getSpec() { return spec; } @@ -682,4 +710,33 @@ public abstract class CommandTestAbstract { PORT_CHECK, NO_PORT_CHECK } + + protected static String escapeTomlString(final String s) { + return StringEscapeUtils.escapeJava(s); + } + + /** + * Check logger calls + * + *

Here we check the calls to logger and not the result of the log line as we don't test the + * logger itself but the fact that we call it. + * + * @param dependentOptions the string representing the list of dependent options names + * @param mainOption the main option name + */ + protected void verifyOptionsConstraintLoggerCall( + final String mainOption, final String... dependentOptions) { + verify(mockLogger, atLeast(1)) + .warn( + stringArgumentCaptor.capture(), + stringArgumentCaptor.capture(), + stringArgumentCaptor.capture()); + assertThat(stringArgumentCaptor.getAllValues().get(0)).isEqualTo(DEPENDENCY_WARNING_MSG); + + for (final String option : dependentOptions) { + assertThat(stringArgumentCaptor.getAllValues().get(1)).contains(option); + } + + assertThat(stringArgumentCaptor.getAllValues().get(2)).isEqualTo(mainOption); + } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java index eee55c3975..55b1c95a7f 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java @@ -20,6 +20,7 @@ import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConf import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.Implementation.SEQUENCED; import static org.mockito.Mockito.mock; +import org.hyperledger.besu.cli.config.ProfileName; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.math.BigInteger; @@ -151,21 +152,21 @@ class ConfigurationOverviewBuilderTest { } @Test - void setTrieLogPruningEnabled() { - final String noTrieLogRetentionThresholdSet = builder.build(); - assertThat(noTrieLogRetentionThresholdSet).doesNotContain("Trie log pruning enabled"); + void setBonsaiLimitTrieLogsEnabled() { + final String noTrieLogRetentionLimitSet = builder.build(); + assertThat(noTrieLogRetentionLimitSet).doesNotContain("Limit trie logs enabled"); - builder.setTrieLogPruningEnabled(); - builder.setTrieLogRetentionThreshold(42); - String trieLogRetentionThresholdSet = builder.build(); - assertThat(trieLogRetentionThresholdSet) - .contains("Trie log pruning enabled") + builder.setLimitTrieLogsEnabled(); + builder.setTrieLogRetentionLimit(42); + String trieLogRetentionLimitSet = builder.build(); + assertThat(trieLogRetentionLimitSet) + .contains("Limit trie logs enabled") .contains("retention: 42"); - assertThat(trieLogRetentionThresholdSet).doesNotContain("prune limit"); + assertThat(trieLogRetentionLimitSet).doesNotContain("prune window"); - builder.setTrieLogPruningLimit(1000); - trieLogRetentionThresholdSet = builder.build(); - assertThat(trieLogRetentionThresholdSet).contains("prune limit: 1000"); + builder.setTrieLogsPruningWindowSize(1000); + trieLogRetentionLimitSet = builder.build(); + assertThat(trieLogRetentionLimitSet).contains("prune window: 1000"); } @Test @@ -209,4 +210,11 @@ class ConfigurationOverviewBuilderTest { final String layeredTxPoolSelected = builder.build(); assertThat(layeredTxPoolSelected).contains("Using JOURNALED worldstate update mode"); } + + @Test + void setProfile() { + builder.setProfile(ProfileName.DEV.name()); + final String profileSelected = builder.build(); + assertThat(profileSelected).contains("Profile: DEV"); + } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigFileDefaultProviderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java similarity index 90% rename from besu/src/test/java/org/hyperledger/besu/cli/TomlConfigFileDefaultProviderTest.java rename to besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java index 5185a4133e..0b47654f58 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigFileDefaultProviderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java @@ -19,7 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.when; -import org.hyperledger.besu.cli.util.TomlConfigFileDefaultProvider; +import org.hyperledger.besu.cli.util.TomlConfigurationDefaultProvider; import org.hyperledger.besu.datatypes.Wei; import java.io.BufferedWriter; @@ -42,7 +42,7 @@ import picocli.CommandLine.Model.OptionSpec; import picocli.CommandLine.ParameterException; @ExtendWith(MockitoExtension.class) -public class TomlConfigFileDefaultProviderTest { +public class TomlConfigurationDefaultProviderTest { @Mock CommandLine mockCommandLine; @Mock CommandSpec mockCommandSpec; @@ -67,8 +67,8 @@ public class TomlConfigFileDefaultProviderTest { fileWriter.write("a-longer-option='1234'"); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); // this option must be found in config assertThat( @@ -152,8 +152,8 @@ public class TomlConfigFileDefaultProviderTest { fileWriter.write("a-double-value-option-int=1"); // should be able to parse int as double fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThat( providerUnderTest.defaultValue( @@ -221,16 +221,9 @@ public class TomlConfigFileDefaultProviderTest { @Test public void configFileNotFoundMustThrow() { - final File nonExistingFile = new File("doesnt.exit"); - - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, nonExistingFile); - assertThatThrownBy( - () -> - providerUnderTest.defaultValue( - OptionSpec.builder("an-option").type(String.class).build())) + () -> TomlConfigurationDefaultProvider.fromFile(mockCommandLine, nonExistingFile)) .isInstanceOf(ParameterException.class) .hasMessage("Unable to read TOML configuration, file not found."); } @@ -240,8 +233,8 @@ public class TomlConfigFileDefaultProviderTest { final File tempConfigFile = Files.createTempFile("invalid", "toml").toFile(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThatThrownBy( () -> @@ -260,8 +253,8 @@ public class TomlConfigFileDefaultProviderTest { fileWriter.write("an-invalid-syntax=======...."); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThatThrownBy( () -> @@ -286,8 +279,8 @@ public class TomlConfigFileDefaultProviderTest { fileWriter.write("invalid_option=true"); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThatThrownBy( () -> @@ -321,8 +314,8 @@ public class TomlConfigFileDefaultProviderTest { fileWriter.newLine(); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThat( providerUnderTest.defaultValue( @@ -361,8 +354,8 @@ public class TomlConfigFileDefaultProviderTest { fileWriter.newLine(); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThatThrownBy( () -> diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java index 35dcaa4bae..21d8baf9ee 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java @@ -16,8 +16,6 @@ package org.hyperledger.besu.cli.options; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; -import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.verify; import org.hyperledger.besu.cli.CommandTestAbstract; @@ -69,7 +67,10 @@ public abstract class AbstractCLIOptionsTest> final TestBesuCommand cmd = parseCommand(cliOptions); final T optionsFromCommand = getOptionsFromBesuCommand(cmd); - assertThat(optionsFromCommand).usingRecursiveComparison().isEqualTo(options); + assertThat(optionsFromCommand) + .usingRecursiveComparison() + .ignoringFields(getNonOptionFields()) + .isEqualTo(options); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); @@ -84,10 +85,10 @@ public abstract class AbstractCLIOptionsTest> final T optionsFromCommand = getOptionsFromBesuCommand(cmd); // Check default values supplied by CLI match expected default values - final String[] fieldsToIgnore = getFieldsWithComputedDefaults().toArray(new String[0]); assertThat(optionsFromCommand) .usingRecursiveComparison() - .ignoringFields(fieldsToIgnore) + .ignoringFields(getFieldsWithComputedDefaults()) + .ignoringFields(getNonOptionFields()) .isEqualTo(defaultOptions); } @@ -95,8 +96,12 @@ public abstract class AbstractCLIOptionsTest> protected abstract D createCustomizedDomainObject(); - protected List getFieldsWithComputedDefaults() { - return Collections.emptyList(); + protected String[] getFieldsWithComputedDefaults() { + return new String[0]; + } + + protected String[] getNonOptionFields() { + return new String[0]; } protected List getFieldsToIgnore() { @@ -125,29 +130,4 @@ public abstract class AbstractCLIOptionsTest> assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).contains(errorMsg); } - - /** - * Check logger calls - * - *

Here we check the calls to logger and not the result of the log line as we don't test the - * logger itself but the fact that we call it. - * - * @param dependentOptions the string representing the list of dependent options names - * @param mainOption the main option name - */ - protected void verifyOptionsConstraintLoggerCall( - final String mainOption, final String... dependentOptions) { - verify(mockLogger, atLeast(1)) - .warn( - stringArgumentCaptor.capture(), - stringArgumentCaptor.capture(), - stringArgumentCaptor.capture()); - assertThat(stringArgumentCaptor.getAllValues().get(0)).isEqualTo(DEPENDENCY_WARNING_MSG); - - for (final String option : dependentOptions) { - assertThat(stringArgumentCaptor.getAllValues().get(1)).contains(option); - } - - assertThat(stringArgumentCaptor.getAllValues().get(2)).isEqualTo(mainOption); - } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java index 4b1fdddb53..74b696cdf9 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java @@ -32,7 +32,9 @@ import org.hyperledger.besu.util.number.PositiveNumber; import java.io.IOException; import java.nio.file.Path; +import java.time.Duration; import java.util.Optional; +import java.util.OptionalInt; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; @@ -361,13 +363,16 @@ public class MiningOptionsTest extends AbstractCLIOptionsTest - assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) - .isEqualTo(PositiveNumber.fromInt(200)), + miningParams -> { + assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) + .isEqualTo(PositiveNumber.fromInt(200)); + assertThat(miningParams.getBlockTxsSelectionMaxTime()) + .isEqualTo(Duration.ofSeconds(POA_BLOCK_PERIOD_SECONDS * 2).toMillis()); + }, "--genesis-file", - genesisFileIBFT2.toString(), + genesisFileClique.toString(), "--poa-block-txs-selection-max-time", "200"); } @@ -407,6 +412,16 @@ public class MiningOptionsTest extends AbstractCLIOptionsTest getFieldsWithComputedDefaults() { - return Arrays.asList("maxTrailingPeers", "computationParallelism"); + protected String[] getFieldsWithComputedDefaults() { + return new String[] {"maxTrailingPeers", "computationParallelism"}; } @Override diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 437053afd5..8aed9ee2d7 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -16,7 +16,7 @@ package org.hyperledger.besu.cli.options.stable; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT; import org.hyperledger.besu.cli.options.AbstractCLIOptionsTest; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -32,50 +32,59 @@ public class DataStorageOptionsTest public void bonsaiTrieLogPruningLimitOption() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit()) - .isEqualTo(1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", - "1"); + assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningWindowSize()) + .isEqualTo(600), + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-window-size", + "600"); } @Test - public void bonsaiTrieLogPruningLimitShouldBePositive() { + public void bonsaiTrieLogPruningWindowSizeShouldBePositive() { internalTestFailure( - "--Xbonsai-trie-log-pruning-limit=0 must be greater than 0", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-trie-logs-pruning-window-size=0 must be greater than 0", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-window-size", "0"); } @Test - public void bonsaiTrieLogRetentionThresholdOption() { + public void bonsaiTrieLogPruningWindowSizeShouldBeAboveRetentionLimit() { + internalTestFailure( + "--Xbonsai-trie-logs-pruning-window-size=512 must be greater than --bonsai-historical-block-limit=512", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-window-size", + "512"); + } + + @Test + public void bonsaiTrieLogRetentionLimitOption() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) - .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()) + .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT + 1), + "--Xbonsai-limit-trie-logs-enabled", + "--bonsai-historical-block-limit", "513"); } @Test - public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { + public void bonsaiTrieLogRetentionLimitOption_boundaryTest() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) - .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()) + .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT), + "--Xbonsai-limit-trie-logs-enabled", + "--bonsai-historical-block-limit", "512"); } @Test - public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() { + public void bonsaiTrieLogRetentionLimitShouldBeAboveMinimum() { internalTestFailure( - "--Xbonsai-trie-log-retention-threshold minimum value is 512", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--bonsai-historical-block-limit minimum value is 512", + "--Xbonsai-limit-trie-logs-enabled", + "--bonsai-historical-block-limit", "511"); } @@ -88,12 +97,11 @@ public class DataStorageOptionsTest protected DataStorageConfiguration createCustomizedDomainObject() { return ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(100L) + .bonsaiMaxLayersToLoad(513L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogPruningEnabled(true) - .bonsaiTrieLogRetentionThreshold(1000L) - .bonsaiTrieLogPruningLimit(20) + .bonsaiLimitTrieLogsEnabled(true) + .bonsaiTrieLogPruningWindowSize(514) .build()) .build(); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 0ba575a9ef..22a7c37523 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -15,11 +15,13 @@ package org.hyperledger.besu.cli.subcommands.storage; +import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; @@ -27,20 +29,26 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -53,20 +61,23 @@ class TrieLogHelperTest { private static final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private static BonsaiWorldStateKeyValueStorage inMemoryWorldState; + private TrieLogHelper nonValidatingTrieLogHelper; - @Mock private MutableBlockchain blockchain; - - @TempDir static Path dataDir; + private static class NonValidatingTrieLogHelper extends TrieLogHelper { + @Override + void validatePruneConfiguration(final DataStorageConfiguration config) {} + } - Path test; + @Mock private MutableBlockchain blockchain; static BlockHeader blockHeader1; static BlockHeader blockHeader2; static BlockHeader blockHeader3; static BlockHeader blockHeader4; + static BlockHeader blockHeader5; - @BeforeAll - public static void setup() throws IOException { + @BeforeEach + public void setup() throws IOException { blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); @@ -78,33 +89,35 @@ class TrieLogHelperTest { new BonsaiWorldStateKeyValueStorage( storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + createTrieLog(blockHeader1); + var updater = inMemoryWorldState.updater(); updater .getTrieLogStorageTransaction() - .put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe()); + .put(blockHeader1.getHash().toArrayUnsafe(), createTrieLog(blockHeader1)); updater .getTrieLogStorageTransaction() - .put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe()); + .put(blockHeader2.getHash().toArrayUnsafe(), createTrieLog(blockHeader2)); updater .getTrieLogStorageTransaction() - .put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe()); + .put(blockHeader3.getHash().toArrayUnsafe(), createTrieLog(blockHeader3)); updater .getTrieLogStorageTransaction() - .put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe()); + .put(blockHeader4.getHash().toArrayUnsafe(), createTrieLog(blockHeader4)); updater .getTrieLogStorageTransaction() - .put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); + .put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5)); updater.getTrieLogStorageTransaction().commit(); - } - @BeforeEach - void createDirectory() throws IOException { - Files.createDirectories(dataDir.resolve("database")); + nonValidatingTrieLogHelper = new NonValidatingTrieLogHelper(); } - @AfterEach - void deleteDirectory() throws IOException { - Files.deleteIfExists(dataDir.resolve("database")); + private static byte[] createTrieLog(final BlockHeader blockHeader) { + TrieLogLayer trieLogLayer = new TrieLogLayer(); + trieLogLayer.setBlockHash(blockHeader.getBlockHash()); + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + TrieLogFactoryImpl.writeTo(trieLogLayer, rlpLog); + return rlpLog.encoded().toArrayUnsafe(); } void mockBlockchainBase() { @@ -114,17 +127,17 @@ class TrieLogHelperTest { } @Test - public void prune() { + public void prune(final @TempDir Path dataDir) throws IOException { + Files.createDirectories(dataDir.resolve("database")); DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) + .bonsaiMaxLayersToLoad(3L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(3) - .build() - .withBonsaiTrieLogRetentionThreshold(3)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); mockBlockchainBase(); @@ -133,79 +146,76 @@ class TrieLogHelperTest { when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); // assert trie logs that will be pruned exist before prune call - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); - - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); + assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader1)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader2)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader3)); + + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); // assert pruned trie logs are not in the DB - assertEquals(inMemoryWorldState.getTrieLog(blockHeader1.getHash()), Optional.empty()); - assertEquals(inMemoryWorldState.getTrieLog(blockHeader2.getHash()), Optional.empty()); + assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash())).isEqualTo(Optional.empty()); + assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash())).isEqualTo(Optional.empty()); // assert retained trie logs are in the DB - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader3)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader4)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader5)); } @Test - public void cantPruneIfNoFinalizedIsFound() { + public void cannotPruneIfNoFinalizedIsFound() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) .bonsaiMaxLayersToLoad(2L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); when(blockchain.getFinalized()).thenReturn(Optional.empty()); - assertThrows( - RuntimeException.class, - () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + assertThatThrownBy( + () -> + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(RuntimeException.class) + .hasMessage("No finalized block present, can't safely run trie log prune"); } @Test - public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { + public void cannotPruneIfUserRetainsMoreLayersThanExistingChainLength() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) + .bonsaiMaxLayersToLoad(10L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(10) - .build() - .withBonsaiTrieLogRetentionThreshold(10)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); - assertThrows( - IllegalArgumentException.class, - () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + assertThatThrownBy( + () -> + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Trying to retain more trie logs than chain length (5), skipping pruning"); } @Test - public void cantPruneIfUserRequiredFurtherThanFinalized() { + public void cannotPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -213,54 +223,277 @@ class TrieLogHelperTest { .bonsaiMaxLayersToLoad(2L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); mockBlockchainBase(); - assertThrows( - IllegalArgumentException.class, - () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + assertThatThrownBy( + () -> + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Trying to prune more layers than the finalized block height, skipping pruning"); } @Test - public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { - Files.delete(dataDir.resolve("database")); + public void skipPruningIfTrieLogCountIsLessThanMaxLayersToLoad() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) + .bonsaiMaxLayersToLoad(6L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .build()) + .build(); + + when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); + + assertThatThrownBy( + () -> + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Trie log count (5) is less than retention limit (6), skipping pruning"); + } + + @Test + public void mismatchInPrunedTrieLogCountShouldNotDeleteFiles(final @TempDir Path dataDir) + throws IOException { + Files.createDirectories(dataDir.resolve("database")); + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(3L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .build()) + .build(); + + mockBlockchainBase(); + when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5)); + when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4)); + when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); + + final BonsaiWorldStateKeyValueStorage inMemoryWorldStateSpy = spy(inMemoryWorldState); + // force a different value the second time the trie log count is called + when(inMemoryWorldStateSpy.streamTrieLogKeys(3L + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE)) + .thenCallRealMethod() + .thenReturn(Stream.empty()); + assertThatThrownBy( + () -> + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldStateSpy, blockchain, dataDir)) + .isInstanceOf(RuntimeException.class) + .hasMessage( + "Remaining trie logs (0) did not match --bonsai-historical-block-limit (3). Trie logs backup files have not been deleted, it is safe to rerun the subcommand."); + } + + @Test + public void trieLogRetentionLimitShouldBeAboveMinimum() { + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(511L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .build()) + .build(); + + TrieLogHelper helper = new TrieLogHelper(); + assertThatThrownBy( + () -> + helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(RuntimeException.class) + .hasMessage("--bonsai-historical-block-limit minimum value is 512"); + } + + @Test + public void trieLogPruningWindowSizeShouldBePositive() { + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(512L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .bonsaiTrieLogPruningWindowSize(0) + .build()) + .build(); + + TrieLogHelper helper = new TrieLogHelper(); + assertThatThrownBy( + () -> + helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(RuntimeException.class) + .hasMessage("--Xbonsai-trie-logs-pruning-window-size=0 must be greater than 0"); + } + + @Test + public void trieLogPruningWindowSizeShouldBeAboveRetentionLimit() { + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(512L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiLimitTrieLogsEnabled(true) + .bonsaiTrieLogPruningWindowSize(512) + .build()) .build(); - assertThrows( - RuntimeException.class, - () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + TrieLogHelper helper = new TrieLogHelper(); + assertThatThrownBy( + () -> + helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(RuntimeException.class) + .hasMessage( + "--Xbonsai-trie-logs-pruning-window-size=512 must be greater than --bonsai-historical-block-limit=512"); + } + + @Test + public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDir) { + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(3L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .build()) + .build(); + + mockBlockchainBase(); + when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5)); + when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4)); + when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); + + assertThatThrownBy( + () -> + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, + inMemoryWorldState, + blockchain, + dataDir.resolve("unknownPath"))) + .isInstanceOf(RuntimeException.class) + .hasCauseExactlyInstanceOf(FileNotFoundException.class); // assert all trie logs are still in the DB - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader1)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader2)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader3)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader4)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader5)); + } + + @Test + public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { + nonValidatingTrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + nonValidatingTrieLogHelper + .readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .findFirst() + .get(); + + assertThat(trieLog.getKey()).isEqualTo(blockHeader1.getHash().toArrayUnsafe()); + assertThat(trieLog.getValue()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + } + + @Test + public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + nonValidatingTrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLogs = + nonValidatingTrieLogHelper + .readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue)); + + assertThat(trieLogs.get(blockHeader1.getHash())) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertThat(trieLogs.get(blockHeader2.getHash())) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertThat(trieLogs.get(blockHeader3.getHash())) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); + } + + @Test + public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + + nonValidatingTrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + nonValidatingTrieLogHelper.readTrieLogsAsRlpFromFile( + dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + + assertThat(inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + } + + @Test + public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + + nonValidatingTrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + nonValidatingTrieLogHelper.readTrieLogsAsRlpFromFile( + dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + + assertThat(inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertThat(inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertThat(inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandlerTest.java b/besu/src/test/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandlerTest.java index bd33703d9c..fbc672dc0a 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandlerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandlerTest.java @@ -145,7 +145,7 @@ public class ConfigOptionSearchAndRunHandlerTest { public void shouldRetrieveConfigFromEnvironmentWhenConfigFileSpecified() throws Exception { final IDefaultValueProvider defaultValueProvider = configParsingHandler.createDefaultValueProvider( - mockCommandLine, Optional.of(new File("foo"))); + mockCommandLine, Optional.of(new File("foo")), Optional.empty()); final String value = defaultValueProvider.defaultValue(OptionSpec.builder("--logging").build()); assertThat(value).isEqualTo("ERROR"); } @@ -153,7 +153,8 @@ public class ConfigOptionSearchAndRunHandlerTest { @Test public void shouldRetrieveConfigFromEnvironmentWhenConfigFileNotSpecified() throws Exception { final IDefaultValueProvider defaultValueProvider = - configParsingHandler.createDefaultValueProvider(mockCommandLine, Optional.empty()); + configParsingHandler.createDefaultValueProvider( + mockCommandLine, Optional.empty(), Optional.empty()); final String value = defaultValueProvider.defaultValue(OptionSpec.builder("--logging").build()); assertThat(value).isEqualTo("ERROR"); } diff --git a/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java index f7ae97cdf5..3df1e8e24d 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java @@ -115,6 +115,7 @@ class TraceServiceImplTest { .getTransactions() .forEach( tx -> { + verify(opTracer).tracePrepareTransaction(any(), eq(tx)); verify(opTracer).traceStartTransaction(any(), eq(tx)); verify(opTracer) .traceEndTransaction( @@ -162,6 +163,7 @@ class TraceServiceImplTest { .getTransactions() .forEach( tx -> { + verify(opTracer).tracePrepareTransaction(any(), eq(tx)); verify(opTracer).traceStartTransaction(any(), eq(tx)); verify(opTracer) .traceEndTransaction( diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index b8dae6c188..3c6792b7b9 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -16,7 +16,7 @@ node-private-key-file="./path/to/privateKey" pid-path="~/.pid" reorg-logging-threshold=0 static-nodes-file="~/besudata/static-nodes.json" - +profile="NONE" # Security Module plugin to use security-module="localfile" @@ -90,6 +90,7 @@ rpc-max-logs-range=100 json-pretty-print-enabled=false cache-last-blocks=512 rpc-gas-cap = 50000000 +rpc-max-trace-filter-range=100 # PRIVACY TLS privacy-tls-enabled=false @@ -232,4 +233,4 @@ Xp2p-tls-crl-file="none.file" Xp2p-tls-clienthello-sni=false #contracts -Xevm-jumpdest-cache-weight-kb=32000 \ No newline at end of file +Xevm-jumpdest-cache-weight-kb=32000 diff --git a/besu/src/test/resources/partial_config.toml b/besu/src/test/resources/partial_config.toml index f2d8eaa061..1763bb6bdc 100644 --- a/besu/src/test/resources/partial_config.toml +++ b/besu/src/test/resources/partial_config.toml @@ -1,4 +1,7 @@ # this is a valid partial TOML config file #mining -miner-coinbase="0x0000000000000000000000000000000000000002" \ No newline at end of file +miner-coinbase="0x0000000000000000000000000000000000000002" + +#network +network="mainnet" \ No newline at end of file diff --git a/config/src/main/resources/profiles/dev.toml b/config/src/main/resources/profiles/dev.toml new file mode 100644 index 0000000000..3c80770b4e --- /dev/null +++ b/config/src/main/resources/profiles/dev.toml @@ -0,0 +1 @@ +network="DEV" \ No newline at end of file diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/ApiConfiguration.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/ApiConfiguration.java index e9f30fca33..42a7f8f62e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/ApiConfiguration.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/ApiConfiguration.java @@ -77,4 +77,9 @@ public abstract class ApiConfiguration { public Long getUpperBoundGasAndPriorityFeeCoefficient() { return DEFAULT_UPPER_BOUND_GAS_AND_PRIORITY_FEE_COEFFICIENT; } + + @Value.Default + public Long getMaxTraceFilterRange() { + return 1000L; + } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java index 8c94f78506..ff65feb846 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java @@ -24,8 +24,10 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.FilterParam import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.RewardTraceGenerator; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; @@ -66,12 +68,15 @@ import org.slf4j.LoggerFactory; public class TraceFilter extends TraceBlock { private static final Logger LOG = LoggerFactory.getLogger(TraceFilter.class); + private final Long maxRange; public TraceFilter( final Supplier blockTracerSupplier, final ProtocolSchedule protocolSchedule, - final BlockchainQueries blockchainQueries) { + final BlockchainQueries blockchainQueries, + final Long maxRange) { super(protocolSchedule, blockchainQueries); + this.maxRange = maxRange; } @Override @@ -88,6 +93,17 @@ public class TraceFilter extends TraceBlock { final long toBlock = resolveBlockNumber(filterParameter.getToBlock()); LOG.trace("Received RPC rpcName={} fromBlock={} toBlock={}", getName(), fromBlock, toBlock); + if (maxRange > 0 && toBlock - fromBlock > maxRange) { + LOG.atDebug() + .setMessage("trace_filter request {} failed:") + .addArgument(requestContext.getRequest()) + .setCause( + new IllegalArgumentException(RpcErrorType.EXCEEDS_RPC_MAX_BLOCK_RANGE.getMessage())) + .log(); + return new JsonRpcErrorResponse( + requestContext.getRequest().getId(), RpcErrorType.EXCEEDS_RPC_MAX_BLOCK_RANGE); + } + final ObjectMapper mapper = new ObjectMapper(); final ArrayNodeWrapper resultArrayNode = new ArrayNodeWrapper( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java index 96faae9fd2..2333409b08 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java @@ -60,7 +60,11 @@ public class TraceJsonRpcMethods extends ApiGroupJsonRpcMethods { new BlockReplay(protocolSchedule, blockchainQueries.getBlockchain()); return mapOf( new TraceReplayBlockTransactions(protocolSchedule, blockchainQueries), - new TraceFilter(() -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries), + new TraceFilter( + () -> new BlockTracer(blockReplay), + protocolSchedule, + blockchainQueries, + apiConfiguration.getMaxTraceFilterRange()), new TraceGet(() -> new BlockTracer(blockReplay), blockchainQueries, protocolSchedule), new TraceTransaction( () -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries), diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java new file mode 100644 index 0000000000..99db885114 --- /dev/null +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java @@ -0,0 +1,79 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.FilterParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; + +import java.util.function.Supplier; + +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class TraceFilterTest { + + private TraceFilter method; + + @Mock Supplier blockTracerSupplier; + @Mock ProtocolSchedule protocolSchedule; + @Mock BlockchainQueries blockchainQueries; + + @ParameterizedTest + @CsvSource({ + "0, 1001, 1000", "0, 5000, 1000", "1, 1002, 1000", "1, 6002, 1000", "1000, 3000, 1000", + "0, 501, 500", "0, 5000, 500", "1, 502, 500", "1, 6002, 500", "1000, 3000, 500" + }) + public void shouldFailIfParamsExceedMaxRange( + final long fromBlock, final long toBlock, final long maxFilterRange) { + final FilterParameter filterParameter = + new FilterParameter( + new BlockParameter(fromBlock), + new BlockParameter(toBlock), + null, + null, + null, + null, + null, + null, + null); + + JsonRpcRequestContext request = + new JsonRpcRequestContext( + new JsonRpcRequest("2.0", "trace_filter", new Object[] {filterParameter})); + + method = + new TraceFilter(blockTracerSupplier, protocolSchedule, blockchainQueries, maxFilterRange); + + final JsonRpcResponse response = method.response(request); + assertThat(response).isInstanceOf(JsonRpcErrorResponse.class); + + final JsonRpcErrorResponse errorResponse = (JsonRpcErrorResponse) response; + assertThat(errorResponse.getErrorType()).isEqualTo(RpcErrorType.EXCEEDS_RPC_MAX_BLOCK_RANGE); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java index 931388b387..4ec3eea860 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java @@ -293,6 +293,8 @@ public class MainnetTransactionProcessor { return TransactionProcessingResult.invalid(validationResult); } + operationTracer.tracePrepareTransaction(worldState, transaction); + final long previousNonce = sender.incrementNonce(); LOG.trace( "Incremented sender {} nonce ({} -> {})", diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java index 9cce0230cd..282c314157 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java @@ -16,18 +16,21 @@ package org.hyperledger.besu.ethereum.storage.keyvalue; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import java.nio.charset.StandardCharsets; + import org.bouncycastle.util.Arrays; public enum KeyValueSegmentIdentifier implements SegmentIdentifier { - BLOCKCHAIN(new byte[] {1}, true), - WORLD_STATE(new byte[] {2}, new int[] {0, 1}), + DEFAULT("default".getBytes(StandardCharsets.UTF_8)), + BLOCKCHAIN(new byte[] {1}, true, true), + WORLD_STATE(new byte[] {2}, new int[] {0, 1}, false, true), PRIVATE_TRANSACTIONS(new byte[] {3}), PRIVATE_STATE(new byte[] {4}), PRUNING_STATE(new byte[] {5}, new int[] {0, 1}), - ACCOUNT_INFO_STATE(new byte[] {6}, new int[] {2}), + ACCOUNT_INFO_STATE(new byte[] {6}, new int[] {2}, false, true), CODE_STORAGE(new byte[] {7}, new int[] {2}), - ACCOUNT_STORAGE_STORAGE(new byte[] {8}, new int[] {2}), - TRIE_BRANCH_STORAGE(new byte[] {9}, new int[] {2}), + ACCOUNT_STORAGE_STORAGE(new byte[] {8}, new int[] {2}, false, true), + TRIE_BRANCH_STORAGE(new byte[] {9}, new int[] {2}, false, true), TRIE_LOG_STORAGE(new byte[] {10}, new int[] {2}), VARIABLES(new byte[] {11}), // formerly GOQUORUM_PRIVATE_WORLD_STATE @@ -45,24 +48,30 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier { private final byte[] id; private final int[] versionList; private final boolean containsStaticData; + private final boolean eligibleToHighSpecFlag; KeyValueSegmentIdentifier(final byte[] id) { this(id, new int[] {0, 1, 2}); } - KeyValueSegmentIdentifier(final byte[] id, final boolean containsStaticData) { - this(id, new int[] {0, 1, 2}, containsStaticData); + KeyValueSegmentIdentifier( + final byte[] id, final boolean containsStaticData, final boolean eligibleToHighSpecFlag) { + this(id, new int[] {0, 1, 2}, containsStaticData, eligibleToHighSpecFlag); } KeyValueSegmentIdentifier(final byte[] id, final int[] versionList) { - this(id, versionList, false); + this(id, versionList, false, false); } KeyValueSegmentIdentifier( - final byte[] id, final int[] versionList, final boolean containsStaticData) { + final byte[] id, + final int[] versionList, + final boolean containsStaticData, + final boolean eligibleToHighSpecFlag) { this.id = id; this.versionList = versionList; this.containsStaticData = containsStaticData; + this.eligibleToHighSpecFlag = eligibleToHighSpecFlag; } @Override @@ -80,6 +89,11 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier { return containsStaticData; } + @Override + public boolean isEligibleToHighSpecFlag() { + return eligibleToHighSpecFlag; + } + @Override public boolean includeInDatabaseVersion(final int version) { return Arrays.contains(versionList, version); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java index c4f8ab0efd..c99da3fe24 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java @@ -42,6 +42,17 @@ public class BonsaiValue implements TrieLog.LogTuple { this.clearedAtLeastOnce = lastStepCleared; } + public BonsaiValue( + final T prior, + final T updated, + final boolean lastStepCleared, + final boolean clearedAtLeastOnce) { + this.prior = prior; + this.updated = updated; + this.lastStepCleared = lastStepCleared; + this.clearedAtLeastOnce = clearedAtLeastOnce; + } + @Override public T getPrior() { return prior; @@ -117,4 +128,8 @@ public class BonsaiValue implements TrieLog.LogTuple { .append(lastStepCleared) .toHashCode(); } + + public BonsaiValue copy() { + return new BonsaiValue(prior, updated, lastStepCleared, clearedAtLeastOnce); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java index 2a0b2cb392..0230a2258b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java @@ -31,7 +31,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; @@ -74,8 +73,7 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { final Optional maxLayersToLoad, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final BesuContext pluginContext, - final EvmConfiguration evmConfiguration, - final TrieLogPruner trieLogPruner) { + final EvmConfiguration evmConfiguration) { this.worldStateKeyValueStorage = worldStateKeyValueStorage; this.cachedWorldStorageManager = new CachedWorldStorageManager(this, worldStateKeyValueStorage); @@ -86,8 +84,7 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { blockchain, worldStateKeyValueStorage, maxLayersToLoad.orElse(RETAINED_LAYERS), - pluginContext, - trieLogPruner); + pluginContext); this.blockchain = blockchain; this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java index 7cb024a259..4c9a520bd2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java @@ -25,7 +25,7 @@ import java.util.Optional; public class NoOpTrieLogManager extends TrieLogManager { public NoOpTrieLogManager() { - super(null, null, 0, null, TrieLogPruner.noOpTrieLogPruner()); + super(null, null, 0, null); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java index b546f208be..19ac6285f5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java @@ -47,19 +47,16 @@ public class TrieLogManager { protected final Subscribers trieLogObservers = Subscribers.create(); protected final TrieLogFactory trieLogFactory; - private final TrieLogPruner trieLogPruner; public TrieLogManager( final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final long maxLayersToLoad, - final BesuContext pluginContext, - final TrieLogPruner trieLogPruner) { + final BesuContext pluginContext) { this.blockchain = blockchain; this.rootWorldStateStorage = worldStateKeyValueStorage; this.maxLayersToLoad = maxLayersToLoad; this.trieLogFactory = setupTrieLogFactory(pluginContext); - this.trieLogPruner = trieLogPruner; } public synchronized void saveTrieLog( @@ -85,8 +82,6 @@ public class TrieLogManager { } finally { if (success) { stateUpdater.commit(); - trieLogPruner.addToPruneQueue(forBlockHeader.getNumber(), forBlockHeader.getBlockHash()); - trieLogPruner.pruneFromQueue(); } else { stateUpdater.rollback(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java index b1bf75818e..b72796f222 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java @@ -20,10 +20,12 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; import java.util.Comparator; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; import java.util.stream.Stream; import com.google.common.collect.ArrayListMultimap; @@ -33,7 +35,7 @@ import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TrieLogPruner { +public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { private static final Logger LOG = LoggerFactory.getLogger(TrieLogPruner.class); @@ -41,6 +43,7 @@ public class TrieLogPruner { private final int loadingLimit; private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; private final Blockchain blockchain; + private final Consumer executeAsync; private final long numBlocksToRetain; private final boolean requireFinalizedBlock; @@ -50,11 +53,13 @@ public class TrieLogPruner { public TrieLogPruner( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain, + final Consumer executeAsync, final long numBlocksToRetain, final int pruningLimit, final boolean requireFinalizedBlock) { this.rootWorldStateStorage = rootWorldStateStorage; this.blockchain = blockchain; + this.executeAsync = executeAsync; this.numBlocksToRetain = numBlocksToRetain; this.pruningLimit = pruningLimit; this.loadingLimit = pruningLimit; // same as pruningLimit for now @@ -166,34 +171,18 @@ public class TrieLogPruner { return wasPruned.size(); } - public static TrieLogPruner noOpTrieLogPruner() { - return new NoOpTrieLogPruner(null, null, 0, 0); - } - - public static class NoOpTrieLogPruner extends TrieLogPruner { - private NoOpTrieLogPruner( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final Blockchain blockchain, - final long numBlocksToRetain, - final int pruningLimit) { - super(rootWorldStateStorage, blockchain, numBlocksToRetain, pruningLimit, true); - } - - @Override - public int initialize() { - // no-op - return -1; - } - - @Override - void addToPruneQueue(final long blockNumber, final Hash blockHash) { - // no-op - } - - @Override - int pruneFromQueue() { - // no-op - return -1; + @Override + public void onTrieLogAdded(final TrieLogEvent event) { + if (TrieLogEvent.Type.ADDED.equals(event.getType())) { + final Hash blockHash = event.layer().getBlockHash(); + final Optional blockNumber = event.layer().getBlockNumber(); + blockNumber.ifPresent( + blockNum -> + executeAsync.accept( + () -> { + addToPruneQueue(blockNum, blockHash); + pruneFromQueue(); + })); } } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java index 6ae3c282bb..2b4612022a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java @@ -121,12 +121,12 @@ public class BonsaiWorldState } /** - * Having a protected method to override the accumulator solves the chicken-egg problem of needing - * a worldstate reference (this) when construction the Accumulator. + * Override the accumulator solves the chicken-egg problem of needing a worldstate reference + * (this) when construction the Accumulator. * * @param accumulator accumulator to use. */ - protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { + public void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { this.accumulator = accumulator; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 9f0dd07d3b..d73c272829 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -60,13 +60,13 @@ public class BonsaiWorldStateUpdateAccumulator implements BonsaiWorldView, TrieLogAccumulator { private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class); - private final Consumer> accountPreloader; - private final Consumer storagePreloader; + protected final Consumer> accountPreloader; + protected final Consumer storagePreloader; private final AccountConsumingMap> accountsToUpdate; private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set

storageToClear = Collections.synchronizedSet(new HashSet<>()); - private final EvmConfiguration evmConfiguration; + protected final EvmConfiguration evmConfiguration; // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the @@ -74,7 +74,7 @@ public class BonsaiWorldStateUpdateAccumulator private final Map>> storageToUpdate = new ConcurrentHashMap<>(); - private boolean isAccumulatorStateChanged; + protected boolean isAccumulatorStateChanged; public BonsaiWorldStateUpdateAccumulator( final BonsaiWorldView world, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index a36b150337..e8fc199b67 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -43,27 +43,21 @@ public interface DataStorageConfiguration { @Value.Immutable interface Unstable { - boolean DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED = false; - long DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = 512L; - long MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; - int DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT = 30_000; + boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false; + long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; + int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000; DataStorageConfiguration.Unstable DEFAULT = ImmutableDataStorageConfiguration.Unstable.builder().build(); @Value.Default - default boolean getBonsaiTrieLogPruningEnabled() { - return DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; + default boolean getBonsaiLimitTrieLogsEnabled() { + return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; } @Value.Default - default long getBonsaiTrieLogRetentionThreshold() { - return DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; - } - - @Value.Default - default int getBonsaiTrieLogPruningLimit() { - return DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; + default int getBonsaiTrieLogPruningWindowSize() { + return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; } } } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 1e2e8db199..a42a1fb2bd 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -30,7 +30,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValue import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -115,8 +114,7 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider { Optional.empty(), cachedMerkleTrieLoader, null, - evmConfiguration, - TrieLogPruner.noOpTrieLogPruner()); + evmConfiguration); } public static MutableWorldState createInMemoryWorldState() { diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index 3e1e937fc5..f13e1d85fc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -67,7 +67,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; @@ -163,8 +162,7 @@ public abstract class AbstractIsolationTests { Optional.of(16L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), null, - EvmConfiguration.DEFAULT, - TrieLogPruner.noOpTrieLogPruner()); + EvmConfiguration.DEFAULT); var ws = archive.getMutable(); genesisState.writeStateTo(ws); protocolContext = new ProtocolContext(blockchain, archive, null, Optional.empty()); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java index 4e9ef325ed..c31a776c77 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java @@ -42,7 +42,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValu import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -127,8 +126,7 @@ class BonsaiWorldStateProviderTest { Optional.of(512L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), null, - EvmConfiguration.DEFAULT, - TrieLogPruner.noOpTrieLogPruner()); + EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); when(blockchain.getChainHeadHeader()).thenReturn(chainHead); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java index a0065e2cb9..e5c716e413 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum.trie.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner.noOpTrieLogPruner; import static org.mockito.Mockito.spy; import org.hyperledger.besu.datatypes.Hash; @@ -57,9 +56,7 @@ class TrieLogManagerTests { @BeforeEach public void setup() { - trieLogManager = - new TrieLogManager( - blockchain, bonsaiWorldStateKeyValueStorage, 512, null, noOpTrieLogPruner()); + trieLogManager = new TrieLogManager(blockchain, bonsaiWorldStateKeyValueStorage, 512, null); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java index af5acbf18c..ac84d35be4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import java.util.Optional; +import java.util.function.Consumer; import java.util.stream.Stream; import org.apache.logging.log4j.Level; @@ -44,6 +45,7 @@ public class TrieLogPrunerTest { private BonsaiWorldStateKeyValueStorage worldState; private Blockchain blockchain; + private final Consumer executeAsync = Runnable::run; @SuppressWarnings("BannedMethod") @BeforeEach @@ -67,7 +69,8 @@ public class TrieLogPrunerTest { when(blockchain.getBlockHeader(header2.getBlockHash())).thenReturn(Optional.empty()); // When - TrieLogPruner trieLogPruner = new TrieLogPruner(worldState, blockchain, 3, loadingLimit, false); + TrieLogPruner trieLogPruner = + new TrieLogPruner(worldState, blockchain, executeAsync, 3, loadingLimit, false); trieLogPruner.initialize(); // Then @@ -86,7 +89,8 @@ public class TrieLogPrunerTest { when(worldState.pruneTrieLog(any(Hash.class))).thenReturn(true); // requireFinalizedBlock = false means this is not a PoS chain TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, blocksToRetain, pruningWindowSize, false); + new TrieLogPruner( + worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, false); trieLogPruner.addToPruneQueue(0, key(0)); // older block outside prune window trieLogPruner.addToPruneQueue(1, key(1)); // block inside the prune window @@ -194,7 +198,8 @@ public class TrieLogPrunerTest { final int pruningWindowSize = (int) chainHeight; when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight); TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, blocksToRetain, pruningWindowSize, true); + new TrieLogPruner( + worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true); trieLogPruner.addToPruneQueue(1, key(1)); trieLogPruner.addToPruneQueue(2, key(2)); @@ -228,6 +233,46 @@ public class TrieLogPrunerTest { assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(1); } + @Test + public void onTrieLogAdded_should_prune() { + // Given + final TriggerableConsumer triggerableConsumer = new TriggerableConsumer(); + TrieLogPruner trieLogPruner = + new TrieLogPruner(worldState, blockchain, triggerableConsumer, 0, 1, false); + assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0); + + final TrieLogLayer layer = new TrieLogLayer(); + layer.setBlockNumber(1L); + layer.setBlockHash(key(1)); + when(blockchain.getChainHeadBlockNumber()).thenReturn(1L); + + // When + trieLogPruner.onTrieLogAdded(new TrieLogAddedEvent(layer)); + verify(worldState, never()).pruneTrieLog(key(1)); + triggerableConsumer.run(); + + // Then + verify(worldState, times(1)).pruneTrieLog(key(1)); + } + + @Test + public void onTrieLogAdded_should_not_prune_when_no_blockNumber() { + // Given + TrieLogPruner trieLogPruner = + new TrieLogPruner(worldState, blockchain, executeAsync, 0, 1, false); + assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0); + + final TrieLogLayer layer = new TrieLogLayer(); + layer.setBlockHash(key(1)); + when(blockchain.getChainHeadBlockNumber()).thenReturn(1L); + + // When + trieLogPruner.onTrieLogAdded(new TrieLogAddedEvent(layer)); + + // Then + verify(worldState, never()).pruneTrieLog(key(1)); + } + private TrieLogPruner setupPrunerAndFinalizedBlock( final long configuredRetainHeight, final long finalizedBlockHeight) { final long chainHeight = 5; @@ -241,7 +286,8 @@ public class TrieLogPrunerTest { .thenReturn(Optional.of(finalizedHeader)); when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight); TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, blocksToRetain, pruningWindowSize, true); + new TrieLogPruner( + worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true); trieLogPruner.addToPruneQueue(1, key(1)); trieLogPruner.addToPruneQueue(2, key(2)); @@ -255,4 +301,18 @@ public class TrieLogPrunerTest { private Hash key(final int k) { return Hash.hash(Bytes.of(k)); } + + private static class TriggerableConsumer implements Consumer { + + private Runnable runnable; + + @Override + public void accept(final Runnable runnable) { + this.runnable = runnable; + } + + public void run() { + runnable.run(); + } + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index e28b0a24ad..c27b473d31 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -139,6 +139,7 @@ public class EthPeers { "peer_limit", "The maximum number of peers this node allows to connect", () -> peerUpperBound); + connectedPeersCounter = metricsSystem.createCounter( BesuMetricCategory.PEERS, "connected_total", "Total number of peers connected"); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java index 6925df9d49..774d35cf18 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java @@ -110,7 +110,7 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { this.blockBroadcaster = new BlockBroadcaster(ethContext); - supportedCapabilities = + this.supportedCapabilities = calculateCapabilities(synchronizerConfiguration, ethereumWireProtocolConfiguration); // Run validators @@ -252,11 +252,14 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { @Override public void stop() { if (stopped.compareAndSet(false, true)) { - LOG.info("Stopping {} Subprotocol.", getSupportedProtocol()); + LOG.atInfo().setMessage("Stopping {} Subprotocol.").addArgument(getSupportedProtocol()).log(); scheduler.stop(); shutdown.countDown(); } else { - LOG.error("Attempted to stop already stopped {} Subprotocol.", getSupportedProtocol()); + LOG.atInfo() + .setMessage("Attempted to stop already stopped {} Subprotocol.") + .addArgument(this::getSupportedProtocol) + .log(); } } @@ -264,7 +267,10 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { public void awaitStop() throws InterruptedException { shutdown.await(); scheduler.awaitStop(); - LOG.info("{} Subprotocol stopped.", getSupportedProtocol()); + LOG.atInfo() + .setMessage("{} Subprotocol stopped.") + .addArgument(this::getSupportedProtocol) + .log(); } @Override @@ -277,8 +283,10 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { EthProtocolLogger.logProcessMessage(cap, code); final EthPeer ethPeer = ethPeers.peer(message.getConnection()); if (ethPeer == null) { - LOG.debug( - "Ignoring message received from unknown peer connection: {}", message.getConnection()); + LOG.atDebug() + .setMessage("Ignoring message received from unknown peer connection: {}") + .addArgument(message::getConnection) + .log(); return; } @@ -288,19 +296,24 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { return; } else if (!ethPeer.statusHasBeenReceived()) { // Peers are required to send status messages before any other message type - LOG.debug( - "{} requires a Status ({}) message to be sent first. Instead, received message {} (BREACH_OF_PROTOCOL). Disconnecting from {}.", - this.getClass().getSimpleName(), - EthPV62.STATUS, - code, - ethPeer); + LOG.atDebug() + .setMessage( + "{} requires a Status ({}) message to be sent first. Instead, received message {} (BREACH_OF_PROTOCOL). Disconnecting from {}.") + .addArgument(() -> this.getClass().getSimpleName()) + .addArgument(EthPV62.STATUS) + .addArgument(code) + .addArgument(ethPeer::toString) + .log(); ethPeer.disconnect(DisconnectReason.BREACH_OF_PROTOCOL); return; } if (this.mergePeerFilter.isPresent()) { if (this.mergePeerFilter.get().disconnectIfGossipingBlocks(message, ethPeer)) { - LOG.debug("Post-merge disconnect: peer still gossiping blocks {}", ethPeer); + LOG.atDebug() + .setMessage("Post-merge disconnect: peer still gossiping blocks {}") + .addArgument(ethPeer::toString) + .log(); handleDisconnect(ethPeer.getConnection(), DisconnectReason.SUBPROTOCOL_TRIGGERED, false); return; } @@ -333,11 +346,12 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { maybeResponseData = ethMessages.dispatch(ethMessage); } } catch (final RLPException e) { - LOG.debug( - "Received malformed message {} (BREACH_OF_PROTOCOL), disconnecting: {}", - messageData.getData(), - ethPeer, - e); + LOG.atDebug() + .setMessage("Received malformed message {} (BREACH_OF_PROTOCOL), disconnecting: {}, {}") + .addArgument(messageData::getData) + .addArgument(ethPeer::toString) + .addArgument(e::toString) + .log(); ethPeer.disconnect(DisconnectMessage.DisconnectReason.BREACH_OF_PROTOCOL); } @@ -368,23 +382,31 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { genesisHash, latestForkId); try { - LOG.trace("Sending status message to {} for connection {}.", peer.getId(), connection); + LOG.atTrace() + .setMessage("Sending status message to {} for connection {}.") + .addArgument(peer::getId) + .addArgument(connection::toString) + .log(); peer.send(status, getSupportedProtocol(), connection); peer.registerStatusSent(connection); } catch (final PeerNotConnected peerNotConnected) { // Nothing to do. } - LOG.trace("{}", ethPeers); + LOG.atTrace().setMessage("{}").addArgument(ethPeers::toString).log(); } @Override public boolean shouldConnect(final Peer peer, final boolean incoming) { - if (peer.getForkId().map(forkId -> forkIdManager.peerCheck(forkId)).orElse(true)) { - LOG.trace("ForkId OK or not available"); + if (peer.getForkId().map(forkIdManager::peerCheck).orElse(true)) { + LOG.atDebug() + .setMessage("ForkId OK or not available for peer {}") + .addArgument(peer::getId) + .log(); if (ethPeers.shouldConnect(peer, incoming)) { return true; } } + LOG.atDebug().setMessage("ForkId check failed for peer {}").addArgument(peer::getId).log(); return false; } @@ -397,11 +419,11 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { LOG.atDebug() .setMessage("Disconnect - {} - {} - {}... - {} peers left") .addArgument(initiatedByPeer ? "Inbound" : "Outbound") - .addArgument(reason) - .addArgument(connection.getPeer().getId().slice(0, 8)) - .addArgument(ethPeers.peerCount()) + .addArgument(reason::toString) + .addArgument(() -> connection.getPeer().getId().slice(0, 8)) + .addArgument(ethPeers::peerCount) .log(); - LOG.trace("{}", ethPeers); + LOG.atTrace().setMessage("{}").addArgument(ethPeers::toString).log(); } } @@ -412,43 +434,41 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { try { if (!status.networkId().equals(networkId)) { LOG.atDebug() - .setMessage("Mismatched network id: {}, EthPeer {}...") - .addArgument(status.networkId()) - .addArgument(peer.getShortNodeId()) - .log(); - LOG.atTrace() - .setMessage("Mismatched network id: {}, EthPeer {}") - .addArgument(status.networkId()) - .addArgument(peer) + .setMessage("Mismatched network id: {}, peer {}") + .addArgument(status::networkId) + .addArgument(() -> getPeerOrPeerId(peer)) .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (!forkIdManager.peerCheck(forkId) && status.protocolVersion() > 63) { - LOG.debug( - "{} has matching network id ({}), but non-matching fork id: {}", - peer, - networkId, - forkId); + LOG.atDebug() + .setMessage("{} has matching network id ({}), but non-matching fork id: {}") + .addArgument(() -> getPeerOrPeerId(peer)) + .addArgument(networkId::toString) + .addArgument(forkId) + .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (forkIdManager.peerCheck(status.genesisHash())) { - LOG.debug( - "{} has matching network id ({}), but non-matching genesis hash: {}", - peer, - networkId, - status.genesisHash()); + LOG.atDebug() + .setMessage("{} has matching network id ({}), but non-matching genesis hash: {}") + .addArgument(() -> getPeerOrPeerId(peer)) + .addArgument(networkId::toString) + .addArgument(status::genesisHash) + .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (mergePeerFilter.isPresent() && mergePeerFilter.get().disconnectIfPoW(status, peer)) { LOG.atDebug() .setMessage("Post-merge disconnect: peer still PoW {}") - .addArgument(peer.getShortNodeId()) + .addArgument(() -> getPeerOrPeerId(peer)) .log(); handleDisconnect(peer.getConnection(), DisconnectReason.SUBPROTOCOL_TRIGGERED, false); } else { - LOG.debug( - "Received status message from {}: {} with connection {}", - peer, - status, - message.getConnection()); + LOG.atDebug() + .setMessage("Received status message from {}: {} with connection {}") + .addArgument(peer::toString) + .addArgument(status::toString) + .addArgument(message::getConnection) + .log(); peer.registerStatusReceived( status.bestHash(), status.totalDifficulty(), @@ -467,6 +487,10 @@ public class EthProtocolManager implements ProtocolManager, MinedBlockObserver { } } + private Object getPeerOrPeerId(final EthPeer peer) { + return LOG.isTraceEnabled() ? peer : peer.getShortNodeId(); + } + @Override public void blockMined(final Block block) { // This assumes the block has already been included in the chain diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java index 61a01bf54f..dd0587b23c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java @@ -142,8 +142,8 @@ public class EthScheduler { txWorkerExecutor.execute(command); } - public CompletableFuture scheduleServiceTask(final Supplier task) { - return CompletableFuture.supplyAsync(task, servicesExecutor); + public void executeServiceTask(final Runnable command) { + servicesExecutor.execute(command); } public CompletableFuture scheduleServiceTask(final EthTask task) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncTargetManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/AbstractSyncTargetManager.java similarity index 96% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncTargetManager.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/AbstractSyncTargetManager.java index c231d13af3..9c6b9327e4 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncTargetManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/AbstractSyncTargetManager.java @@ -32,9 +32,9 @@ import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class SyncTargetManager { +public abstract class AbstractSyncTargetManager { - private static final Logger LOG = LoggerFactory.getLogger(SyncTargetManager.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractSyncTargetManager.class); private final SynchronizerConfiguration config; private final ProtocolSchedule protocolSchedule; @@ -42,7 +42,7 @@ public abstract class SyncTargetManager { private final EthContext ethContext; private final MetricsSystem metricsSystem; - protected SyncTargetManager( + protected AbstractSyncTargetManager( final SynchronizerConfiguration config, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java index d1c3d00883..5ae78ca280 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java @@ -43,7 +43,7 @@ public class PipelineChainDownloader implements ChainDownloader { private static final Logger LOG = LoggerFactory.getLogger(PipelineChainDownloader.class); static final Duration PAUSE_AFTER_ERROR_DURATION = Duration.ofSeconds(2); private final SyncState syncState; - private final SyncTargetManager syncTargetManager; + private final AbstractSyncTargetManager syncTargetManager; private final DownloadPipelineFactory downloadPipelineFactory; private final EthScheduler scheduler; @@ -55,7 +55,7 @@ public class PipelineChainDownloader implements ChainDownloader { public PipelineChainDownloader( final SyncState syncState, - final SyncTargetManager syncTargetManager, + final AbstractSyncTargetManager syncTargetManager, final DownloadPipelineFactory downloadPipelineFactory, final EthScheduler scheduler, final MetricsSystem metricsSystem) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index 6a681bec28..aee56cda22 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -129,12 +129,16 @@ public class BackwardSyncContext { backwardChain.addNewHash(newBlockHash); } - final Status status = getOrStartSyncSession(); - backwardChain - .getBlock(newBlockHash) - .ifPresent( - newTargetBlock -> status.updateTargetHeight(newTargetBlock.getHeader().getNumber())); - return status.currentFuture; + if (isReady()) { + final Status status = getOrStartSyncSession(); + backwardChain + .getBlock(newBlockHash) + .ifPresent( + newTargetBlock -> status.updateTargetHeight(newTargetBlock.getHeader().getNumber())); + return status.currentFuture; + } else { + return CompletableFuture.failedFuture(new Throwable("Backward sync is not ready")); + } } public synchronized CompletableFuture syncBackwardsUntil(final Block newPivot) { @@ -142,9 +146,13 @@ public class BackwardSyncContext { backwardChain.appendTrustedBlock(newPivot); } - final Status status = getOrStartSyncSession(); - status.updateTargetHeight(newPivot.getHeader().getNumber()); - return status.currentFuture; + if (isReady()) { + final Status status = getOrStartSyncSession(); + status.updateTargetHeight(newPivot.getHeader().getNumber()); + return status.currentFuture; + } else { + return CompletableFuture.failedFuture(new Throwable("Backward sync is not ready")); + } } private Status getOrStartSyncSession() { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java index 6443bf2824..1b53c25004 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.eth.sync.PipelineChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; -import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncTargetManager; +import org.hyperledger.besu.ethereum.eth.sync.fastsync.SyncTargetManager; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -38,8 +38,8 @@ public class CheckpointSyncChainDownloader extends FastSyncChainDownloader { final MetricsSystem metricsSystem, final FastSyncState fastSyncState) { - final FastSyncTargetManager syncTargetManager = - new FastSyncTargetManager( + final SyncTargetManager syncTargetManager = + new SyncTargetManager( config, worldStateStorageCoordinator, protocolSchedule, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java index 250e1e8e3e..36f56ccfd8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java @@ -38,8 +38,8 @@ public class FastSyncChainDownloader { final MetricsSystem metricsSystem, final FastSyncState fastSyncState) { - final FastSyncTargetManager syncTargetManager = - new FastSyncTargetManager( + final SyncTargetManager syncTargetManager = + new SyncTargetManager( config, worldStateStorageCoordinator, protocolSchedule, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java index 206b76c09a..f7e42b32f6 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java @@ -136,8 +136,8 @@ public class FastSyncDownloadPipelineFactory implements DownloadPipelineFactory new DownloadBodiesStep(protocolSchedule, ethContext, metricsSystem); final DownloadReceiptsStep downloadReceiptsStep = new DownloadReceiptsStep(ethContext, metricsSystem); - final FastImportBlocksStep importBlockStep = - new FastImportBlocksStep( + final ImportBlocksStep importBlockStep = + new ImportBlocksStep( protocolSchedule, protocolContext, attachedValidationPolicy, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index 7a66d41b34..e9746d2612 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -45,7 +45,9 @@ public class FastSyncDownloader { private static final Duration FAST_SYNC_RETRY_DELAY = Duration.ofSeconds(5); - private static final Logger LOG = LoggerFactory.getLogger(FastSyncDownloader.class); + @SuppressWarnings("PrivateStaticFinalLoggers") + protected final Logger LOG = LoggerFactory.getLogger(getClass()); + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final WorldStateDownloader worldStateDownloader; private final TaskCollection taskCollection; @@ -76,7 +78,7 @@ public class FastSyncDownloader { public CompletableFuture start() { if (!running.compareAndSet(false, true)) { - throw new IllegalStateException("FastSyncDownloader already running"); + throw new IllegalStateException("SyncDownloader already running"); } LOG.info("Starting sync"); return start(initialFastSyncState); @@ -112,7 +114,7 @@ public class FastSyncDownloader { protected CompletableFuture handleFailure(final Throwable error) { trailingPeerRequirements = Optional.empty(); Throwable rootCause = ExceptionUtils.rootCause(error); - if (rootCause instanceof FastSyncException) { + if (rootCause instanceof SyncException) { return CompletableFuture.failedFuture(error); } else if (rootCause instanceof StalledDownloadException) { LOG.debug("Stalled sync re-pivoting to newer block."); @@ -125,7 +127,7 @@ public class FastSyncDownloader { return start(FastSyncState.EMPTY_SYNC_STATE); } else { LOG.error( - "Encountered an unexpected error during fast sync. Restarting sync in " + "Encountered an unexpected error during sync. Restarting sync in " + FAST_SYNC_RETRY_DELAY.getSeconds() + " seconds.", error); @@ -137,7 +139,7 @@ public class FastSyncDownloader { public void stop() { synchronized (this) { if (running.compareAndSet(true, false)) { - LOG.info("Stopping fast sync"); + LOG.info("Stopping sync"); // Cancelling the world state download will also cause the chain download to be cancelled. worldStateDownloader.cancel(); } @@ -154,7 +156,7 @@ public class FastSyncDownloader { MoreFiles.deleteRecursively(fastSyncDataDirectory, RecursiveDeleteOption.ALLOW_INSECURE); } } catch (final IOException e) { - LOG.error("Unable to clean up fast sync state", e); + LOG.error("Unable to clean up sync state", e); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStep.java similarity index 96% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStep.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStep.java index c2297faa19..bcf8f00307 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStep.java @@ -33,8 +33,8 @@ import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FastImportBlocksStep implements Consumer> { - private static final Logger LOG = LoggerFactory.getLogger(FastImportBlocksStep.class); +public class ImportBlocksStep implements Consumer> { + private static final Logger LOG = LoggerFactory.getLogger(ImportBlocksStep.class); private static final long PRINT_DELAY = TimeUnit.SECONDS.toMillis(30L); private final ProtocolSchedule protocolSchedule; @@ -46,7 +46,7 @@ public class FastImportBlocksStep implements Consumer> { private OptionalLong logStartBlock = OptionalLong.empty(); private final BlockHeader pivotHeader; - public FastImportBlocksStep( + public ImportBlocksStep( final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final ValidationPolicy headerValidationPolicy, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetriever.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetriever.java index e20bb16799..6ddd7db901 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetriever.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetriever.java @@ -156,8 +156,7 @@ public class PivotBlockRetriever { || pivotBlockNumber.get() <= BlockHeader.GENESIS_BLOCK_NUMBER) { LOG.info("Max retries reached, cancel pivot block download."); // Pivot block selection has failed - result.completeExceptionally( - new FastSyncException(FastSyncError.PIVOT_BLOCK_HEADER_MISMATCH)); + result.completeExceptionally(new SyncException(SyncError.PIVOT_BLOCK_HEADER_MISMATCH)); return; } else { LOG.info("Move pivot block back to {} and retry.", pivotBlockNumber); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncError.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncError.java similarity index 96% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncError.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncError.java index d836173357..d1277ddfdc 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncError.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncError.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync; -public enum FastSyncError { +public enum SyncError { NO_PEERS_AVAILABLE, PIVOT_BLOCK_HEADER_MISMATCH, UNEXPECTED_ERROR diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncException.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncException.java similarity index 69% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncException.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncException.java index 79baf02c43..72e928dc18 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncException.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncException.java @@ -14,21 +14,21 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync; -public class FastSyncException extends RuntimeException { +public class SyncException extends RuntimeException { - private final FastSyncError error; + private final SyncError error; - public FastSyncException(final FastSyncError error) { - super("Fast sync failed: " + error); + public SyncException(final SyncError error) { + super("Sync failed: " + error); this.error = error; } - public FastSyncError getError() { + public SyncError getError() { return error; } - public FastSyncException(final Throwable error) { + public SyncException(final Throwable error) { super(error); - this.error = FastSyncError.UNEXPECTED_ERROR; + this.error = SyncError.UNEXPECTED_ERROR; } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncTargetManager.java similarity index 96% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncTargetManager.java index b0482541c5..60fa2db571 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncTargetManager.java @@ -23,7 +23,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.EthPeers; -import org.hyperledger.besu.ethereum.eth.sync.SyncTargetManager; +import org.hyperledger.besu.ethereum.eth.sync.AbstractSyncTargetManager; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.tasks.RetryingGetHeaderFromPeerByNumberTask; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -39,8 +39,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FastSyncTargetManager extends SyncTargetManager { - private static final Logger LOG = LoggerFactory.getLogger(FastSyncTargetManager.class); +public class SyncTargetManager extends AbstractSyncTargetManager { + private static final Logger LOG = LoggerFactory.getLogger(SyncTargetManager.class); private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final ProtocolSchedule protocolSchedule; @@ -53,7 +53,7 @@ public class FastSyncTargetManager extends SyncTargetManager { private final int logDebugRepeatDelay = 15; private final int logInfoRepeatDelay = 120; - public FastSyncTargetManager( + public SyncTargetManager( final SynchronizerConfiguration config, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java index 43c2cfa19c..ee3d36a31a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.eth.sync.SyncTargetManager; +import org.hyperledger.besu.ethereum.eth.sync.AbstractSyncTargetManager; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncTarget; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -34,7 +34,7 @@ import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class FullSyncTargetManager extends SyncTargetManager { +class FullSyncTargetManager extends AbstractSyncTargetManager { private static final Logger LOG = LoggerFactory.getLogger(FullSyncTargetManager.class); private final ProtocolContext protocolContext; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java index 6f08ecb1aa..581905b44b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java @@ -26,13 +26,8 @@ import org.hyperledger.besu.services.tasks.TaskCollection; import java.nio.file.Path; import java.util.concurrent.CompletableFuture; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class SnapSyncDownloader extends FastSyncDownloader { - private static final Logger LOG = LoggerFactory.getLogger(SnapSyncDownloader.class); - public SnapSyncDownloader( final FastSyncActions fastSyncActions, final WorldStateStorageCoordinator worldStateStorageCoordinator, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapsyncMetricsManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java similarity index 98% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapsyncMetricsManager.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java index 25af2e4262..da7607de7f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapsyncMetricsManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync; import static io.netty.util.internal.ObjectUtil.checkNonEmpty; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.HEAL_TRIE; +import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.HEAL_TRIE; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -37,9 +37,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Manages the metrics related to the SnapSync process. */ -public class SnapsyncMetricsManager { +public class SnapSyncMetricsManager { - private static final Logger LOG = LoggerFactory.getLogger(SnapsyncMetricsManager.class); + private static final Logger LOG = LoggerFactory.getLogger(SnapSyncMetricsManager.class); private static final long PRINT_DELAY = TimeUnit.MINUTES.toMillis(1); private final MetricsSystem metricsSystem; @@ -79,7 +79,7 @@ public class SnapsyncMetricsManager { private long lastNotifyTimestamp; - public SnapsyncMetricsManager(final MetricsSystem metricsSystem, final EthContext ethContext) { + public SnapSyncMetricsManager(final MetricsSystem metricsSystem, final EthContext ethContext) { this.metricsSystem = metricsSystem; this.ethContext = ethContext; percentageProgress = new AtomicReference<>(new BigDecimal(0)); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index fbff9061c5..ae670b9db1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -87,7 +87,7 @@ public class SnapWorldDownloadState extends WorldDownloadState private OptionalLong blockObserverId; // metrics around the snapsync - private final SnapsyncMetricsManager metricsManager; + private final SnapSyncMetricsManager metricsManager; public SnapWorldDownloadState( final WorldStateStorageCoordinator worldStateStorageCoordinator, @@ -97,7 +97,7 @@ public class SnapWorldDownloadState extends WorldDownloadState final InMemoryTasksPriorityQueues pendingRequests, final int maxRequestsWithoutProgress, final long minMillisBeforeStalling, - final SnapsyncMetricsManager metricsManager, + final SnapSyncMetricsManager metricsManager, final Clock clock) { super( worldStateStorageCoordinator, @@ -417,7 +417,7 @@ public class SnapWorldDownloadState extends WorldDownloadState __ -> {}); } - public SnapsyncMetricsManager getMetricsManager() { + public SnapSyncMetricsManager getMetricsManager() { return metricsManager; } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index 2166ed7b55..6c8f100975 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.DOWNLOAD; +import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.DOWNLOAD; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest.createAccountRangeDataRequest; import org.hyperledger.besu.datatypes.Hash; @@ -133,8 +133,8 @@ public class SnapWorldStateDownloader implements WorldStateDownloader { stateRoot, snapTaskCollection.size()); - final SnapsyncMetricsManager snapsyncMetricsManager = - new SnapsyncMetricsManager(metricsSystem, ethContext); + final SnapSyncMetricsManager snapsyncMetricsManager = + new SnapSyncMetricsManager(metricsSystem, ethContext); final SnapWorldDownloadState newDownloadState = new SnapWorldDownloadState( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 4977d3b943..7829ad75eb 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -18,7 +18,7 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_R import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.findNewBeginElementInRange; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.ACCOUNT_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.DOWNLOAD; +import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.DOWNLOAD; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index 428ab47189..05bbeed3da 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -16,7 +16,7 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.HEAL_FLAT; +import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.HEAL_FLAT; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloaderTest.java index babacfc5e0..91d33a1114 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloaderTest.java @@ -52,7 +52,7 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) public class PipelineChainDownloaderTest { - @Mock private SyncTargetManager syncTargetManager; + @Mock private AbstractSyncTargetManager syncTargetManager; @Mock private DownloadPipelineFactory downloadPipelineFactory; @Mock private EthScheduler scheduler; @Mock private Pipeline downloadPipeline; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index 3187f55ef9..27c931bb78 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -48,7 +48,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; -import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState; +import org.hyperledger.besu.ethereum.referencetests.ForestReferenceTestWorldState; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -154,7 +154,7 @@ public class BackwardSyncContextTest { new BlockProcessingOutputs( // use forest-based worldstate since it does not require // blockheader stateroot to match actual worldstate root - DefaultReferenceTestWorldState.create(Collections.emptyMap()), + ForestReferenceTestWorldState.create(Collections.emptyMap()), blockDataGenerator.receipts(block)))); }); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java index e38a9647b7..e1848493dd 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java @@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState; +import org.hyperledger.besu.ethereum.referencetests.ForestReferenceTestWorldState; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.nio.charset.StandardCharsets; @@ -138,7 +138,7 @@ public class ForwardSyncStepTest { return new BlockProcessingResult( Optional.of( new BlockProcessingOutputs( - DefaultReferenceTestWorldState.create(Collections.emptyMap()), + ForestReferenceTestWorldState.create(Collections.emptyMap()), blockDataGenerator.receipts(block)))); }); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksPercentageCalculationTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksPercentageCalculationTest.java index a68194abbe..ed5303897a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksPercentageCalculationTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksPercentageCalculationTest.java @@ -22,11 +22,11 @@ public class FastImportBlocksPercentageCalculationTest { @Test public void blocksPercent_calculations() { - assertThat(FastImportBlocksStep.getBlocksPercent(1, 1)).isEqualByComparingTo(100l); - assertThat(FastImportBlocksStep.getBlocksPercent(1, 100)).isEqualByComparingTo(1l); - assertThat(FastImportBlocksStep.getBlocksPercent(0, 100)).isEqualByComparingTo(0l); - assertThat(FastImportBlocksStep.getBlocksPercent(99, 0)).isEqualByComparingTo(0l); - assertThat(FastImportBlocksStep.getBlocksPercent(1, 1000)).isEqualByComparingTo(0l); - assertThat(FastImportBlocksStep.getBlocksPercent(1, 10000)).isEqualByComparingTo(0l); + assertThat(ImportBlocksStep.getBlocksPercent(1, 1)).isEqualByComparingTo(100l); + assertThat(ImportBlocksStep.getBlocksPercent(1, 100)).isEqualByComparingTo(1l); + assertThat(ImportBlocksStep.getBlocksPercent(0, 100)).isEqualByComparingTo(0l); + assertThat(ImportBlocksStep.getBlocksPercent(99, 0)).isEqualByComparingTo(0l); + assertThat(ImportBlocksStep.getBlocksPercent(1, 1000)).isEqualByComparingTo(0l); + assertThat(ImportBlocksStep.getBlocksPercent(1, 10000)).isEqualByComparingTo(0l); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index 34beae396f..1b136b4c3f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -181,11 +181,11 @@ public class FastSyncDownloaderTest { public void shouldAbortIfSelectPivotBlockFails(final DataStorageFormat dataStorageFormat) { setup(dataStorageFormat); when(fastSyncActions.selectPivotBlock(FastSyncState.EMPTY_SYNC_STATE)) - .thenThrow(new FastSyncException(FastSyncError.UNEXPECTED_ERROR)); + .thenThrow(new SyncException(SyncError.UNEXPECTED_ERROR)); final CompletableFuture result = downloader.start(); - assertCompletedExceptionally(result, FastSyncError.UNEXPECTED_ERROR); + assertCompletedExceptionally(result, SyncError.UNEXPECTED_ERROR); verify(fastSyncActions).selectPivotBlock(FastSyncState.EMPTY_SYNC_STATE); verifyNoMoreInteractions(fastSyncActions); @@ -224,10 +224,10 @@ public class FastSyncDownloaderTest { assertThat(result).isNotDone(); - worldStateFuture.completeExceptionally(new FastSyncException(FastSyncError.NO_PEERS_AVAILABLE)); + worldStateFuture.completeExceptionally(new SyncException(SyncError.NO_PEERS_AVAILABLE)); verify(chainDownloader).cancel(); chainFuture.completeExceptionally(new CancellationException()); - assertCompletedExceptionally(result, FastSyncError.NO_PEERS_AVAILABLE); + assertCompletedExceptionally(result, SyncError.NO_PEERS_AVAILABLE); assertThat(chainFuture).isCancelled(); } @@ -264,8 +264,8 @@ public class FastSyncDownloaderTest { assertThat(result).isNotDone(); - chainFuture.completeExceptionally(new FastSyncException(FastSyncError.NO_PEERS_AVAILABLE)); - assertCompletedExceptionally(result, FastSyncError.NO_PEERS_AVAILABLE); + chainFuture.completeExceptionally(new SyncException(SyncError.NO_PEERS_AVAILABLE)); + assertCompletedExceptionally(result, SyncError.NO_PEERS_AVAILABLE); assertThat(worldStateFuture).isCancelled(); } @@ -594,13 +594,13 @@ public class FastSyncDownloaderTest { } private void assertCompletedExceptionally( - final CompletableFuture future, final FastSyncError expectedError) { + final CompletableFuture future, final SyncError expectedError) { assertThat(future).isCompletedExceptionally(); future.exceptionally( actualError -> { assertThat(actualError) - .isInstanceOf(FastSyncException.class) - .extracting(ex -> ((FastSyncException) ex).getError()) + .isInstanceOf(SyncException.class) + .extracting(ex -> ((SyncException) ex).getError()) .isEqualTo(expectedError); return null; }); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStepTest.java similarity index 97% rename from ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStepTest.java rename to ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStepTest.java index 448608df71..70c9e10eba 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStepTest.java @@ -45,7 +45,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -public class FastImportBlocksStepTest { +public class ImportBlocksStepTest { @Mock private ProtocolSchedule protocolSchedule; @Mock private ProtocolSpec protocolSpec; @@ -56,7 +56,7 @@ public class FastImportBlocksStepTest { @Mock private BlockHeader pivotHeader; private final BlockDataGenerator gen = new BlockDataGenerator(); - private FastImportBlocksStep importBlocksStep; + private ImportBlocksStep importBlocksStep; @BeforeEach public void setUp() { @@ -66,7 +66,7 @@ public class FastImportBlocksStepTest { when(ommerValidationPolicy.getValidationModeForNextBlock()).thenReturn(LIGHT); importBlocksStep = - new FastImportBlocksStep( + new ImportBlocksStep( protocolSchedule, protocolContext, validationPolicy, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java index 7e539c8a59..723c5afe65 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java @@ -376,9 +376,9 @@ public class PivotBlockRetrieverTest { assertThat(future).isCompletedExceptionally(); assertThatThrownBy(future::get) - .hasRootCauseInstanceOf(FastSyncException.class) - .extracting(e -> ((FastSyncException) ExceptionUtils.rootCause(e)).getError()) - .isEqualTo(FastSyncError.PIVOT_BLOCK_HEADER_MISMATCH); + .hasRootCauseInstanceOf(SyncException.class) + .extracting(e -> ((SyncException) ExceptionUtils.rootCause(e)).getError()) + .isEqualTo(SyncError.PIVOT_BLOCK_HEADER_MISMATCH); } @ParameterizedTest @@ -406,9 +406,9 @@ public class PivotBlockRetrieverTest { assertThat(future).isCompletedExceptionally(); assertThatThrownBy(future::get) - .hasRootCauseInstanceOf(FastSyncException.class) - .extracting(e -> ((FastSyncException) ExceptionUtils.rootCause(e)).getError()) - .isEqualTo(FastSyncError.PIVOT_BLOCK_HEADER_MISMATCH); + .hasRootCauseInstanceOf(SyncException.class) + .extracting(e -> ((SyncException) ExceptionUtils.rootCause(e)).getError()) + .isEqualTo(SyncError.PIVOT_BLOCK_HEADER_MISMATCH); } private Responder responderForFakeBlocks(final long... blockNumbers) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index 033dffec3d..2e3dbb2d08 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -54,7 +54,7 @@ public class PersistDataStepTest { @BeforeEach public void setUp() { - when(downloadState.getMetricsManager()).thenReturn(mock(SnapsyncMetricsManager.class)); + when(downloadState.getMetricsManager()).thenReturn(mock(SnapSyncMetricsManager.class)); } @Test diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 8034994457..d4a4dbe8d2 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -85,7 +85,7 @@ public class SnapWorldDownloadStateTest { private final SnapSyncProcessState snapSyncState = mock(SnapSyncProcessState.class); private final SnapSyncStatePersistenceManager snapContext = mock(SnapSyncStatePersistenceManager.class); - private final SnapsyncMetricsManager metricsManager = mock(SnapsyncMetricsManager.class); + private final SnapSyncMetricsManager metricsManager = mock(SnapSyncMetricsManager.class); private final Blockchain blockchain = mock(Blockchain.class); private final DynamicPivotBlockSelector dynamicPivotBlockManager = mock(DynamicPivotBlockSelector.class); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index fdaeff76f3..5778f06413 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -19,9 +19,9 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; +import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.storage.StorageProvider; @@ -68,7 +68,7 @@ public class AccountFlatDatabaseHealingRangeRequestTest { @BeforeEach public void setup() { Mockito.when(downloadState.getMetricsManager()) - .thenReturn(Mockito.mock(SnapsyncMetricsManager.class)); + .thenReturn(Mockito.mock(SnapSyncMetricsManager.class)); Mockito.when(downloadState.getAccountsHealingList()).thenReturn(new HashSet<>()); } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java index 1dbdc63e92..c3e831a25a 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java @@ -271,6 +271,7 @@ public class T8nExecutor { final TransactionProcessingResult result; try { tracer = tracerManager.getManagedTracer(i, transaction.getHash()); + tracer.tracePrepareTransaction(worldStateUpdater, transaction); tracer.traceStartTransaction(worldStateUpdater, transaction); result = processor.processTransaction( diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java index 036e592e1d..86bb079a29 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java @@ -32,7 +32,7 @@ public class DiscoveryConfiguration { private List bootnodes = new ArrayList<>(); private String dnsDiscoveryURL; private boolean discoveryV5Enabled = false; - private boolean filterOnEnrForkId = false; + private boolean filterOnEnrForkId = NetworkingConfiguration.DEFAULT_FILTER_ON_ENR_FORK_ID; public static DiscoveryConfiguration create() { return new DiscoveryConfiguration(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java index 0de53cfd78..478e361737 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java @@ -23,6 +23,7 @@ public class NetworkingConfiguration { public static final int DEFAULT_INITIATE_CONNECTIONS_FREQUENCY_SEC = 30; public static final int DEFAULT_CHECK_MAINTAINED_CONNECTIONS_FREQUENCY_SEC = 60; public static final int DEFAULT_PEER_LOWER_BOUND = 25; + public static final boolean DEFAULT_FILTER_ON_ENR_FORK_ID = true; private DiscoveryConfiguration discovery = new DiscoveryConfiguration(); private RlpxConfiguration rlpx = new RlpxConfiguration(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index 2324630d83..a4af09481c 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerRequirement; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PingPacketData; import org.hyperledger.besu.ethereum.p2p.discovery.internal.TimerUtil; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; @@ -73,6 +74,8 @@ public abstract class PeerDiscoveryAgent { // The devp2p specification says only accept packets up to 1280, but some // clients ignore that, so we add in a little extra padding. private static final int MAX_PACKET_SIZE_BYTES = 1600; + private static final List PING_PACKET_SOURCE_IGNORED = + List.of("127.0.0.1", "255.255.255.255"); protected final List bootstrapPeers; private final List peerRequirements = new CopyOnWriteArrayList<>(); @@ -81,6 +84,7 @@ public abstract class PeerDiscoveryAgent { private final MetricsSystem metricsSystem; private final RlpxAgent rlpxAgent; private final ForkIdManager forkIdManager; + private final PeerTable peerTable; /* The peer controller, which takes care of the state machine of peers. */ protected Optional controller = Optional.empty(); @@ -109,7 +113,8 @@ public abstract class PeerDiscoveryAgent { final MetricsSystem metricsSystem, final StorageProvider storageProvider, final ForkIdManager forkIdManager, - final RlpxAgent rlpxAgent) { + final RlpxAgent rlpxAgent, + final PeerTable peerTable) { this.metricsSystem = metricsSystem; checkArgument(nodeKey != null, "nodeKey cannot be null"); checkArgument(config != null, "provided configuration cannot be null"); @@ -130,6 +135,7 @@ public abstract class PeerDiscoveryAgent { this.forkIdManager = forkIdManager; this.forkIdSupplier = () -> forkIdManager.getForkIdForChainHead().getForkIdAsBytesList(); this.rlpxAgent = rlpxAgent; + this.peerTable = peerTable; } protected abstract TimerUtil createTimer(); @@ -263,9 +269,9 @@ public abstract class PeerDiscoveryAgent { .peerRequirement(PeerRequirement.combine(peerRequirements)) .peerPermissions(peerPermissions) .metricsSystem(metricsSystem) - .forkIdManager(forkIdManager) .filterOnEnrForkId((config.isFilterOnEnrForkIdEnabled())) .rlpxAgent(rlpxAgent) + .peerTable(peerTable) .build(); } @@ -282,27 +288,7 @@ public abstract class PeerDiscoveryAgent { .flatMap(Endpoint::getTcpPort) .orElse(udpPort); - // If the host is present in the P2P PING packet itself, use that as the endpoint. If the P2P - // PING packet specifies 127.0.0.1 (the default if a custom value is not specified with - // --p2p-host or via a suitable --nat-method) we ignore it in favour of the UDP source address. - // The likelihood is that the UDP source will be 127.0.0.1 anyway, but this reduces the chance - // of an unexpected change in behaviour as a result of - // https://github.com/hyperledger/besu/issues/6224 being fixed. - final String host = - packet - .getPacketData(PingPacketData.class) - .flatMap(PingPacketData::getFrom) - .map(Endpoint::getHost) - .filter(abc -> !abc.equals("127.0.0.1")) - .stream() - .peek( - h -> - LOG.trace( - "Using \"From\" endpoint {} specified in ping packet. Ignoring UDP source host {}", - h, - sourceEndpoint.getHost())) - .findFirst() - .orElseGet(sourceEndpoint::getHost); + final String host = deriveHost(sourceEndpoint, packet); // Notify the peer controller. final DiscoveryPeer peer = @@ -317,6 +303,38 @@ public abstract class PeerDiscoveryAgent { controller.ifPresent(c -> c.onMessage(packet, peer)); } + /** + * method to derive the host from the source endpoint and the P2P PING packet. If the host is + * present in the P2P PING packet itself, use that as the endpoint. If the P2P PING packet + * specifies 127.0.0.1 (the default if a custom value is not specified with --p2p-host or via a + * suitable --nat-method) we ignore it in favour of the UDP source address. Some implementations + * send 127.0.0.1 or 255.255.255.255 anyway, but this reduces the chance of an unexpected change + * in behaviour as a result of https://github.com/hyperledger/besu/issues/6224 being fixed. + * + * @param sourceEndpoint source endpoint of the packet + * @param packet P2P PING packet + * @return host address as string + */ + static String deriveHost(final Endpoint sourceEndpoint, final Packet packet) { + return packet + .getPacketData(PingPacketData.class) + .flatMap(PingPacketData::getFrom) + .map(Endpoint::getHost) + .filter( + fromAddr -> + (!PING_PACKET_SOURCE_IGNORED.contains(fromAddr) + && InetAddresses.isInetAddress(fromAddr))) + .stream() + .peek( + h -> + LOG.trace( + "Using \"From\" endpoint {} specified in ping packet. Ignoring UDP source host {}", + h, + sourceEndpoint.getHost())) + .findFirst() + .orElseGet(sourceEndpoint::getHost); + } + /** * Send a packet to the given recipient. * diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java index 27a2be8beb..ef098896a6 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController.AsyncExecutor; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.discovery.internal.TimerUtil; import org.hyperledger.besu.ethereum.p2p.discovery.internal.VertxTimerUtil; import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions; @@ -73,7 +74,8 @@ public class VertxPeerDiscoveryAgent extends PeerDiscoveryAgent { final MetricsSystem metricsSystem, final StorageProvider storageProvider, final ForkIdManager forkIdManager, - final RlpxAgent rlpxAgent) { + final RlpxAgent rlpxAgent, + final PeerTable peerTable) { super( nodeKey, config, @@ -82,7 +84,8 @@ public class VertxPeerDiscoveryAgent extends PeerDiscoveryAgent { metricsSystem, storageProvider, forkIdManager, - rlpxAgent); + rlpxAgent, + peerTable); checkArgument(vertx != null, "vertx instance cannot be null"); this.vertx = vertx; diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java index ec829d2070..af3790def5 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java @@ -21,8 +21,6 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkId; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.peers.Peer; @@ -129,7 +127,6 @@ public class PeerDiscoveryController { private final DiscoveryProtocolLogger discoveryProtocolLogger; private final LabelledMetric interactionCounter; private final LabelledMetric interactionRetryCounter; - private final ForkIdManager forkIdManager; private final boolean filterOnEnrForkId; private final RlpxAgent rlpxAgent; @@ -161,7 +158,6 @@ public class PeerDiscoveryController { final PeerPermissions peerPermissions, final MetricsSystem metricsSystem, final Optional> maybeCacheForEnrRequests, - final ForkIdManager forkIdManager, final boolean filterOnEnrForkId, final RlpxAgent rlpxAgent) { this.timerUtil = timerUtil; @@ -197,11 +193,11 @@ public class PeerDiscoveryController { "discovery_interaction_retry_count", "Total number of interaction retries performed", "type"); + this.cachedEnrRequests = maybeCacheForEnrRequests.orElse( CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, SECONDS).build()); - this.forkIdManager = forkIdManager; this.filterOnEnrForkId = filterOnEnrForkId; } @@ -314,6 +310,7 @@ public class PeerDiscoveryController { } final DiscoveryPeer peer = resolvePeer(sender); + final Bytes peerId = peer.getId(); switch (packet.getType()) { case PING: if (peerPermissions.allowInboundBonding(peer)) { @@ -333,10 +330,10 @@ public class PeerDiscoveryController { if (filterOnEnrForkId) { requestENR(peer); } - bondingPeers.invalidate(peer.getId()); + bondingPeers.invalidate(peerId); addToPeerTable(peer); recursivePeerRefreshState.onBondingComplete(peer); - Optional.ofNullable(cachedEnrRequests.getIfPresent(peer.getId())) + Optional.ofNullable(cachedEnrRequests.getIfPresent(peerId)) .ifPresent(cachedEnrRequest -> processEnrRequest(peer, cachedEnrRequest)); }); break; @@ -360,12 +357,12 @@ public class PeerDiscoveryController { if (PeerDiscoveryStatus.BONDED.equals(peer.getStatus())) { processEnrRequest(peer, packet); } else if (PeerDiscoveryStatus.BONDING.equals(peer.getStatus())) { - LOG.trace("ENR_REQUEST cached for bonding peer Id: {}", peer.getId()); + LOG.trace("ENR_REQUEST cached for bonding peer Id: {}", peerId); // Due to UDP, it may happen that we receive the ENR_REQUEST just before the PONG. // Because peers want to send the ENR_REQUEST directly after the pong. // If this happens we don't want to ignore the request but process when bonded. // this cache allows to keep the request and to respond after having processed the PONG - cachedEnrRequests.put(peer.getId(), packet); + cachedEnrRequests.put(peerId, packet); } break; case ENR_RESPONSE: @@ -376,26 +373,6 @@ public class PeerDiscoveryController { packet.getPacketData(ENRResponsePacketData.class); final NodeRecord enr = packetData.get().getEnr(); peer.setNodeRecord(enr); - - final Optional maybeForkId = peer.getForkId(); - if (maybeForkId.isPresent()) { - if (forkIdManager.peerCheck(maybeForkId.get())) { - connectOnRlpxLayer(peer); - LOG.debug( - "Peer {} PASSED fork id check. ForkId received: {}", - sender.getId(), - maybeForkId.get()); - } else { - LOG.debug( - "Peer {} FAILED fork id check. ForkId received: {}", - sender.getId(), - maybeForkId.get()); - } - } else { - // if the peer hasn't sent the ForkId try to connect to it anyways - connectOnRlpxLayer(peer); - LOG.debug("No fork id sent by peer: {}", peer.getId()); - } }); break; } @@ -431,9 +408,7 @@ public class PeerDiscoveryController { if (peer.getStatus() != PeerDiscoveryStatus.BONDED) { peer.setStatus(PeerDiscoveryStatus.BONDED); - if (!filterOnEnrForkId) { - connectOnRlpxLayer(peer); - } + connectOnRlpxLayer(peer); } final PeerTable.AddResult result = peerTable.tryAdd(peer); @@ -560,8 +535,6 @@ public class PeerDiscoveryController { */ @VisibleForTesting void requestENR(final DiscoveryPeer peer) { - peer.setStatus(PeerDiscoveryStatus.ENR_REQUESTED); - final Consumer action = interaction -> { final ENRRequestPacketData data = ENRRequestPacketData.create(); @@ -838,7 +811,6 @@ public class PeerDiscoveryController { private Cache cachedEnrRequests = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, SECONDS).build(); - private ForkIdManager forkIdManager; private RlpxAgent rlpxAgent; private Builder() {} @@ -846,10 +818,6 @@ public class PeerDiscoveryController { public PeerDiscoveryController build() { validate(); - if (peerTable == null) { - peerTable = new PeerTable(this.nodeKey.getPublicKey().getEncodedBytes(), 16); - } - return new PeerDiscoveryController( nodeKey, localPeer, @@ -864,7 +832,6 @@ public class PeerDiscoveryController { peerPermissions, metricsSystem, Optional.of(cachedEnrRequests), - forkIdManager, filterOnEnrForkId, rlpxAgent); } @@ -875,8 +842,8 @@ public class PeerDiscoveryController { validateRequiredDependency(timerUtil, "TimerUtil"); validateRequiredDependency(workerExecutor, "AsyncExecutor"); validateRequiredDependency(metricsSystem, "MetricsSystem"); - validateRequiredDependency(forkIdManager, "ForkIdManager"); validateRequiredDependency(rlpxAgent, "RlpxAgent"); + validateRequiredDependency(peerTable, "PeerTable"); } private void validateRequiredDependency(final Object object, final String name) { @@ -970,11 +937,5 @@ public class PeerDiscoveryController { this.rlpxAgent = rlpxAgent; return this; } - - public Builder forkIdManager(final ForkIdManager forkIdManager) { - checkNotNull(forkIdManager); - this.forkIdManager = forkIdManager; - return this; - } } } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java index e153acbfc2..f0e0be1fe2 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java @@ -56,26 +56,21 @@ public class PeerTable { * Builds a new peer table, where distance is calculated using the provided nodeId as a baseline. * * @param nodeId The ID of the node where this peer table is stored. - * @param bucketSize The maximum length of each k-bucket. */ - public PeerTable(final Bytes nodeId, final int bucketSize) { + public PeerTable(final Bytes nodeId) { this.keccak256 = Hash.keccak256(nodeId); this.table = Stream.generate(() -> new Bucket(DEFAULT_BUCKET_SIZE)) .limit(N_BUCKETS + 1) .toArray(Bucket[]::new); this.distanceCache = new ConcurrentHashMap<>(); - this.maxEntriesCnt = N_BUCKETS * bucketSize; + this.maxEntriesCnt = N_BUCKETS * DEFAULT_BUCKET_SIZE; // A bloom filter with 4096 expected insertions of 64-byte keys with a 0.1% false positive // probability yields a memory footprint of ~7.5kb. buildBloomFilter(); } - public PeerTable(final Bytes nodeId) { - this(nodeId, DEFAULT_BUCKET_SIZE); - } - /** * Returns the table's representation of a peer, if it exists. * @@ -83,11 +78,12 @@ public class PeerTable { * @return The stored representation. */ public Optional get(final PeerId peer) { - if (!idBloom.mightContain(peer.getId())) { + final Bytes peerId = peer.getId(); + if (!idBloom.mightContain(peerId)) { return Optional.empty(); } final int distance = distanceFrom(peer); - return table[distance].getAndTouch(peer.getId()); + return table[distance].getAndTouch(peerId); } /** diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java index ec65934b29..11352b38cc 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryAgent; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.discovery.VertxPeerDiscoveryAgent; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeerPrivileges; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; @@ -383,11 +384,12 @@ public class DefaultP2PNetwork implements P2PNetwork { @VisibleForTesting void attemptPeerConnections() { LOG.trace("Initiating connections to discovered peers."); - rlpxAgent.connect( + final Stream toTry = streamDiscoveredPeers() .filter(peer -> peer.getStatus() == PeerDiscoveryStatus.BONDED) .filter(peerDiscoveryAgent::checkForkId) - .sorted(Comparator.comparing(DiscoveryPeer::getLastAttemptedConnection))); + .sorted(Comparator.comparing(DiscoveryPeer::getLastAttemptedConnection)); + toTry.forEach(rlpxAgent::connect); } @Override @@ -511,6 +513,7 @@ public class DefaultP2PNetwork implements P2PNetwork { private Supplier> allConnectionsSupplier; private Supplier> allActiveConnectionsSupplier; private int peersLowerBound; + private PeerTable peerTable; public P2PNetwork build() { validate(); @@ -528,6 +531,7 @@ public class DefaultP2PNetwork implements P2PNetwork { final MutableLocalNode localNode = MutableLocalNode.create(config.getRlpx().getClientId(), 5, supportedCapabilities); final PeerPrivileges peerPrivileges = new DefaultPeerPrivileges(maintainedPeers); + peerTable = new PeerTable(nodeKey.getPublicKey().getEncodedBytes()); rlpxAgent = rlpxAgent == null ? createRlpxAgent(localNode, peerPrivileges) : rlpxAgent; peerDiscoveryAgent = peerDiscoveryAgent == null ? createDiscoveryAgent() : peerDiscoveryAgent; @@ -572,7 +576,8 @@ public class DefaultP2PNetwork implements P2PNetwork { metricsSystem, storageProvider, forkIdManager, - rlpxAgent); + rlpxAgent, + peerTable); } private RlpxAgent createRlpxAgent( @@ -589,6 +594,7 @@ public class DefaultP2PNetwork implements P2PNetwork { .allConnectionsSupplier(allConnectionsSupplier) .allActiveConnectionsSupplier(allActiveConnectionsSupplier) .peersLowerBound(peersLowerBound) + .peerTable(peerTable) .build(); } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java index 98a1f60df3..4a8e227d3d 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.peers.PeerPrivileges; @@ -162,13 +163,6 @@ public class RlpxAgent { } } - public void connect(final Stream peerStream) { - if (!localNode.isReady()) { - return; - } - peerStream.forEach(this::connect); - } - public void disconnect(final Bytes peerId, final DisconnectReason reason) { try { allActiveConnectionsSupplier @@ -206,6 +200,7 @@ public class RlpxAgent { + this.getClass().getSimpleName() + " has finished starting")); } + // Check peer is valid final EnodeURL enode = peer.getEnodeURL(); if (!enode.isListening()) { @@ -380,6 +375,7 @@ public class RlpxAgent { private Supplier> allConnectionsSupplier; private Supplier> allActiveConnectionsSupplier; private int peersLowerBound; + private PeerTable peerTable; private Builder() {} @@ -399,12 +395,13 @@ public class RlpxAgent { localNode, connectionEvents, metricsSystem, - p2pTLSConfiguration.get()); + p2pTLSConfiguration.get(), + peerTable); } else { LOG.debug("Using default NettyConnectionInitializer"); connectionInitializer = new NettyConnectionInitializer( - nodeKey, config, localNode, connectionEvents, metricsSystem); + nodeKey, config, localNode, connectionEvents, metricsSystem, peerTable); } } @@ -499,5 +496,10 @@ public class RlpxAgent { this.peersLowerBound = peersLowerBound; return this; } + + public Builder peerTable(final PeerTable peerTable) { + this.peerTable = peerTable; + return this; + } } } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java index 003a6ab1d9..80be0a673d 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; @@ -60,6 +61,7 @@ abstract class AbstractHandshakeHandler extends SimpleChannelInboundHandler subProtocols, @@ -70,7 +72,8 @@ abstract class AbstractHandshakeHandler extends SimpleChannelInboundHandler { + if (ff.isSuccess()) { + LOG.trace("Successfully wrote hello message"); + } + }); + msg.retain(); + ctx.fireChannelRead(msg); } - - LOG.trace("Sending framed hello"); - - // Exchange keys done - final Framer framer = this.framerProvider.buildFramer(handshaker.secrets()); - - final ByteToMessageDecoder deFramer = - new DeFramer( - framer, - subProtocols, - localNode, - expectedPeer, - connectionEventDispatcher, - connectionFuture, - metricsSystem, - inboundInitiated); - - ctx.channel() - .pipeline() - .replace(this, "DeFramer", deFramer) - .addBefore("DeFramer", "validate", new ValidateFirstOutboundMessage(framer)); - - ctx.writeAndFlush(new OutboundMessage(null, HelloMessage.create(localNode.getPeerInfo()))) - .addListener( - ff -> { - if (ff.isSuccess()) { - LOG.trace("Successfully wrote hello message"); - } - }); - msg.retain(); - ctx.fireChannelRead(msg); } private void disconnect( diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java index b39a6b8219..c7c600d3fd 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; +import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.network.exceptions.BreachOfProtocolException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.IncompatiblePeerException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.PeerChannelClosedException; @@ -70,6 +72,7 @@ final class DeFramer extends ByteToMessageDecoder { private final Optional expectedPeer; private final List subProtocols; private final boolean inboundInitiated; + private final PeerTable peerTable; private boolean hellosExchanged; private final LabelledMetric outboundMessagesCounter; @@ -81,7 +84,8 @@ final class DeFramer extends ByteToMessageDecoder { final PeerConnectionEventDispatcher connectionEventDispatcher, final CompletableFuture connectFuture, final MetricsSystem metricsSystem, - final boolean inboundInitiated) { + final boolean inboundInitiated, + final PeerTable peerTable) { this.framer = framer; this.subProtocols = subProtocols; this.localNode = localNode; @@ -89,6 +93,7 @@ final class DeFramer extends ByteToMessageDecoder { this.connectFuture = connectFuture; this.connectionEventDispatcher = connectionEventDispatcher; this.inboundInitiated = inboundInitiated; + this.peerTable = peerTable; this.outboundMessagesCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.NETWORK, @@ -105,8 +110,11 @@ final class DeFramer extends ByteToMessageDecoder { while ((message = framer.deframe(in)) != null) { if (hellosExchanged) { + out.add(message); + } else if (message.getCode() == WireMessageCodes.HELLO) { + hellosExchanged = true; // Decode first hello and use the payload to modify pipeline final PeerInfo peerInfo; @@ -129,13 +137,27 @@ final class DeFramer extends ByteToMessageDecoder { subProtocols, localNode.getPeerInfo().getCapabilities(), peerInfo.getCapabilities()); - final Optional peer = expectedPeer.or(() -> createPeer(peerInfo, ctx)); - if (peer.isEmpty()) { - LOG.debug("Failed to create connection for peer {}", peerInfo); - connectFuture.completeExceptionally(new PeerChannelClosedException(peerInfo)); - ctx.close(); - return; + + Optional peer; + if (expectedPeer.isPresent()) { + peer = expectedPeer; + } else { + // This is an inbound "Hello" message. Create peer from information from the Hello message + peer = createPeer(peerInfo, ctx); + if (peer.isEmpty()) { + LOG.debug("Failed to create connection for peer {}", peerInfo); + connectFuture.completeExceptionally(new PeerChannelClosedException(peerInfo)); + ctx.close(); + return; + } + // If we can find the DiscoveryPeer for the peer in the PeerTable we use it, because + // it could contains additional information, like the fork id. + final Optional discoveryPeer = peerTable.get(peer.get()); + if (discoveryPeer.isPresent()) { + peer = Optional.of(discoveryPeer.get()); + } } + final PeerConnection connection = new NettyPeerConnection( ctx, @@ -176,7 +198,9 @@ final class DeFramer extends ByteToMessageDecoder { capabilityMultiplexer, connection, connectionEventDispatcher, waitingForPong), new MessageFramer(capabilityMultiplexer, framer)); connectFuture.complete(connection); + } else if (message.getCode() == WireMessageCodes.DISCONNECT) { + final DisconnectMessage disconnectMessage = DisconnectMessage.readFrom(message); LOG.debug( "Peer {} disconnected before sending HELLO. Reason: {}", @@ -185,8 +209,10 @@ final class DeFramer extends ByteToMessageDecoder { ctx.close(); connectFuture.completeExceptionally( new PeerDisconnectedException(disconnectMessage.getReason())); + } else { // Unexpected message - disconnect + LOG.debug( "Message received before HELLO's exchanged (BREACH_OF_PROTOCOL), disconnecting. Peer: {}, Code: {}, Data: {}", expectedPeer.map(Peer::getEnodeURLString).orElse("unknown"), diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java index 184cf5cf8c..962de68f98 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; import org.hyperledger.besu.cryptoservices.NodeKey; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnectionEventDispatcher; @@ -40,7 +41,8 @@ final class HandshakeHandlerInbound extends AbstractHandshakeHandler { final PeerConnectionEventDispatcher connectionEventDispatcher, final MetricsSystem metricsSystem, final HandshakerProvider handshakerProvider, - final FramerProvider framerProvider) { + final FramerProvider framerProvider, + final PeerTable peerTable) { super( subProtocols, localNode, @@ -50,7 +52,8 @@ final class HandshakeHandlerInbound extends AbstractHandshakeHandler { metricsSystem, handshakerProvider, framerProvider, - true); + true, + peerTable); handshaker.prepareResponder(nodeKey); } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java index 205b6f655c..46e600d74b 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.cryptoservices.NodeKey; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; @@ -50,7 +51,8 @@ final class HandshakeHandlerOutbound extends AbstractHandshakeHandler { final PeerConnectionEventDispatcher connectionEventDispatcher, final MetricsSystem metricsSystem, final HandshakerProvider handshakerProvider, - final FramerProvider framerProvider) { + final FramerProvider framerProvider, + final PeerTable peerTable) { super( subProtocols, localNode, @@ -60,7 +62,8 @@ final class HandshakeHandlerOutbound extends AbstractHandshakeHandler { metricsSystem, handshakerProvider, framerProvider, - false); + false, + peerTable); handshaker.prepareInitiator( nodeKey, SignatureAlgorithmFactory.getInstance().createPublicKey(peer.getId())); this.first = handshaker.firstMessage(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java index c20e511df9..f386c59a38 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.ConnectCallback; @@ -68,6 +69,7 @@ public class NettyConnectionInitializer private final PeerConnectionEventDispatcher eventDispatcher; private final MetricsSystem metricsSystem; private final Subscribers connectSubscribers = Subscribers.create(); + private final PeerTable peerTable; private ChannelFuture server; private final EventLoopGroup boss = new NioEventLoopGroup(1); @@ -80,12 +82,14 @@ public class NettyConnectionInitializer final RlpxConfiguration config, final LocalNode localNode, final PeerConnectionEventDispatcher eventDispatcher, - final MetricsSystem metricsSystem) { + final MetricsSystem metricsSystem, + final PeerTable peerTable) { this.nodeKey = nodeKey; this.config = config; this.localNode = localNode; this.eventDispatcher = eventDispatcher; this.metricsSystem = metricsSystem; + this.peerTable = peerTable; metricsSystem.createIntegerGauge( BesuMetricCategory.NETWORK, @@ -244,7 +248,8 @@ public class NettyConnectionInitializer eventDispatcher, metricsSystem, this, - this); + this, + peerTable); } @Nonnull @@ -259,7 +264,8 @@ public class NettyConnectionInitializer eventDispatcher, metricsSystem, this, - this); + this, + peerTable); } @Nonnull diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java index 4e6010771f..db41c1574c 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java @@ -19,6 +19,7 @@ import static org.hyperledger.besu.ethereum.p2p.rlpx.RlpxFrameConstants.LENGTH_M import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.plain.PlainFramer; @@ -55,7 +56,8 @@ public class NettyTLSConnectionInitializer extends NettyConnectionInitializer { final LocalNode localNode, final PeerConnectionEventDispatcher eventDispatcher, final MetricsSystem metricsSystem, - final TLSConfiguration p2pTLSConfiguration) { + final TLSConfiguration p2pTLSConfiguration, + final PeerTable peerTable) { this( nodeKey, config, @@ -63,7 +65,8 @@ public class NettyTLSConnectionInitializer extends NettyConnectionInitializer { eventDispatcher, metricsSystem, defaultTlsContextFactorySupplier(p2pTLSConfiguration), - p2pTLSConfiguration.getClientHelloSniHeaderEnabled()); + p2pTLSConfiguration.getClientHelloSniHeaderEnabled(), + peerTable); } @VisibleForTesting @@ -74,8 +77,9 @@ public class NettyTLSConnectionInitializer extends NettyConnectionInitializer { final PeerConnectionEventDispatcher eventDispatcher, final MetricsSystem metricsSystem, final Supplier tlsContextFactorySupplier, - final Boolean clientHelloSniHeaderEnabled) { - super(nodeKey, config, localNode, eventDispatcher, metricsSystem); + final Boolean clientHelloSniHeaderEnabled, + final PeerTable peerTable) { + super(nodeKey, config, localNode, eventDispatcher, metricsSystem, peerTable); if (tlsContextFactorySupplier != null) { this.tlsContextFactorySupplier = Optional.of(Suppliers.memoize(tlsContextFactorySupplier::get)); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java index 240f4673eb..aae83ffdc8 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java @@ -36,6 +36,7 @@ import org.hyperledger.besu.ethereum.p2p.discovery.internal.MockPeerDiscoveryAge import org.hyperledger.besu.ethereum.p2p.discovery.internal.NeighborsPacketData; import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PacketType; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PingPacketData; import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeer; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.p2p.peers.Peer; @@ -53,6 +54,7 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt64; import org.ethereum.beacon.discovery.schema.NodeRecord; import org.junit.jupiter.api.Test; @@ -834,6 +836,47 @@ public class PeerDiscoveryAgentTest { assertThat(agent.isActive()).isFalse(); } + @Test + public void assertHostCorrectlyRevertsOnIgnoredPacketFrom() { + final String sourceHost = "UDP_SOURCE_ORIGIN_HOST"; + final String localHost = "127.0.0.1"; + final String broadcastDefaultHost = "255.255.255.255"; + final String routableHost = "50.50.50.50"; + + Endpoint source = new Endpoint(sourceHost, 30303, Optional.empty()); + Endpoint endpointLocal = new Endpoint(localHost, 30303, Optional.empty()); + Endpoint endpointBroadcast = new Endpoint(broadcastDefaultHost, 30303, Optional.empty()); + Endpoint endpointRoutable = new Endpoint(routableHost, 30303, Optional.empty()); + + Packet mockLocal = + when(mock(Packet.class).getPacketData(any())) + .thenReturn( + Optional.of( + PingPacketData.create(Optional.of(endpointLocal), endpointLocal, UInt64.ONE))) + .getMock(); + Packet mockBroadcast = + when(mock(Packet.class).getPacketData(any())) + .thenReturn( + Optional.of( + PingPacketData.create( + Optional.of(endpointBroadcast), endpointLocal, UInt64.ONE))) + .getMock(); + Packet mockWellFormed = + when(mock(Packet.class).getPacketData(any())) + .thenReturn( + Optional.of( + PingPacketData.create( + Optional.of(endpointRoutable), endpointLocal, UInt64.ONE))) + .getMock(); + + // assert a pingpacketdata from address of 127.0.0.1 reverts to the udp source host + assertThat(PeerDiscoveryAgent.deriveHost(source, mockLocal)).isEqualTo(sourceHost); + // assert that 255.255.255.255 reverts to the udp source host + assertThat(PeerDiscoveryAgent.deriveHost(source, mockBroadcast)).isEqualTo(sourceHost); + // assert that a well-formed routable address in the ping packet data is used + assertThat(PeerDiscoveryAgent.deriveHost(source, mockWellFormed)).isEqualTo(routableHost); + } + protected void bondViaIncomingPing( final MockPeerDiscoveryAgent agent, final MockPeerDiscoveryAgent otherNode) { final Packet pingPacket = helper.createPingPacket(otherNode, agent); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java index 464273243a..ffc9fb1c30 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java @@ -295,6 +295,7 @@ public class PeerDiscoveryTestHelper { config.setAdvertisedHost(advertisedHost); config.setBindPort(port); config.setActive(active); + config.setFilterOnEnrForkId(false); final ForkIdManager mockForkIdManager = mock(ForkIdManager.class); final ForkId forkId = new ForkId(Bytes.EMPTY, Bytes.EMPTY); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java index ea6e1593b8..88196f18b5 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java @@ -63,7 +63,8 @@ public class MockPeerDiscoveryAgent extends PeerDiscoveryAgent { new NoOpMetricsSystem(), new InMemoryKeyValueStorageProvider(), forkIdManager, - rlpxAgent); + rlpxAgent, + new PeerTable(nodeKey.getPublicKey().getEncodedBytes())); this.agentNetwork = agentNetwork; } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java index 182366ac03..ca0d7c6430 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java @@ -35,8 +35,6 @@ import org.hyperledger.besu.crypto.Hash; import org.hyperledger.besu.crypto.SignatureAlgorithm; import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkId; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.Endpoint; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; @@ -1480,14 +1478,12 @@ public class PeerDiscoveryControllerTest { } @Test - public void shouldFiltersOnForkIdSuccess() { + public void forkIdShouldBeAvailableIfEnrPacketContainsForkId() { final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); final List peers = helper.createDiscoveryPeers(nodeKeys); - final ForkIdManager forkIdManager = mock(ForkIdManager.class); final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = prepareForForkIdCheck(forkIdManager, nodeKeys, sender, true); + final Packet enrPacket = prepareForForkIdCheck(nodeKeys, sender, true); - when(forkIdManager.peerCheck(any(ForkId.class))).thenReturn(true); controller.onMessage(enrPacket, sender); final Optional maybePeer = @@ -1501,35 +1497,12 @@ public class PeerDiscoveryControllerTest { verify(controller, times(1)).connectOnRlpxLayer(eq(maybePeer.get())); } - @Test - public void shouldFiltersOnForkIdFailure() { - final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); - final List peers = helper.createDiscoveryPeers(nodeKeys); - final ForkIdManager forkIdManager = mock(ForkIdManager.class); - final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = prepareForForkIdCheck(forkIdManager, nodeKeys, sender, true); - - when(forkIdManager.peerCheck(any(ForkId.class))).thenReturn(false); - controller.onMessage(enrPacket, sender); - - final Optional maybePeer = - controller - .streamDiscoveredPeers() - .filter(p -> p.getId().equals(sender.getId())) - .findFirst(); - - assertThat(maybePeer.isPresent()).isTrue(); - assertThat(maybePeer.get().getForkId().isPresent()).isTrue(); - verify(controller, never()).connectOnRlpxLayer(eq(maybePeer.get())); - } - @Test public void shouldStillCallConnectIfNoForkIdSent() { final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); final List peers = helper.createDiscoveryPeers(nodeKeys); final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = - prepareForForkIdCheck(mock(ForkIdManager.class), nodeKeys, sender, false); + final Packet enrPacket = prepareForForkIdCheck(nodeKeys, sender, false); controller.onMessage(enrPacket, sender); @@ -1546,10 +1519,7 @@ public class PeerDiscoveryControllerTest { @NotNull private Packet prepareForForkIdCheck( - final ForkIdManager forkIdManager, - final List nodeKeys, - final DiscoveryPeer sender, - final boolean sendForkId) { + final List nodeKeys, final DiscoveryPeer sender, final boolean sendForkId) { final HashMap packetTypeBytesHashMap = new HashMap<>(); final OutboundMessageHandler outboundMessageHandler = (dp, pa) -> packetTypeBytesHashMap.put(pa.getType(), pa.getHash()); @@ -1573,7 +1543,6 @@ public class PeerDiscoveryControllerTest { .outboundMessageHandler(outboundMessageHandler) .enrCache(enrs) .filterOnForkId(true) - .forkIdManager(forkIdManager) .build(); // Mock the creation of the PING packet, so that we can control the hash, which gets validated @@ -1720,7 +1689,6 @@ public class PeerDiscoveryControllerTest { private Cache enrs = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, TimeUnit.SECONDS).build(); private boolean filterOnForkId = false; - private ForkIdManager forkIdManager; public static ControllerBuilder create() { return new ControllerBuilder(); @@ -1776,11 +1744,6 @@ public class PeerDiscoveryControllerTest { return this; } - public ControllerBuilder forkIdManager(final ForkIdManager forkIdManager) { - this.forkIdManager = forkIdManager; - return this; - } - PeerDiscoveryController build() { checkNotNull(nodeKey); if (localPeer == null) { @@ -1803,7 +1766,6 @@ public class PeerDiscoveryControllerTest { .peerPermissions(peerPermissions) .metricsSystem(new NoOpMetricsSystem()) .cacheForEnrRequests(enrs) - .forkIdManager(forkIdManager == null ? mock(ForkIdManager.class) : forkIdManager) .filterOnEnrForkId(filterOnForkId) .rlpxAgent(mock(RlpxAgent.class)) .build()); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java index 949b318906..6320c90962 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryTestHelper; @@ -72,7 +71,6 @@ public class PeerDiscoveryTableRefreshTest { .tableRefreshIntervalMs(0) .metricsSystem(new NoOpMetricsSystem()) .rlpxAgent(mock(RlpxAgent.class)) - .forkIdManager(mock(ForkIdManager.class)) .build()); controller.start(); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java index c0909a9b8b..dff9d23165 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java @@ -43,7 +43,7 @@ public class PeerTableTest { @Test public void addPeer() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final List peers = helper.createDiscoveryPeers(5); for (final DiscoveryPeer peer : peers) { @@ -63,7 +63,7 @@ public class PeerTableTest { .ipAddress("127.0.0.1") .discoveryAndListeningPorts(12345) .build()); - final PeerTable table = new PeerTable(localPeer.getId(), 16); + final PeerTable table = new PeerTable(localPeer.getId()); final PeerTable.AddResult result = table.tryAdd(localPeer); assertThat(result.getOutcome()).isEqualTo(AddOutcome.SELF); @@ -72,7 +72,7 @@ public class PeerTableTest { @Test public void peerExists() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); assertThat(table.tryAdd(peer).getOutcome()).isEqualTo(AddOutcome.ADDED); @@ -87,7 +87,7 @@ public class PeerTableTest { @Test public void peerExists_withDifferentIp() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -107,7 +107,7 @@ public class PeerTableTest { @Test public void peerExists_withDifferentUdpPort() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -127,7 +127,7 @@ public class PeerTableTest { @Test public void peerExists_withDifferentIdAndUdpPort() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -147,7 +147,7 @@ public class PeerTableTest { @Test public void evictExistingPeerShouldEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); table.tryAdd(peer); @@ -158,7 +158,7 @@ public class PeerTableTest { @Test public void evictPeerFromEmptyTableShouldNotEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); final EvictResult evictResult = table.tryEvict(peer); @@ -167,7 +167,7 @@ public class PeerTableTest { @Test public void evictAbsentPeerShouldNotEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); final List otherPeers = helper.createDiscoveryPeers(5); otherPeers.forEach(table::tryAdd); @@ -179,7 +179,7 @@ public class PeerTableTest { @Test public void evictSelfPeerShouldReturnSelfOutcome() { final DiscoveryPeer peer = helper.createDiscoveryPeer(); - final PeerTable table = new PeerTable(peer.getId(), 16); + final PeerTable table = new PeerTable(peer.getId()); final EvictResult evictResult = table.tryEvict(peer); assertThat(evictResult.getOutcome()).isEqualTo(EvictOutcome.SELF); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java index e79abb883d..5d26f8cc6e 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java @@ -57,7 +57,7 @@ public class RecursivePeerRefreshStateTest { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 100); @@ -180,7 +180,7 @@ public class RecursivePeerRefreshStateTest { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 1); @@ -466,7 +466,7 @@ public class RecursivePeerRefreshStateTest { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 100); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java index a54c9038b8..68a4f76b58 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java @@ -55,7 +55,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; import java.util.stream.Stream; import io.vertx.core.Context; @@ -82,7 +81,7 @@ public final class DefaultP2PNetworkTest { @Mock PeerDiscoveryAgent discoveryAgent; @Mock RlpxAgent rlpxAgent; - @Captor private ArgumentCaptor> peerStreamCaptor; + @Captor private ArgumentCaptor peerCaptor; private final NetworkingConfiguration config = NetworkingConfiguration.create() @@ -276,12 +275,9 @@ public final class DefaultP2PNetworkTest { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); + verify(rlpxAgent, times(1)).connect(peerCaptor.capture()); - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.contains(discoPeer)).isTrue(); - assertThat(capturedPeers.size()).isEqualTo(1); + assertThat(peerCaptor.getValue()).isEqualTo(discoPeer); } @Test @@ -293,12 +289,7 @@ public final class DefaultP2PNetworkTest { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); - - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.contains(discoPeer)).isFalse(); - assertThat(capturedPeers.size()).isEqualTo(0); + verify(rlpxAgent, times(0)).connect(any()); } @Test @@ -314,14 +305,7 @@ public final class DefaultP2PNetworkTest { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); - - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.size()).isEqualTo(3); - assertThat(capturedPeers.get(0)).isEqualTo(discoPeers.get(1)); - assertThat(capturedPeers.get(1)).isEqualTo(discoPeers.get(0)); - assertThat(capturedPeers.get(2)).isEqualTo(discoPeers.get(2)); + verify(rlpxAgent, times(3)).connect(any()); } @Test diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java index d2fdf8c76a..c639579eee 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java @@ -296,7 +296,7 @@ public class RlpxAgentTest { Stream.generate(PeerTestHelper::createPeer).limit(peerNo); agent = spy(agent); - agent.connect(peerStream); + peerStream.forEach(agent::connect); assertThat(agent.getMapOfCompletableFutures().size()).isEqualTo(peerNo); verify(agent, times(peerNo)).connect(any(Peer.class)); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java index f5030fd9ad..952ebf7c2d 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java @@ -24,6 +24,9 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.hyperledger.besu.ethereum.forkid.ForkId; +import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.network.exceptions.BreachOfProtocolException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.IncompatiblePeerException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.PeerChannelClosedException; @@ -104,7 +107,7 @@ public class DeFramerTest { private final LocalNode localNode = LocalNode.create(clientId, p2pVersion, capabilities, localEnode); - private final DeFramer deFramer = createDeFramer(null); + private final DeFramer deFramer = createDeFramer(null, Optional.empty()); @BeforeEach @SuppressWarnings("unchecked") @@ -219,7 +222,7 @@ public class DeFramerTest { final Peer peer = createRemotePeer(); final PeerInfo remotePeerInfo = new PeerInfo(p2pVersion, clientId, capabilities, 0, peer.getId()); - final DeFramer deFramer = createDeFramer(null); + final DeFramer deFramer = createDeFramer(null, Optional.empty()); final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); @@ -260,6 +263,39 @@ public class DeFramerTest { assertThat(out.size()).isEqualTo(1); } + @Test + public void decode_duringHandshakeFindsPeerInPeerTable() + throws ExecutionException, InterruptedException { + final ChannelFuture future = NettyMocks.channelFuture(false); + when(channel.closeFuture()).thenReturn(future); + + final Peer peer = createRemotePeer(); + final PeerInfo remotePeerInfo = + new PeerInfo(p2pVersion, clientId, capabilities, 0, peer.getId()); + + final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); + final Bytes nodeId = helloMessage.getPeerInfo().getNodeId(); + final String enodeURLString = + "enode://" + nodeId.toString().substring(2) + "@" + "12.13.14.15:30303?discport=30301"; + final Optional discoveryPeer = + DiscoveryPeer.from(DefaultPeer.fromURI(enodeURLString)); + final ForkId forkId = new ForkId(Bytes.fromHexString("0x190a55ad"), 4L); + discoveryPeer.orElseThrow().setForkId(forkId); + final DeFramer deFramer = createDeFramer(null, discoveryPeer); + final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); + when(framer.deframe(eq(data))) + .thenReturn(new RawMessage(helloMessage.getCode(), helloMessage.getData())) + .thenReturn(null); + final List out = new ArrayList<>(); + deFramer.decode(ctx, data, out); + + assertThat(connectFuture).isDone(); + assertThat(connectFuture).isNotCompletedExceptionally(); + final PeerConnection peerConnection = connectFuture.get(); + assertThat(peerConnection.getPeerInfo()).isEqualTo(remotePeerInfo); + assertThat(peerConnection.getPeer().getForkId().orElseThrow()).isEqualTo(forkId); + } + @Test public void decode_handlesUnexpectedPeerId() { final ChannelFuture future = NettyMocks.channelFuture(false); @@ -274,7 +310,7 @@ public class DeFramerTest { capabilities, peer.getEnodeURL().getListeningPortOrZero(), mismatchedId); - final DeFramer deFramer = createDeFramer(peer); + final DeFramer deFramer = createDeFramer(peer, Optional.empty()); final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); @@ -414,7 +450,10 @@ public class DeFramerTest { forPeer.getId()); } - private DeFramer createDeFramer(final Peer expectedPeer) { + private DeFramer createDeFramer( + final Peer expectedPeer, final Optional peerInPeerTable) { + final PeerTable peerTable = new PeerTable(localNode.getPeerInfo().getNodeId()); + peerInPeerTable.ifPresent(peerTable::tryAdd); return new DeFramer( framer, Arrays.asList(MockSubProtocol.create("eth")), @@ -423,6 +462,7 @@ public class DeFramerTest { connectionEventDispatcher, connectFuture, new NoOpMetricsSystem(), - true); + true, + peerTable); } } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java index d4637952b6..f7f3dbc512 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnectionEventDispatcher; @@ -44,6 +45,7 @@ import io.netty.handler.codec.compression.SnappyFrameDecoder; import io.netty.handler.codec.compression.SnappyFrameEncoder; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; +import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -95,7 +97,8 @@ public class NettyTLSConnectionInitializerTest { eventDispatcher, new NoOpMetricsSystem(), () -> tlsContextFactory, - clientHelloSniHeaderEnabled); + clientHelloSniHeaderEnabled, + new PeerTable(Bytes.random(64))); } @Test diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java index b56053f54a..8c1bd51552 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java @@ -23,7 +23,10 @@ import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdat import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; import org.hyperledger.besu.evm.internal.EvmConfiguration; +import java.util.concurrent.ConcurrentHashMap; + import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdateAccumulator { private final BonsaiPreImageProxy preImageProxy; @@ -43,4 +46,29 @@ public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdate // by default do not save hash preImages return preImageProxy.hashAndSavePreImage(bytes); } + + public BonsaiReferenceTestUpdateAccumulator createDetachedAccumulator() { + final BonsaiReferenceTestUpdateAccumulator copy = + new BonsaiReferenceTestUpdateAccumulator( + wrappedWorldView(), + accountPreloader, + storagePreloader, + preImageProxy, + evmConfiguration); + getAccountsToUpdate().forEach((k, v) -> copy.getAccountsToUpdate().put(k, v.copy())); + getCodeToUpdate().forEach((k, v) -> copy.getCodeToUpdate().put(k, v.copy())); + copy.getStorageToClear().addAll(getStorageToClear()); + getStorageToUpdate() + .forEach( + (k, v) -> { + StorageConsumingMap> newMap = + new StorageConsumingMap<>(k, new ConcurrentHashMap<>(), v.getConsumer()); + v.forEach((key, value) -> newMap.put(key, value.copy())); + copy.getStorageToUpdate().put(k, newMap); + }); + copy.updatedAccounts.putAll(updatedAccounts); + copy.deletedAccounts.addAll(deletedAccounts); + copy.isAccumulatorStateChanged = true; + return copy; + } } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index cf414dee0d..88ae0bf862 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -23,19 +23,25 @@ import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.Map; +import java.util.Optional; import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -99,6 +105,99 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState // The test harness validates the root hash, no need to validate in-line for reference test } + @Override + public void processExtraStateStorageFormatValidation(final BlockHeader blockHeader) { + if (blockHeader != null) { + final Hash parentStateRoot = getWorldStateRootHash(); + final BonsaiReferenceTestUpdateAccumulator originalUpdater = + ((BonsaiReferenceTestUpdateAccumulator) updater()).createDetachedAccumulator(); + + // validate trielog generation with persisted state + validateStateRolling(parentStateRoot, originalUpdater, blockHeader, false); + // validate trielog generation with frozen state + validateStateRolling(parentStateRoot, originalUpdater, blockHeader, true); + } + } + + /** + * TrieLog is an important part of Bonsai, so it's important to verify the generation of the + * TrieLog by performing rollbacks and rollforwards. + * + * @param blockHeader header of the block to import + */ + private void validateStateRolling( + final Hash parentStateRoot, + final BonsaiReferenceTestUpdateAccumulator originalUpdater, + final BlockHeader blockHeader, + final boolean isFrozenState) { + // With Bonsai, a TrieLog is generated when the state is persisted. Therefore, we generate the + // TrieLog by triggering a state persist in order to closely match the real case scenario. + generateTrieLogFromState(blockHeader, originalUpdater, isFrozenState); + final TrieLog trieLogFromFrozenState = + trieLogManager + .getTrieLogLayer(blockHeader.getBlockHash()) + .orElseThrow(() -> new RuntimeException("trielog not found during test")); + // trying rollback rollfoward with frozen state + validateTrieLog(parentStateRoot, blockHeader, trieLogFromFrozenState); + } + + private void validateTrieLog( + final Hash parentStateRoot, final BlockHeader blockHeader, final TrieLog trieLog) { + + try (var bonsaiWorldState = createBonsaiWorldState(false)) { + BonsaiWorldStateUpdateAccumulator updaterForState = + (BonsaiWorldStateUpdateAccumulator) bonsaiWorldState.updater(); + updaterForState.rollForward(trieLog); + updaterForState.commit(); + bonsaiWorldState.persist(blockHeader); + Hash generatedRootHash = bonsaiWorldState.rootHash(); + if (!bonsaiWorldState.rootHash().equals(blockHeader.getStateRoot())) { + throw new RuntimeException( + "state root becomes invalid following a rollForward %s != %s" + .formatted(blockHeader.getStateRoot(), generatedRootHash)); + } + + updaterForState = (BonsaiWorldStateUpdateAccumulator) bonsaiWorldState.updater(); + updaterForState.rollBack(trieLog); + updaterForState.commit(); + bonsaiWorldState.persist(null); + generatedRootHash = bonsaiWorldState.rootHash(); + if (!bonsaiWorldState.rootHash().equals(parentStateRoot)) { + throw new RuntimeException( + "state root becomes invalid following a rollBackward %s != %s" + .formatted(parentStateRoot, generatedRootHash)); + } + } + } + + private void generateTrieLogFromState( + final BlockHeader blockHeader, + final BonsaiReferenceTestUpdateAccumulator originalUpdater, + final boolean isFrozen) { + // generate trielog + BonsaiReferenceTestUpdateAccumulator updaterForState = + originalUpdater.createDetachedAccumulator(); + try (var bonsaiWorldState = createBonsaiWorldState(isFrozen)) { + bonsaiWorldState.setAccumulator(updaterForState); + updaterForState.commit(); + bonsaiWorldState.persist(blockHeader); + } + } + + private BonsaiWorldState createBonsaiWorldState(final boolean isFrozen) { + BonsaiWorldState bonsaiWorldState = + new BonsaiWorldState( + new BonsaiWorldStateLayerStorage(worldStateStorage), + cachedMerkleTrieLoader, + cachedWorldStorageManager, + trieLogManager, + evmConfiguration); + if (isFrozen) { + bonsaiWorldState.freeze(); // freeze state + } + return bonsaiWorldState; + } + @JsonCreator public static BonsaiReferenceTestWorldState create( final Map accounts) { @@ -111,7 +210,7 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState final EvmConfiguration evmConfiguration) { final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); - final TrieLogManager trieLogManager = new NoOpTrieLogManager(); + final TrieLogManager trieLogManager = new ReferenceTestsInMemoryTrieLogManager(); final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); @@ -150,6 +249,40 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState return this.refTestStorage.streamAccounts(this, startKeyHash, limit); } + static class ReferenceTestsInMemoryTrieLogManager extends TrieLogManager { + + private final Cache trieLogCache = + CacheBuilder.newBuilder().maximumSize(5).build(); + + public ReferenceTestsInMemoryTrieLogManager() { + super(null, null, 0, null); + } + + @Override + public synchronized void saveTrieLog( + final BonsaiWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final BonsaiWorldState forWorldState) { + // notify trie log added observers, synchronously + TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); + trieLogCache.put(forBlockHeader.getHash(), trieLogFactory.serialize(trieLog)); + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + } + + @Override + public long getMaxLayersToLoad() { + return 0; + } + + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + final byte[] trielog = trieLogCache.getIfPresent(blockHash); + trieLogCache.invalidate(blockHash); // remove trielog from the cache + return Optional.ofNullable(trieLogFactory.deserialize(trielog)); + } + } + @Override protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ForestReferenceTestWorldState.java similarity index 67% rename from ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java rename to ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ForestReferenceTestWorldState.java index d9ed862676..9ab158a554 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ForestReferenceTestWorldState.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.referencetests; import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -27,28 +28,41 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonCreator; -public class DefaultReferenceTestWorldState extends ForestMutableWorldState +public class ForestReferenceTestWorldState extends ForestMutableWorldState implements ReferenceTestWorldState { - DefaultReferenceTestWorldState() { + ForestReferenceTestWorldState() { super( new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); } - public DefaultReferenceTestWorldState(final WorldState worldState) { + public ForestReferenceTestWorldState(final WorldState worldState) { super(worldState, EvmConfiguration.DEFAULT); } @Override public ReferenceTestWorldState copy() { - return new DefaultReferenceTestWorldState(this); + return new ForestReferenceTestWorldState(this); + } + + /** + * Executes additional validation checks that are specific to the storage format. + * + *

Depending on the storage format (e.g., Bonsai, etc.), this method performs additional checks + * to validate the state. This could include validating the TrieLog and rolling for Bonsai, or + * potentially other checks for other modes. This method is intended to be used before the state + * root has been validated, to ensure the integrity of other aspects of the state. + */ + @Override + public void processExtraStateStorageFormatValidation(final BlockHeader blockHeader) { + // nothing more to verify with forest } @JsonCreator public static ReferenceTestWorldState create(final Map accounts) { - final ReferenceTestWorldState worldState = new DefaultReferenceTestWorldState(); + final ReferenceTestWorldState worldState = new ForestReferenceTestWorldState(); final WorldUpdater updater = worldState.updater(); for (final Map.Entry entry : accounts.entrySet()) { diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java index 3d4db1edaa..ae446a7607 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.referencetests; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -90,6 +91,8 @@ public interface ReferenceTestWorldState extends MutableWorldState { ReferenceTestWorldState copy(); + void processExtraStateStorageFormatValidation(final BlockHeader blockHeader); + @JsonCreator static ReferenceTestWorldState create(final Map accounts) { // delegate to a Bonsai reference test world state: diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java index e0b0c7332e..5314ff5cc8 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java @@ -28,7 +28,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; @@ -151,7 +150,7 @@ public class GeneralStateReferenceTestTools { return; } - final MutableWorldState worldState = initialWorldState.copy(); + final ReferenceTestWorldState worldState = initialWorldState.copy(); // Several of the GeneralStateTests check if the transaction could potentially // consume more gas than is left for the block it's attempted to be included in. // This check is performed within the `BlockImporter` rather than inside the @@ -193,6 +192,7 @@ public class GeneralStateReferenceTestTools { worldStateUpdater.deleteAccount(coinbase.getAddress()); } worldStateUpdater.commit(); + worldState.processExtraStateStorageFormatValidation(blockHeader); worldState.persist(blockHeader); // Check the world state root hash. diff --git a/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java b/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java index edaed8e241..5bd6de591e 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java @@ -68,7 +68,15 @@ public interface OperationTracer { final MessageFrame frame, final Optional haltReason) {} /** - * Trace the start of a transaction. + * Trace the start of a transaction, before sender account alteration but after validation. + * + * @param worldView an immutable view of the execution context + * @param transaction the transaction which will be processed + */ + default void tracePrepareTransaction(final WorldView worldView, final Transaction transaction) {} + + /** + * Trace the start of a transaction, before execution but after sender account alteration. * * @param worldView an immutable view of the execution context * @param transaction the transaction which will be processed diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 1dcfd00f7e..c72ad93433 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'IGq+V3KaStHCRFkeK3KwPxJYKO4RX9YM1O4JYITk8S8=' + knownHash = 'ZsovOR0oPfomcLP4b+HjikWzM0Tx6sCwi68mf5qwZf4=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java index c535966876..5db7fea509 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java @@ -56,4 +56,12 @@ public interface SegmentIdentifier { * @return true if the segment contains only static data */ boolean containsStaticData(); + + /** + * This flag defines which segment is eligible for the high spec flag, so basically what column + * family is involved with high spec flag + * + * @return true if the segment is involved with the high spec flag + */ + boolean isEligibleToHighSpecFlag(); } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index c6157217e3..a7192cc3b5 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -70,18 +70,16 @@ import org.slf4j.LoggerFactory; public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValueStorage { private static final Logger LOG = LoggerFactory.getLogger(RocksDBColumnarKeyValueStorage.class); - static final String DEFAULT_COLUMN = "default"; private static final int ROCKSDB_FORMAT_VERSION = 5; private static final long ROCKSDB_BLOCK_SIZE = 32768; /** RocksDb blockcache size when using the high spec option */ protected static final long ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC = 1_073_741_824L; /** RocksDb memtable size when using the high spec option */ - protected static final long ROCKSDB_MEMTABLE_SIZE_HIGH_SPEC = 1_073_741_824L; + protected static final long ROCKSDB_MEMTABLE_SIZE_HIGH_SPEC = 536_870_912L; /** Max total size of all WAL file, after which a flush is triggered */ protected static final long WAL_MAX_TOTAL_SIZE = 1_073_741_824L; /** Expected size of a single WAL file, to determine how many WAL files to keep around */ protected static final long EXPECTED_WAL_FILE_SIZE = 67_108_864L; - /** RocksDb number of log files to keep on disk */ private static final long NUMBER_OF_LOG_FILES_TO_KEEP = 7; /** RocksDb Time to roll a log file (1 day = 3600 * 24 seconds) */ @@ -144,7 +142,6 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu this.rocksDBMetricsFactory = rocksDBMetricsFactory; try { - final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); trimmedSegments = new ArrayList<>(defaultSegments); final List existingColumnFamilies = RocksDB.listColumnFamilies(new Options(), configuration.getDatabaseDir().toString()); @@ -156,14 +153,9 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu .noneMatch(existed -> Arrays.equals(existed, ignorableSegment.getId()))) .forEach(trimmedSegments::remove); columnDescriptors = - trimmedSegments.stream().map(this::createColumnDescriptor).collect(Collectors.toList()); - columnDescriptors.add( - new ColumnFamilyDescriptor( - DEFAULT_COLUMN.getBytes(StandardCharsets.UTF_8), - columnFamilyOptions - .setTtl(0) - .setCompressionType(CompressionType.LZ4_COMPRESSION) - .setTableFormatConfig(createBlockBasedTableConfig(configuration)))); + trimmedSegments.stream() + .map(segment -> createColumnDescriptor(segment, configuration)) + .collect(Collectors.toList()); setGlobalOptions(configuration, stats); @@ -174,6 +166,80 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu } } + /** + * Create a Column Family Descriptor for a given segment It defines basically the different + * options to apply to the corresponding Column Family + * + * @param segment the segment identifier + * @param configuration RocksDB configuration + * @return a column family descriptor + */ + private ColumnFamilyDescriptor createColumnDescriptor( + final SegmentIdentifier segment, final RocksDBConfiguration configuration) { + + BlockBasedTableConfig basedTableConfig = createBlockBasedTableConfig(segment, configuration); + + final var options = + new ColumnFamilyOptions() + .setTtl(0) + .setCompressionType(CompressionType.LZ4_COMPRESSION) + .setTableFormatConfig(basedTableConfig); + + if (segment.containsStaticData()) { + options + .setEnableBlobFiles(true) + .setEnableBlobGarbageCollection(false) + .setMinBlobSize(100) + .setBlobCompressionType(CompressionType.LZ4_COMPRESSION); + } + + return new ColumnFamilyDescriptor(segment.getId(), options); + } + + /*** + * Create a Block Base Table configuration for each segment, depending on the configuration in place + * and the segment itself + * + * @param segment The segment related to the column family + * @param config RocksDB configuration + * @return Block Base Table configuration + */ + private BlockBasedTableConfig createBlockBasedTableConfig( + final SegmentIdentifier segment, final RocksDBConfiguration config) { + final LRUCache cache = + new LRUCache( + config.isHighSpec() && segment.isEligibleToHighSpecFlag() + ? ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC + : config.getCacheCapacity()); + return new BlockBasedTableConfig() + .setFormatVersion(ROCKSDB_FORMAT_VERSION) + .setBlockCache(cache) + .setFilterPolicy(new BloomFilter(10, false)) + .setPartitionFilters(true) + .setCacheIndexAndFilterBlocks(false) + .setBlockSize(ROCKSDB_BLOCK_SIZE); + } + + /*** + * Set Global options (DBOptions) + * + * @param configuration RocksDB configuration + * @param stats The statistics object + */ + private void setGlobalOptions(final RocksDBConfiguration configuration, final Statistics stats) { + options = new DBOptions(); + options + .setCreateIfMissing(true) + .setMaxOpenFiles(configuration.getMaxOpenFiles()) + .setStatistics(stats) + .setCreateMissingColumnFamilies(true) + .setLogFileTimeToRoll(TIME_TO_ROLL_LOG_FILE) + .setKeepLogFileNum(NUMBER_OF_LOG_FILES_TO_KEEP) + .setEnv(Env.getDefault().setBackgroundThreads(configuration.getBackgroundThreadCount())) + .setMaxTotalWalSize(WAL_MAX_TOTAL_SIZE) + .setRecycleLogFileNum(WAL_MAX_TOTAL_SIZE / EXPECTED_WAL_FILE_SIZE); + } + /** * Parse RocksDBException and wrap in StorageException * @@ -219,42 +285,6 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu } } - private ColumnFamilyDescriptor createColumnDescriptor(final SegmentIdentifier segment) { - final var options = - new ColumnFamilyOptions() - .setTtl(0) - .setCompressionType(CompressionType.LZ4_COMPRESSION) - .setTableFormatConfig(createBlockBasedTableConfig(configuration)); - - if (segment.containsStaticData()) { - options - .setEnableBlobFiles(true) - .setEnableBlobGarbageCollection(false) - .setMinBlobSize(100) - .setBlobCompressionType(CompressionType.LZ4_COMPRESSION); - } - - return new ColumnFamilyDescriptor(segment.getId(), options); - } - - private void setGlobalOptions(final RocksDBConfiguration configuration, final Statistics stats) { - options = new DBOptions(); - options - .setCreateIfMissing(true) - .setMaxOpenFiles(configuration.getMaxOpenFiles()) - .setMaxTotalWalSize(WAL_MAX_TOTAL_SIZE) - .setRecycleLogFileNum(WAL_MAX_TOTAL_SIZE / EXPECTED_WAL_FILE_SIZE) - .setStatistics(stats) - .setCreateMissingColumnFamilies(true) - .setLogFileTimeToRoll(TIME_TO_ROLL_LOG_FILE) - .setKeepLogFileNum(NUMBER_OF_LOG_FILES_TO_KEEP) - .setEnv(Env.getDefault().setBackgroundThreads(configuration.getBackgroundThreadCount())); - - if (configuration.isHighSpec()) { - options.setDbWriteBufferSize(ROCKSDB_MEMTABLE_SIZE_HIGH_SPEC); - } - } - void initMetrics() { metrics = rocksDBMetricsFactory.create(metricsSystem, configuration, getDB(), stats); } @@ -287,19 +317,6 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValu })); } - BlockBasedTableConfig createBlockBasedTableConfig(final RocksDBConfiguration config) { - final LRUCache cache = - new LRUCache( - config.isHighSpec() ? ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC : config.getCacheCapacity()); - return new BlockBasedTableConfig() - .setFormatVersion(ROCKSDB_FORMAT_VERSION) - .setBlockCache(cache) - .setFilterPolicy(new BloomFilter(10, false)) - .setPartitionFilters(true) - .setCacheIndexAndFilterBlocks(false) - .setBlockSize(ROCKSDB_BLOCK_SIZE); - } - /** * Safe method to map segment identifier to column handle. * diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java index e0ed3f7b39..65d8ee7b58 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java @@ -45,7 +45,7 @@ public class RocksDBKeyValuePrivacyStorageFactoryTest { @TempDir private Path temporaryFolder; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final SegmentIdentifier segment = TestSegment.BAR; - private final List segments = List.of(segment); + private final List segments = List.of(TestSegment.DEFAULT, segment); @Test public void shouldDetectVersion1DatabaseIfNoMetadataFileFound() throws Exception { diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java index c3a8c32eb2..8f1e116b3a 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java @@ -50,7 +50,7 @@ public class RocksDBKeyValueStorageFactoryTest { @TempDir public Path temporaryFolder; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final SegmentIdentifier segment = TestSegment.FOO; - private final List segments = List.of(segment); + private final List segments = List.of(TestSegment.DEFAULT, segment); @Test public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest.java index 7e8cef52cc..bf838f9c4f 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest.java @@ -39,7 +39,7 @@ public class OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest new RocksDBConfigurationBuilder() .databaseDir(Files.createTempDirectory("segmentedStore")) .build(), - Arrays.asList(TestSegment.FOO, TestSegment.BAR), + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), List.of(), new NoOpMetricsSystem(), RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java index 5462d239b9..80b9685b79 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java @@ -194,13 +194,17 @@ public abstract class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValu SegmentedKeyValueStorage store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of(TestSegment.EXPERIMENTAL)); store.close(); // new db will be backward compatible with db without knowledge of experimental column family store = - createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); + createSegmentedStore( + testPath, + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), + List.of()); store.close(); } @@ -212,14 +216,18 @@ public abstract class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValu SegmentedKeyValueStorage store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of()); store.close(); // new db will not be backward compatible with db without knowledge of experimental column // family try { - createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); + createSegmentedStore( + testPath, + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), + List.of()); fail("DB without knowledge of experimental column family should fail"); } catch (StorageException e) { assertThat(e.getMessage()).contains("Unhandled column families"); @@ -230,7 +238,8 @@ public abstract class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValu store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of(TestSegment.EXPERIMENTAL)); store.close(); } @@ -242,27 +251,35 @@ public abstract class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValu SegmentedKeyValueStorage store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of(TestSegment.EXPERIMENTAL)); store.close(); // new db will be backward compatible with db without knowledge of experimental column family store = - createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); + createSegmentedStore( + testPath, + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), + List.of()); store.close(); // Create new db without ignoring experimental colum family will add column to db store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of()); store.close(); // Now, the db will be backward incompatible with db without knowledge of experimental column // family try { - createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); + createSegmentedStore( + testPath, + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), + List.of()); fail("DB without knowledge of experimental column family should fail"); } catch (StorageException e) { assertThat(e.getMessage()).contains("Unhandled column families"); @@ -293,7 +310,10 @@ public abstract class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValu final SegmentedKeyValueStorage store = createSegmentedStore( - folder, metricsSystemMock, List.of(TestSegment.FOO), List.of(TestSegment.EXPERIMENTAL)); + folder, + metricsSystemMock, + List.of(TestSegment.DEFAULT, TestSegment.FOO), + List.of(TestSegment.EXPERIMENTAL)); KeyValueStorage keyValueStorage = new SegmentedKeyValueStorageAdapter(TestSegment.FOO, store); @@ -343,24 +363,28 @@ public abstract class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValu } public enum TestSegment implements SegmentIdentifier { + DEFAULT("default".getBytes(StandardCharsets.UTF_8)), FOO(new byte[] {1}), BAR(new byte[] {2}), EXPERIMENTAL(new byte[] {3}), - STATIC_DATA(new byte[] {4}, true); + STATIC_DATA(new byte[] {4}, true, false); private final byte[] id; private final String nameAsUtf8; private final boolean containsStaticData; + private final boolean eligibleToHighSpecFlag; TestSegment(final byte[] id) { - this(id, false); + this(id, false, false); } - TestSegment(final byte[] id, final boolean containsStaticData) { + TestSegment( + final byte[] id, final boolean containsStaticData, final boolean eligibleToHighSpecFlag) { this.id = id; this.nameAsUtf8 = new String(id, StandardCharsets.UTF_8); this.containsStaticData = containsStaticData; + this.eligibleToHighSpecFlag = eligibleToHighSpecFlag; } @Override @@ -377,6 +401,11 @@ public abstract class RocksDBColumnarKeyValueStorageTest extends AbstractKeyValu public boolean containsStaticData() { return containsStaticData; } + + @Override + public boolean isEligibleToHighSpecFlag() { + return eligibleToHighSpecFlag; + } } protected abstract SegmentedKeyValueStorage createSegmentedStore() throws Exception; diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorageTest.java index bed64f5840..374f553456 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorageTest.java @@ -36,7 +36,7 @@ public class TransactionDBRocksDBColumnarKeyValueStorageTest protected SegmentedKeyValueStorage createSegmentedStore() throws Exception { return new TransactionDBRocksDBColumnarKeyValueStorage( new RocksDBConfigurationBuilder().databaseDir(getTempSubFolder(folder)).build(), - Arrays.asList(TestSegment.FOO, TestSegment.BAR), + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), List.of(), new NoOpMetricsSystem(), RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java index dbd62eaff9..e3d82ca9dc 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java @@ -50,6 +50,11 @@ public class InMemoryKeyValueStorage extends SegmentedKeyValueStorageAdapter { public boolean containsStaticData() { return false; } + + @Override + public boolean isEligibleToHighSpecFlag() { + return false; + } }; private static ConcurrentMap>> asSegmentMap(