Merge branch 'main' into 7311-add-cli-feature-toggle-for-peertask-system

pull/7633/head
Matilda-Clerke 2 months ago committed by GitHub
commit bc11e0c8bb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 3
      CHANGELOG.md
  2. 125
      acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java
  3. 23
      acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java
  4. 5
      besu/src/main/java/org/hyperledger/besu/Besu.java
  5. 175
      besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
  6. 12
      besu/src/main/java/org/hyperledger/besu/components/BesuCommandModule.java
  7. 4
      besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java
  8. 16
      besu/src/main/java/org/hyperledger/besu/components/BesuPluginContextModule.java
  9. 2
      besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
  10. 49
      besu/src/main/scripts/besu-entry.sh
  11. 1
      besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java
  12. 7
      besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java
  13. 16
      besu/src/test/java/org/hyperledger/besu/components/MockBesuCommandModule.java
  14. 1
      build.gradle
  15. 16
      docker/Dockerfile
  16. 8
      docker/test.sh
  17. 10
      docker/tests/02/goss.yaml
  18. 2
      docker/tests/dgoss
  19. 2
      ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java
  20. 5
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java
  21. 21
      ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java
  22. 18
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java
  23. 25
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java
  24. 12
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessage.java
  25. 495
      ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerGetAccountRangeTest.java
  26. 2
      ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerTest.java
  27. 83
      ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessageTest.java
  28. 6
      metrics/core/src/main/java/org/hyperledger/besu/metrics/MetricsSystemModule.java
  29. 2
      metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java
  30. 20
      testutil/src/main/java/org/hyperledger/besu/testutil/BlockTestUtil.java
  31. 111
      testutil/src/main/resources/snap/snapGenesis.json
  32. BIN
      testutil/src/main/resources/snap/testBlockchain.blocks

@ -11,7 +11,10 @@
- Remove privacy test classes support [#7569](https://github.com/hyperledger/besu/pull/7569) - Remove privacy test classes support [#7569](https://github.com/hyperledger/besu/pull/7569)
### Bug fixes ### Bug fixes
- Fix mounted data path directory permissions for besu user [#7575](https://github.com/hyperledger/besu/pull/7575)
- Fix for `debug_traceCall` to handle transactions without specified gas price. [#7510](https://github.com/hyperledger/besu/pull/7510) - Fix for `debug_traceCall` to handle transactions without specified gas price. [#7510](https://github.com/hyperledger/besu/pull/7510)
- Corrects a regression where custom plugin services are not initialized correctly. [#7625](https://github.com/hyperledger/besu/pull/7625)
## 24.9.1 ## 24.9.1

@ -75,6 +75,70 @@ public class ProcessBesuNodeRunner implements BesuNodeRunner {
final Path dataDir = node.homeDirectory(); final Path dataDir = node.homeDirectory();
final List<String> params = commandlineArgs(node, dataDir);
LOG.info("Creating besu process with params {}", params);
final ProcessBuilder processBuilder =
new ProcessBuilder(params)
.directory(new File(System.getProperty("user.dir")).getParentFile().getParentFile())
.redirectErrorStream(true)
.redirectInput(Redirect.INHERIT);
if (!node.getPlugins().isEmpty()) {
processBuilder
.environment()
.put(
"BESU_OPTS",
"-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath().toString());
}
// Use non-blocking randomness for acceptance tests
processBuilder
.environment()
.put(
"JAVA_OPTS",
"-Djava.security.properties="
+ "acceptance-tests/tests/build/resources/test/acceptanceTesting.security");
// add additional environment variables
processBuilder.environment().putAll(node.getEnvironment());
try {
int debugPort = Integer.parseInt(System.getenv("BESU_DEBUG_CHILD_PROCESS_PORT"));
LOG.warn("Waiting for debugger to attach to SUSPENDED child process");
String debugOpts =
" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:" + debugPort;
String prevJavaOpts = processBuilder.environment().get("JAVA_OPTS");
if (prevJavaOpts == null) {
processBuilder.environment().put("JAVA_OPTS", debugOpts);
} else {
processBuilder.environment().put("JAVA_OPTS", prevJavaOpts + debugOpts);
}
} catch (NumberFormatException e) {
LOG.debug(
"Child process may be attached to by exporting BESU_DEBUG_CHILD_PROCESS_PORT=<port> to env");
}
try {
checkState(
isNotAliveOrphan(node.getName()),
"A live process with name: %s, already exists. Cannot create another with the same name as it would orphan the first",
node.getName());
final Process process = processBuilder.start();
process.onExit().thenRun(() -> node.setExitCode(process.exitValue()));
outputProcessorExecutor.execute(() -> printOutput(node, process));
besuProcesses.put(node.getName(), process);
} catch (final IOException e) {
LOG.error("Error starting BesuNode process", e);
}
if (node.getRunCommand().isEmpty()) {
waitForFile(dataDir, "besu.ports");
waitForFile(dataDir, "besu.networks");
}
MDC.remove("node");
}
private List<String> commandlineArgs(final BesuNode node, final Path dataDir) {
final List<String> params = new ArrayList<>(); final List<String> params = new ArrayList<>();
params.add("build/install/besu/bin/besu"); params.add("build/install/besu/bin/besu");
@ -388,66 +452,7 @@ public class ProcessBesuNodeRunner implements BesuNodeRunner {
} }
params.addAll(node.getRunCommand()); params.addAll(node.getRunCommand());
return params;
LOG.info("Creating besu process with params {}", params);
final ProcessBuilder processBuilder =
new ProcessBuilder(params)
.directory(new File(System.getProperty("user.dir")).getParentFile().getParentFile())
.redirectErrorStream(true)
.redirectInput(Redirect.INHERIT);
if (!node.getPlugins().isEmpty()) {
processBuilder
.environment()
.put(
"BESU_OPTS",
"-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath().toString());
}
// Use non-blocking randomness for acceptance tests
processBuilder
.environment()
.put(
"JAVA_OPTS",
"-Djava.security.properties="
+ "acceptance-tests/tests/build/resources/test/acceptanceTesting.security");
// add additional environment variables
processBuilder.environment().putAll(node.getEnvironment());
try {
int debugPort = Integer.parseInt(System.getenv("BESU_DEBUG_CHILD_PROCESS_PORT"));
LOG.warn("Waiting for debugger to attach to SUSPENDED child process");
String debugOpts =
" -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:" + debugPort;
String prevJavaOpts = processBuilder.environment().get("JAVA_OPTS");
if (prevJavaOpts == null) {
processBuilder.environment().put("JAVA_OPTS", debugOpts);
} else {
processBuilder.environment().put("JAVA_OPTS", prevJavaOpts + debugOpts);
}
} catch (NumberFormatException e) {
LOG.debug(
"Child process may be attached to by exporting BESU_DEBUG_CHILD_PROCESS_PORT=<port> to env");
}
try {
checkState(
isNotAliveOrphan(node.getName()),
"A live process with name: %s, already exists. Cannot create another with the same name as it would orphan the first",
node.getName());
final Process process = processBuilder.start();
process.onExit().thenRun(() -> node.setExitCode(process.exitValue()));
outputProcessorExecutor.execute(() -> printOutput(node, process));
besuProcesses.put(node.getName(), process);
} catch (final IOException e) {
LOG.error("Error starting BesuNode process", e);
}
if (node.getRunCommand().isEmpty()) {
waitForFile(dataDir, "besu.ports");
waitForFile(dataDir, "besu.networks");
}
MDC.remove("node");
} }
private boolean isNotAliveOrphan(final String name) { private boolean isNotAliveOrphan(final String name) {

@ -138,7 +138,8 @@ public class ThreadBesuNodeRunner implements BesuNodeRunner {
final PermissioningServiceImpl permissioningService = new PermissioningServiceImpl(); final PermissioningServiceImpl permissioningService = new PermissioningServiceImpl();
GlobalOpenTelemetry.resetForTest(); GlobalOpenTelemetry.resetForTest();
final ObservableMetricsSystem metricsSystem = component.getObservableMetricsSystem(); final ObservableMetricsSystem metricsSystem =
(ObservableMetricsSystem) component.getMetricsSystem();
final List<EnodeURL> bootnodes = final List<EnodeURL> bootnodes =
node.getConfiguration().getBootnodes().stream().map(EnodeURLImpl::fromURI).toList(); node.getConfiguration().getBootnodes().stream().map(EnodeURLImpl::fromURI).toList();
@ -280,6 +281,16 @@ public class ThreadBesuNodeRunner implements BesuNodeRunner {
this.toProvide = toProvide; this.toProvide = toProvide;
} }
@Provides
@Singleton
MetricsConfiguration provideMetricsConfiguration() {
if (toProvide.getMetricsConfiguration() != null) {
return toProvide.getMetricsConfiguration();
} else {
return MetricsConfiguration.builder().build();
}
}
@Provides @Provides
public BesuNode provideBesuNodeRunner() { public BesuNode provideBesuNodeRunner() {
return toProvide; return toProvide;
@ -410,13 +421,13 @@ public class ThreadBesuNodeRunner implements BesuNodeRunner {
public BesuController provideBesuController( public BesuController provideBesuController(
final SynchronizerConfiguration synchronizerConfiguration, final SynchronizerConfiguration synchronizerConfiguration,
final BesuControllerBuilder builder, final BesuControllerBuilder builder,
final ObservableMetricsSystem metricsSystem, final MetricsSystem metricsSystem,
final KeyValueStorageProvider storageProvider, final KeyValueStorageProvider storageProvider,
final MiningParameters miningParameters) { final MiningParameters miningParameters) {
builder builder
.synchronizerConfiguration(synchronizerConfiguration) .synchronizerConfiguration(synchronizerConfiguration)
.metricsSystem(metricsSystem) .metricsSystem((ObservableMetricsSystem) metricsSystem)
.dataStorageConfiguration(DataStorageConfiguration.DEFAULT_FOREST_CONFIG) .dataStorageConfiguration(DataStorageConfiguration.DEFAULT_FOREST_CONFIG)
.ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
.clock(Clock.systemUTC()) .clock(Clock.systemUTC())
@ -562,12 +573,6 @@ public class ThreadBesuNodeRunner implements BesuNodeRunner {
return besuCommand; return besuCommand;
} }
@Provides
@Singleton
MetricsConfiguration provideMetricsConfiguration() {
return MetricsConfiguration.builder().build();
}
@Provides @Provides
@Named("besuCommandLogger") @Named("besuCommandLogger")
@Singleton @Singleton

@ -16,6 +16,7 @@ package org.hyperledger.besu;
import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.cli.BesuCommand;
import org.hyperledger.besu.cli.logging.BesuLoggingConfigurationFactory; import org.hyperledger.besu.cli.logging.BesuLoggingConfigurationFactory;
import org.hyperledger.besu.components.BesuComponent;
import org.hyperledger.besu.components.DaggerBesuComponent; import org.hyperledger.besu.components.DaggerBesuComponent;
import io.netty.util.internal.logging.InternalLoggerFactory; import io.netty.util.internal.logging.InternalLoggerFactory;
@ -36,13 +37,15 @@ public final class Besu {
*/ */
public static void main(final String... args) { public static void main(final String... args) {
setupLogging(); setupLogging();
final BesuCommand besuCommand = DaggerBesuComponent.create().getBesuCommand(); final BesuComponent besuComponent = DaggerBesuComponent.create();
final BesuCommand besuCommand = besuComponent.getBesuCommand();
int exitCode = int exitCode =
besuCommand.parse( besuCommand.parse(
new RunLast(), new RunLast(),
besuCommand.parameterExceptionHandler(), besuCommand.parameterExceptionHandler(),
besuCommand.executionExceptionHandler(), besuCommand.executionExceptionHandler(),
System.in, System.in,
besuComponent,
args); args);
System.exit(exitCode); System.exit(exitCode);

@ -16,6 +16,7 @@ package org.hyperledger.besu.cli;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Preconditions.checkState;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList; import static java.util.Arrays.asList;
import static java.util.Collections.singletonList; import static java.util.Collections.singletonList;
import static org.hyperledger.besu.cli.DefaultCommandValues.getDefaultBesuDataPath; import static org.hyperledger.besu.cli.DefaultCommandValues.getDefaultBesuDataPath;
@ -84,6 +85,7 @@ import org.hyperledger.besu.cli.util.BesuCommandCustomFactory;
import org.hyperledger.besu.cli.util.CommandLineUtils; import org.hyperledger.besu.cli.util.CommandLineUtils;
import org.hyperledger.besu.cli.util.ConfigDefaultValueProviderStrategy; import org.hyperledger.besu.cli.util.ConfigDefaultValueProviderStrategy;
import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.cli.util.VersionProvider;
import org.hyperledger.besu.components.BesuComponent;
import org.hyperledger.besu.config.CheckpointConfigOptions; import org.hyperledger.besu.config.CheckpointConfigOptions;
import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.config.GenesisConfigFile;
import org.hyperledger.besu.config.GenesisConfigOptions; import org.hyperledger.besu.config.GenesisConfigOptions;
@ -144,7 +146,6 @@ import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract;
import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.BesuMetricCategory;
import org.hyperledger.besu.metrics.MetricCategoryRegistryImpl; import org.hyperledger.besu.metrics.MetricCategoryRegistryImpl;
import org.hyperledger.besu.metrics.MetricsProtocol; import org.hyperledger.besu.metrics.MetricsProtocol;
import org.hyperledger.besu.metrics.MetricsSystemFactory;
import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.metrics.StandardMetricCategory; import org.hyperledger.besu.metrics.StandardMetricCategory;
import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
@ -203,16 +204,23 @@ import org.hyperledger.besu.util.number.Fraction;
import org.hyperledger.besu.util.number.Percentage; import org.hyperledger.besu.util.number.Percentage;
import org.hyperledger.besu.util.number.PositiveNumber; import org.hyperledger.besu.util.number.PositiveNumber;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigInteger; import java.math.BigInteger;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.SocketException; import java.net.SocketException;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.net.URL;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.attribute.GroupPrincipal;
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.UserPrincipal;
import java.time.Clock; import java.time.Clock;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -232,6 +240,7 @@ import java.util.function.Supplier;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.common.base.Suppliers; import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
@ -243,6 +252,8 @@ import org.apache.commons.net.util.SubnetUtils.SubnetInfo;
import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.units.bigints.UInt256; import org.apache.tuweni.units.bigints.UInt256;
import org.slf4j.Logger; import org.slf4j.Logger;
import oshi.PlatformEnum;
import oshi.SystemInfo;
import picocli.AutoComplete; import picocli.AutoComplete;
import picocli.CommandLine; import picocli.CommandLine;
import picocli.CommandLine.Command; import picocli.CommandLine.Command;
@ -382,6 +393,28 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
arity = "1") arity = "1")
private final Optional<String> identityString = Optional.empty(); private final Optional<String> identityString = Optional.empty();
private Boolean printPathsAndExit = Boolean.FALSE;
private String besuUserName = "besu";
@Option(
names = "--print-paths-and-exit",
paramLabel = "<username>",
description = "Print the configured paths and exit without starting the node.",
arity = "0..1")
void setUserName(final String userName) {
PlatformEnum currentPlatform = SystemInfo.getCurrentPlatform();
// Only allow on Linux and macOS
if (currentPlatform == PlatformEnum.LINUX || currentPlatform == PlatformEnum.MACOS) {
if (userName != null) {
besuUserName = userName;
}
printPathsAndExit = Boolean.TRUE;
} else {
throw new UnsupportedOperationException(
"--print-paths-and-exit is only supported on Linux and macOS.");
}
}
// P2P Discovery Option Group // P2P Discovery Option Group
@CommandLine.ArgGroup(validate = false, heading = "@|bold P2P Discovery Options|@%n") @CommandLine.ArgGroup(validate = false, heading = "@|bold P2P Discovery Options|@%n")
P2PDiscoveryOptionGroup p2PDiscoveryOptionGroup = new P2PDiscoveryOptionGroup(); P2PDiscoveryOptionGroup p2PDiscoveryOptionGroup = new P2PDiscoveryOptionGroup();
@ -390,6 +423,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
private final TransactionPoolValidatorServiceImpl transactionValidatorServiceImpl; private final TransactionPoolValidatorServiceImpl transactionValidatorServiceImpl;
private final TransactionSimulationServiceImpl transactionSimulationServiceImpl; private final TransactionSimulationServiceImpl transactionSimulationServiceImpl;
private final BlockchainServiceImpl blockchainServiceImpl; private final BlockchainServiceImpl blockchainServiceImpl;
private BesuComponent besuComponent;
static class P2PDiscoveryOptionGroup { static class P2PDiscoveryOptionGroup {
@ -864,9 +898,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
private BesuController besuController; private BesuController besuController;
private BesuConfigurationImpl pluginCommonConfiguration; private BesuConfigurationImpl pluginCommonConfiguration;
private final Supplier<ObservableMetricsSystem> metricsSystem =
Suppliers.memoize(() -> MetricsSystemFactory.create(metricsConfiguration()));
private Vertx vertx; private Vertx vertx;
private EnodeDnsConfiguration enodeDnsConfiguration; private EnodeDnsConfiguration enodeDnsConfiguration;
private KeyValueStorageProvider keyValueStorageProvider; private KeyValueStorageProvider keyValueStorageProvider;
@ -996,6 +1027,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
* @param parameterExceptionHandler Handler for exceptions related to command line parameters. * @param parameterExceptionHandler Handler for exceptions related to command line parameters.
* @param executionExceptionHandler Handler for exceptions during command execution. * @param executionExceptionHandler Handler for exceptions during command execution.
* @param in The input stream for commands. * @param in The input stream for commands.
* @param besuComponent The Besu component.
* @param args The command line arguments. * @param args The command line arguments.
* @return The execution result status code. * @return The execution result status code.
*/ */
@ -1004,8 +1036,12 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
final BesuParameterExceptionHandler parameterExceptionHandler, final BesuParameterExceptionHandler parameterExceptionHandler,
final BesuExecutionExceptionHandler executionExceptionHandler, final BesuExecutionExceptionHandler executionExceptionHandler,
final InputStream in, final InputStream in,
final BesuComponent besuComponent,
final String... args) { final String... args) {
if (besuComponent == null) {
throw new IllegalArgumentException("BesuComponent must be provided");
}
this.besuComponent = besuComponent;
initializeCommandLineSettings(in); initializeCommandLineSettings(in);
// Create the execution strategy chain. // Create the execution strategy chain.
@ -1093,6 +1129,12 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
try { try {
configureLogging(true); configureLogging(true);
if (printPathsAndExit) {
// Print configured paths requiring read/write permissions to be adjusted
checkPermissionsAndPrintPaths(besuUserName);
System.exit(0); // Exit before any services are started
}
// set merge config on the basis of genesis config // set merge config on the basis of genesis config
setMergeConfigOptions(); setMergeConfigOptions();
@ -1103,7 +1145,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
logger.info("Starting Besu"); logger.info("Starting Besu");
// Need to create vertx after cmdline has been parsed, such that metricsSystem is configurable // Need to create vertx after cmdline has been parsed, such that metricsSystem is configurable
vertx = createVertx(createVertxOptions(metricsSystem.get())); vertx = createVertx(createVertxOptions(besuComponent.getMetricsSystem()));
validateOptions(); validateOptions();
@ -1138,6 +1180,104 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
} }
} }
private void checkPermissionsAndPrintPaths(final String userName) {
// Check permissions for the data path
checkPermissions(dataDir(), userName, false);
// Check permissions for genesis file
try {
if (genesisFile != null) {
checkPermissions(genesisFile.toPath(), userName, true);
}
} catch (Exception e) {
commandLine
.getOut()
.println("Error: Failed checking genesis file: Reason: " + e.getMessage());
}
}
// Helper method to check permissions on a given path
private void checkPermissions(final Path path, final String besuUser, final boolean readOnly) {
try {
// Get the permissions of the file
// check if besu user is the owner - get owner permissions if yes
// else, check if besu user and owner are in the same group - if yes, check the group
// permission
// otherwise check permissions for others
// Get the owner of the file or directory
UserPrincipal owner = Files.getOwner(path);
boolean hasReadPermission, hasWritePermission;
// Get file permissions
Set<PosixFilePermission> permissions = Files.getPosixFilePermissions(path);
// Check if besu is the owner
if (owner.getName().equals(besuUser)) {
// Owner permissions
hasReadPermission = permissions.contains(PosixFilePermission.OWNER_READ);
hasWritePermission = permissions.contains(PosixFilePermission.OWNER_WRITE);
} else {
// Get the group of the file
// Get POSIX file attributes and then group
PosixFileAttributes attrs = Files.readAttributes(path, PosixFileAttributes.class);
GroupPrincipal group = attrs.group();
// Check if besu user belongs to this group
boolean isMember = isGroupMember(besuUserName, group);
if (isMember) {
// Group's permissions
hasReadPermission = permissions.contains(PosixFilePermission.GROUP_READ);
hasWritePermission = permissions.contains(PosixFilePermission.GROUP_WRITE);
} else {
// Others' permissions
hasReadPermission = permissions.contains(PosixFilePermission.OTHERS_READ);
hasWritePermission = permissions.contains(PosixFilePermission.OTHERS_WRITE);
}
}
if (!hasReadPermission || (!readOnly && !hasWritePermission)) {
String accessType = readOnly ? "READ" : "READ_WRITE";
commandLine.getOut().println("PERMISSION_CHECK_PATH:" + path + ":" + accessType);
}
} catch (Exception e) {
// Do nothing upon catching an error
commandLine
.getOut()
.println(
"Error: Failed to check permissions for path: '"
+ path
+ "'. Reason: "
+ e.getMessage());
}
}
private static boolean isGroupMember(final String userName, final GroupPrincipal group)
throws IOException {
// Get the groups of the user by executing 'id -Gn username'
Process process = Runtime.getRuntime().exec(new String[] {"id", "-Gn", userName});
BufferedReader reader =
new BufferedReader(new InputStreamReader(process.getInputStream(), UTF_8));
// Read the output of the command
String line = reader.readLine();
boolean isMember = false;
if (line != null) {
// Split the groups
Iterable<String> userGroups = Splitter.on(" ").split(line);
// Check if any of the user's groups match the file's group
for (String grp : userGroups) {
if (grp.equals(group.getName())) {
isMember = true;
break;
}
}
}
return isMember;
}
@VisibleForTesting @VisibleForTesting
void setBesuConfiguration(final BesuConfigurationImpl pluginCommonConfiguration) { void setBesuConfiguration(final BesuConfigurationImpl pluginCommonConfiguration) {
this.pluginCommonConfiguration = pluginCommonConfiguration; this.pluginCommonConfiguration = pluginCommonConfiguration;
@ -1390,8 +1530,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
} }
private void setReleaseMetrics() { private void setReleaseMetrics() {
metricsSystem besuComponent
.get() .getMetricsSystem()
.createLabelledGauge( .createLabelledGauge(
StandardMetricCategory.PROCESS, "release", "Release information", "version") StandardMetricCategory.PROCESS, "release", "Release information", "version")
.labels(() -> 1, BesuInfo.version()); .labels(() -> 1, BesuInfo.version());
@ -1855,7 +1995,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
.miningParameters(miningParametersSupplier.get()) .miningParameters(miningParametersSupplier.get())
.transactionPoolConfiguration(buildTransactionPoolConfiguration()) .transactionPoolConfiguration(buildTransactionPoolConfiguration())
.nodeKey(new NodeKey(securityModule())) .nodeKey(new NodeKey(securityModule()))
.metricsSystem(metricsSystem.get()) .metricsSystem((ObservableMetricsSystem) besuComponent.getMetricsSystem())
.messagePermissioningProviders(permissioningService.getMessagePermissioningProviders()) .messagePermissioningProviders(permissioningService.getMessagePermissioningProviders())
.privacyParameters(privacyParameters()) .privacyParameters(privacyParameters())
.clock(Clock.systemUTC()) .clock(Clock.systemUTC())
@ -1875,7 +2015,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
.randomPeerPriority(p2PDiscoveryOptionGroup.randomPeerPriority) .randomPeerPriority(p2PDiscoveryOptionGroup.randomPeerPriority)
.chainPruningConfiguration(unstableChainPruningOptions.toDomainObject()) .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject())
.cacheLastBlocks(numberOfblocksToCache) .cacheLastBlocks(numberOfblocksToCache)
.genesisStateHashCacheEnabled(genesisStateHashCacheEnabled); .genesisStateHashCacheEnabled(genesisStateHashCacheEnabled)
.besuComponent(besuComponent);
} }
private JsonRpcConfiguration createEngineJsonRpcConfiguration( private JsonRpcConfiguration createEngineJsonRpcConfiguration(
@ -2277,7 +2418,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
p2pTLSConfiguration.ifPresent(runnerBuilder::p2pTLSConfiguration); p2pTLSConfiguration.ifPresent(runnerBuilder::p2pTLSConfiguration);
final ObservableMetricsSystem metricsSystem = this.metricsSystem.get();
final Runner runner = final Runner runner =
runnerBuilder runnerBuilder
.vertx(vertx) .vertx(vertx)
@ -2304,7 +2444,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
.pidPath(pidPath) .pidPath(pidPath)
.dataDir(dataDir()) .dataDir(dataDir())
.bannedNodeIds(p2PDiscoveryOptionGroup.bannedNodeIds) .bannedNodeIds(p2PDiscoveryOptionGroup.bannedNodeIds)
.metricsSystem(metricsSystem) .metricsSystem((ObservableMetricsSystem) besuComponent.getMetricsSystem())
.permissioningService(permissioningService) .permissioningService(permissioningService)
.metricsConfiguration(metricsConfiguration) .metricsConfiguration(metricsConfiguration)
.staticNodes(staticNodes) .staticNodes(staticNodes)
@ -2471,7 +2611,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
* @return Instance of MetricsSystem * @return Instance of MetricsSystem
*/ */
public MetricsSystem getMetricsSystem() { public MetricsSystem getMetricsSystem() {
return metricsSystem.get(); return besuComponent.getMetricsSystem();
} }
private Set<EnodeURL> loadStaticNodes() throws IOException { private Set<EnodeURL> loadStaticNodes() throws IOException {
@ -2809,4 +2949,13 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
return builder.build(); return builder.build();
} }
/**
* Returns the plugin context.
*
* @return the plugin context.
*/
public BesuPluginContextImpl getBesuPluginContext() {
return besuPluginContext;
}
} }

@ -42,9 +42,7 @@ public class BesuCommandModule {
@Provides @Provides
@Singleton @Singleton
BesuCommand provideBesuCommand( BesuCommand provideBesuCommand(final @Named("besuCommandLogger") Logger commandLogger) {
final BesuPluginContextImpl pluginContext,
final @Named("besuCommandLogger") Logger commandLogger) {
final BesuCommand besuCommand = final BesuCommand besuCommand =
new BesuCommand( new BesuCommand(
RlpBlockImporter::new, RlpBlockImporter::new,
@ -52,7 +50,7 @@ public class BesuCommandModule {
RlpBlockExporter::new, RlpBlockExporter::new,
new RunnerBuilder(), new RunnerBuilder(),
new BesuController.Builder(), new BesuController.Builder(),
pluginContext, new BesuPluginContextImpl(),
System.getenv(), System.getenv(),
commandLogger); commandLogger);
besuCommand.toCommandLine(); besuCommand.toCommandLine();
@ -71,4 +69,10 @@ public class BesuCommandModule {
Logger provideBesuCommandLogger() { Logger provideBesuCommandLogger() {
return Besu.getFirstLogger(); return Besu.getFirstLogger();
} }
@Provides
@Singleton
BesuPluginContextImpl provideBesuPluginContextImpl(final BesuCommand provideFrom) {
return provideFrom.getBesuPluginContext();
}
} }

@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.eth.transactions.BlobCacheModule;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoaderModule; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoaderModule;
import org.hyperledger.besu.metrics.MetricsSystemModule; import org.hyperledger.besu.metrics.MetricsSystemModule;
import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.services.BesuPluginContextImpl; import org.hyperledger.besu.services.BesuPluginContextImpl;
import javax.inject.Named; import javax.inject.Named;
@ -60,7 +60,7 @@ public interface BesuComponent {
* *
* @return ObservableMetricsSystem * @return ObservableMetricsSystem
*/ */
ObservableMetricsSystem getObservableMetricsSystem(); MetricsSystem getMetricsSystem();
/** /**
* a Logger specifically configured to provide configuration feedback to users. * a Logger specifically configured to provide configuration feedback to users.

@ -14,9 +14,7 @@
*/ */
package org.hyperledger.besu.components; package org.hyperledger.besu.components;
import org.hyperledger.besu.plugin.services.BesuConfiguration;
import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuConfigurationImpl;
import org.hyperledger.besu.services.BesuPluginContextImpl;
import javax.inject.Singleton; import javax.inject.Singleton;
@ -35,18 +33,4 @@ public class BesuPluginContextModule {
BesuConfigurationImpl provideBesuPluginConfig() { BesuConfigurationImpl provideBesuPluginConfig() {
return new BesuConfigurationImpl(); return new BesuConfigurationImpl();
} }
/**
* Creates a BesuPluginContextImpl, used for plugin service discovery.
*
* @param pluginConfig the BesuConfigurationImpl
* @return the BesuPluginContext
*/
@Provides
@Singleton
public BesuPluginContextImpl provideBesuPluginContext(final BesuConfigurationImpl pluginConfig) {
BesuPluginContextImpl retval = new BesuPluginContextImpl();
retval.addService(BesuConfiguration.class, pluginConfig);
return retval;
}
} }

@ -552,7 +552,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
checkNotNull(evmConfiguration, "Missing evm config"); checkNotNull(evmConfiguration, "Missing evm config");
checkNotNull(networkingConfiguration, "Missing network configuration"); checkNotNull(networkingConfiguration, "Missing network configuration");
checkNotNull(dataStorageConfiguration, "Missing data storage configuration"); checkNotNull(dataStorageConfiguration, "Missing data storage configuration");
checkNotNull(besuComponent, "Must supply a BesuComponent");
prepForBuild(); prepForBuild();
final ProtocolSchedule protocolSchedule = createProtocolSchedule(); final ProtocolSchedule protocolSchedule = createProtocolSchedule();

@ -0,0 +1,49 @@
#!/bin/bash
##
## Copyright contributors to Hyperledger Besu.
##
## Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
## the License. You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
## an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
## specific language governing permissions and limitations under the License.
##
## SPDX-License-Identifier: Apache-2.0
##
# Run Besu first to get paths needing permission adjustment
output=$(/opt/besu/bin/besu --print-paths-and-exit $BESU_USER_NAME "$@")
# Parse the output to find the paths and their required access types
echo "$output" | while IFS=: read -r prefix path accessType; do
if [[ "$prefix" == "PERMISSION_CHECK_PATH" ]]; then
# Change ownership to besu user and group
chown -R $BESU_USER_NAME:$BESU_USER_NAME $path
# Ensure read/write permissions for besu user
echo "Setting permissions for: $path with access: $accessType"
if [[ "$accessType" == "READ" ]]; then
# Set read-only permissions for besu user
# Add execute for directories to allow access
find $path -type d -exec chmod u+rx {} \;
find $path -type f -exec chmod u+r {} \;
elif [[ "$accessType" == "READ_WRITE" ]]; then
# Set read/write permissions for besu user
# Add execute for directories to allow access
find $path -type d -exec chmod u+rwx {} \;
find $path -type f -exec chmod u+rw {} \;
fi
fi
done
# Finally, run Besu with the actual arguments passed to the container
# Construct the command as a single string
COMMAND="/opt/besu/bin/besu $@"
# Switch to the besu user and execute the command
exec su -s /bin/bash $BESU_USER_NAME -c "$COMMAND"

@ -390,6 +390,7 @@ public class BesuCommandTest extends CommandTestAbstract {
options.remove(spec.optionsMap().get("--config-file")); options.remove(spec.optionsMap().get("--config-file"));
options.remove(spec.optionsMap().get("--help")); options.remove(spec.optionsMap().get("--help"));
options.remove(spec.optionsMap().get("--version")); options.remove(spec.optionsMap().get("--version"));
options.remove(spec.optionsMap().get("--print-paths-and-exit"));
for (final String tomlKey : tomlResult.keySet()) { for (final String tomlKey : tomlResult.keySet()) {
final CommandLine.Model.OptionSpec optionSpec = spec.optionsMap().get("--" + tomlKey); final CommandLine.Model.OptionSpec optionSpec = spec.optionsMap().get("--" + tomlKey);

@ -42,6 +42,7 @@ import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
import org.hyperledger.besu.cli.options.unstable.MetricsCLIOptions; import org.hyperledger.besu.cli.options.unstable.MetricsCLIOptions;
import org.hyperledger.besu.cli.options.unstable.NetworkingOptions; import org.hyperledger.besu.cli.options.unstable.NetworkingOptions;
import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions; import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions;
import org.hyperledger.besu.components.BesuComponent;
import org.hyperledger.besu.config.GenesisConfigOptions; import org.hyperledger.besu.config.GenesisConfigOptions;
import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.controller.BesuController;
import org.hyperledger.besu.controller.BesuControllerBuilder; import org.hyperledger.besu.controller.BesuControllerBuilder;
@ -69,6 +70,7 @@ import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
import org.hyperledger.besu.plugin.services.PicoCLIOptions; import org.hyperledger.besu.plugin.services.PicoCLIOptions;
import org.hyperledger.besu.plugin.services.StorageService; import org.hyperledger.besu.plugin.services.StorageService;
@ -204,6 +206,9 @@ public abstract class CommandTestAbstract {
@Mock(lenient = true) @Mock(lenient = true)
protected BesuController mockController; protected BesuController mockController;
@Mock(lenient = true)
protected BesuComponent mockBesuComponent;
@Mock protected RlpBlockExporter rlpBlockExporter; @Mock protected RlpBlockExporter rlpBlockExporter;
@Mock protected JsonBlockImporter jsonBlockImporter; @Mock protected JsonBlockImporter jsonBlockImporter;
@Mock protected RlpBlockImporter rlpBlockImporter; @Mock protected RlpBlockImporter rlpBlockImporter;
@ -344,6 +349,7 @@ public abstract class CommandTestAbstract {
when(mockRunnerBuilder.allowedSubnets(any())).thenReturn(mockRunnerBuilder); when(mockRunnerBuilder.allowedSubnets(any())).thenReturn(mockRunnerBuilder);
when(mockRunnerBuilder.poaDiscoveryRetryBootnodes(anyBoolean())).thenReturn(mockRunnerBuilder); when(mockRunnerBuilder.poaDiscoveryRetryBootnodes(anyBoolean())).thenReturn(mockRunnerBuilder);
when(mockRunnerBuilder.build()).thenReturn(mockRunner); when(mockRunnerBuilder.build()).thenReturn(mockRunner);
when(mockBesuComponent.getMetricsSystem()).thenReturn(new NoOpMetricsSystem());
final SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance(); final SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance();
@ -451,6 +457,7 @@ public abstract class CommandTestAbstract {
besuCommand.parameterExceptionHandler(), besuCommand.parameterExceptionHandler(),
besuCommand.executionExceptionHandler(), besuCommand.executionExceptionHandler(),
in, in,
mockBesuComponent,
args); args);
return besuCommand; return besuCommand;
} }

@ -18,6 +18,9 @@ import static org.mockito.Mockito.mock;
import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.cli.BesuCommand;
import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
import org.hyperledger.besu.plugin.services.BesuConfiguration;
import org.hyperledger.besu.services.BesuConfigurationImpl;
import org.hyperledger.besu.services.BesuPluginContextImpl;
import javax.inject.Named; import javax.inject.Named;
import javax.inject.Singleton; import javax.inject.Singleton;
@ -47,4 +50,17 @@ public class MockBesuCommandModule {
Logger provideBesuCommandLogger() { Logger provideBesuCommandLogger() {
return LoggerFactory.getLogger(MockBesuCommandModule.class); return LoggerFactory.getLogger(MockBesuCommandModule.class);
} }
/**
* Creates a BesuPluginContextImpl, used for plugin service discovery.
*
* @return the BesuPluginContext
*/
@Provides
@Singleton
public BesuPluginContextImpl provideBesuPluginContext() {
BesuPluginContextImpl retval = new BesuPluginContextImpl();
retval.addService(BesuConfiguration.class, new BesuConfigurationImpl());
return retval;
}
} }

@ -1097,6 +1097,7 @@ distributions {
from("build/reports/license/license-dependency.html") { into "." } from("build/reports/license/license-dependency.html") { into "." }
from("./docs/GettingStartedBinaries.md") { into "." } from("./docs/GettingStartedBinaries.md") { into "." }
from("./docs/DocsArchive0.8.0.html") { into "." } from("./docs/DocsArchive0.8.0.html") { into "." }
from("./besu/src/main/scripts/besu-entry.sh") { into "./bin/" }
from(autocomplete) { into "." } from(autocomplete) { into "." }
} }
} }

@ -18,7 +18,8 @@ RUN apt-get update $NO_PROXY_CACHE && \
chown besu:besu /opt/besu && \ chown besu:besu /opt/besu && \
chmod 0755 /opt/besu chmod 0755 /opt/besu
USER besu ARG BESU_USER=besu
USER ${BESU_USER}
WORKDIR /opt/besu WORKDIR /opt/besu
COPY --chown=besu:besu besu /opt/besu/ COPY --chown=besu:besu besu /opt/besu/
@ -43,7 +44,18 @@ ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu,service.version=$VERSION"
ENV OLDPATH="${PATH}" ENV OLDPATH="${PATH}"
ENV PATH="/opt/besu/bin:${OLDPATH}" ENV PATH="/opt/besu/bin:${OLDPATH}"
ENTRYPOINT ["besu"]
# The entry script just sets permissions as needed based on besu config
# and is replaced by the besu process running as besu user.
# Suppressing this warning as there's no risk here because the root user
# only sets permissions and does not continue running the main process.
# hadolint ignore=DL3002
USER root
RUN chmod +x /opt/besu/bin/besu-entry.sh
ENV BESU_USER_NAME=${BESU_USER}
ENTRYPOINT ["besu-entry.sh"]
HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]" HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]"
# Build-time metadata as defined at http://label-schema.org # Build-time metadata as defined at http://label-schema.org

@ -41,4 +41,12 @@ bash $TEST_PATH/dgoss run --sysctl net.ipv6.conf.all.disable_ipv6=1 $DOCKER_IMAG
--graphql-http-enabled \ --graphql-http-enabled \
> ./reports/01.xml || i=`expr $i + 1` > ./reports/01.xml || i=`expr $i + 1`
if [[ $i != 0 ]]; then exit $i; fi
# Test for directory permissions
GOSS_FILES_PATH=$TEST_PATH/02 \
bash $TEST_PATH/dgoss run --sysctl net.ipv6.conf.all.disable_ipv6=1 -v besu-data:/var/lib/besu $DOCKER_IMAGE --data-path=/var/lib/besu \
--network=dev \
> ./reports/02.xml || i=`expr $i + 1`
exit $i exit $i

@ -0,0 +1,10 @@
---
# runtime docker tests
file:
/var/lib/besu:
exists: true
owner: besu
mode: "0755"
process:
java:
running: true

@ -76,7 +76,7 @@ GOSS_PATH="${GOSS_PATH:-$(which goss 2> /dev/null || true)}"
[[ $GOSS_PATH ]] || { error "Couldn't find goss installation, please set GOSS_PATH to it"; } [[ $GOSS_PATH ]] || { error "Couldn't find goss installation, please set GOSS_PATH to it"; }
[[ ${GOSS_OPTS+x} ]] || GOSS_OPTS="--color --format documentation" [[ ${GOSS_OPTS+x} ]] || GOSS_OPTS="--color --format documentation"
[[ ${GOSS_WAIT_OPTS+x} ]] || GOSS_WAIT_OPTS="-r 30s -s 1s > /dev/null" [[ ${GOSS_WAIT_OPTS+x} ]] || GOSS_WAIT_OPTS="-r 30s -s 1s > /dev/null"
GOSS_SLEEP=${GOSS_SLEEP:-0.2} GOSS_SLEEP=${GOSS_SLEEP:-1.0}
case "$1" in case "$1" in
run) run)

@ -401,7 +401,7 @@ public class BlockTransactionSelector {
LOG.atTrace() LOG.atTrace()
.setMessage("Selected {} for block creation, evaluated in {}") .setMessage("Selected {} for block creation, evaluated in {}")
.addArgument(transaction::toTraceLog) .addArgument(transaction::toTraceLog)
.addArgument(evaluationContext.getPendingTransaction()) .addArgument(evaluationContext.getEvaluationTimer())
.log(); .log();
return SELECTED; return SELECTED;
} }

@ -15,6 +15,7 @@
package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache;
import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import dagger.Module; import dagger.Module;
import dagger.Provides; import dagger.Provides;
@ -24,7 +25,7 @@ public class BonsaiCachedMerkleTrieLoaderModule {
@Provides @Provides
BonsaiCachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( BonsaiCachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule(
final ObservableMetricsSystem metricsSystem) { final MetricsSystem metricsSystem) {
return new BonsaiCachedMerkleTrieLoader(metricsSystem); return new BonsaiCachedMerkleTrieLoader((ObservableMetricsSystem) metricsSystem);
} }
} }

@ -85,6 +85,13 @@ public class BlockchainSetupUtil {
return blockchain; return blockchain;
} }
public Blockchain importAllBlocks(
final HeaderValidationMode headerValidationMode,
final HeaderValidationMode ommerValidationMode) {
importBlocks(blocks, headerValidationMode, ommerValidationMode);
return blockchain;
}
public void importFirstBlocks(final int count) { public void importFirstBlocks(final int count) {
importBlocks(blocks.subList(0, count)); importBlocks(blocks.subList(0, count));
} }
@ -126,6 +133,10 @@ public class BlockchainSetupUtil {
return createForEthashChain(BlockTestUtil.getUpgradedForkResources(), DataStorageFormat.FOREST); return createForEthashChain(BlockTestUtil.getUpgradedForkResources(), DataStorageFormat.FOREST);
} }
public static BlockchainSetupUtil forSnapTesting(final DataStorageFormat storageFormat) {
return createForEthashChain(BlockTestUtil.getSnapTestChainResources(), storageFormat);
}
public static BlockchainSetupUtil createForEthashChain( public static BlockchainSetupUtil createForEthashChain(
final ChainResources chainResources, final DataStorageFormat storageFormat) { final ChainResources chainResources, final DataStorageFormat storageFormat) {
return create( return create(
@ -241,6 +252,13 @@ public class BlockchainSetupUtil {
} }
private void importBlocks(final List<Block> blocks) { private void importBlocks(final List<Block> blocks) {
importBlocks(blocks, HeaderValidationMode.FULL, HeaderValidationMode.FULL);
}
private void importBlocks(
final List<Block> blocks,
final HeaderValidationMode headerValidationMode,
final HeaderValidationMode ommerValidationMode) {
for (final Block block : blocks) { for (final Block block : blocks) {
if (block.getHeader().getNumber() == BlockHeader.GENESIS_BLOCK_NUMBER) { if (block.getHeader().getNumber() == BlockHeader.GENESIS_BLOCK_NUMBER) {
continue; continue;
@ -248,7 +266,8 @@ public class BlockchainSetupUtil {
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader()); final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
final BlockImporter blockImporter = protocolSpec.getBlockImporter(); final BlockImporter blockImporter = protocolSpec.getBlockImporter();
final BlockImportResult result = final BlockImportResult result =
blockImporter.importBlock(protocolContext, block, HeaderValidationMode.FULL); blockImporter.importBlock(
protocolContext, block, headerValidationMode, ommerValidationMode);
if (!result.isImported()) { if (!result.isImported()) {
throw new IllegalStateException("Unable to import block " + block.getHeader().getNumber()); throw new IllegalStateException("Unable to import block " + block.getHeader().getNumber());
} }

@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration;
import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData;
import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.CompactEncoding;
import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrie;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider;
@ -240,12 +241,9 @@ class SnapServer implements BesuEvents.InitialSyncCompletionListener {
stopWatch, stopWatch,
maxResponseBytes, maxResponseBytes,
(pair) -> { (pair) -> {
var rlpOutput = new BytesValueRLPOutput(); Bytes bytes =
rlpOutput.startList(); AccountRangeMessage.toSlimAccount(RLP.input(pair.getSecond()));
rlpOutput.writeBytes(pair.getFirst()); return Hash.SIZE + bytes.size();
rlpOutput.writeRLPBytes(pair.getSecond());
rlpOutput.endList();
return rlpOutput.encodedSize();
}); });
final Bytes32 endKeyBytes = range.endKeyHash(); final Bytes32 endKeyBytes = range.endKeyHash();
@ -257,11 +255,15 @@ class SnapServer implements BesuEvents.InitialSyncCompletionListener {
storage.streamFlatAccounts(range.startKeyHash(), shouldContinuePredicate); storage.streamFlatAccounts(range.startKeyHash(), shouldContinuePredicate);
if (accounts.isEmpty() && shouldContinuePredicate.shouldContinue.get()) { if (accounts.isEmpty() && shouldContinuePredicate.shouldContinue.get()) {
var fromNextHash =
range.endKeyHash().compareTo(range.startKeyHash()) >= 0
? range.endKeyHash()
: range.startKeyHash();
// fetch next account after range, if it exists // fetch next account after range, if it exists
LOGGER.debug( LOGGER.debug(
"found no accounts in range, taking first value starting from {}", "found no accounts in range, taking first value starting from {}",
asLogHash(range.endKeyHash())); asLogHash(fromNextHash));
accounts = storage.streamFlatAccounts(range.endKeyHash(), UInt256.MAX_VALUE, 1L); accounts = storage.streamFlatAccounts(fromNextHash, UInt256.MAX_VALUE, 1L);
} }
final var worldStateProof = final var worldStateProof =

@ -14,6 +14,7 @@
*/ */
package org.hyperledger.besu.ethereum.eth.messages.snap; package org.hyperledger.besu.ethereum.eth.messages.snap;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData;
import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
@ -28,6 +29,7 @@ import java.util.NavigableMap;
import java.util.Optional; import java.util.Optional;
import java.util.TreeMap; import java.util.TreeMap;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import kotlin.collections.ArrayDeque; import kotlin.collections.ArrayDeque;
import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes;
@ -117,7 +119,8 @@ public final class AccountRangeMessage extends AbstractSnapMessageData {
return ImmutableAccountRangeData.builder().accounts(accounts).proofs(proofs).build(); return ImmutableAccountRangeData.builder().accounts(accounts).proofs(proofs).build();
} }
private Bytes toFullAccount(final RLPInput rlpInput) { @VisibleForTesting
public static Bytes toFullAccount(final RLPInput rlpInput) {
final StateTrieAccountValue accountValue = StateTrieAccountValue.readFrom(rlpInput); final StateTrieAccountValue accountValue = StateTrieAccountValue.readFrom(rlpInput);
final BytesValueRLPOutput rlpOutput = new BytesValueRLPOutput(); final BytesValueRLPOutput rlpOutput = new BytesValueRLPOutput();
@ -131,6 +134,26 @@ public final class AccountRangeMessage extends AbstractSnapMessageData {
return rlpOutput.encoded(); return rlpOutput.encoded();
} }
public static Bytes toSlimAccount(final RLPInput rlpInput) {
StateTrieAccountValue accountValue = StateTrieAccountValue.readFrom(rlpInput);
var rlpOutput = new BytesValueRLPOutput();
rlpOutput.startList();
rlpOutput.writeLongScalar(accountValue.getNonce());
rlpOutput.writeUInt256Scalar(accountValue.getBalance());
if (accountValue.getStorageRoot().equals(Hash.EMPTY_TRIE_HASH)) {
rlpOutput.writeNull();
} else {
rlpOutput.writeBytes(accountValue.getStorageRoot());
}
if (accountValue.getCodeHash().equals(Hash.EMPTY)) {
rlpOutput.writeNull();
} else {
rlpOutput.writeBytes(accountValue.getCodeHash());
}
rlpOutput.endList();
return rlpOutput.encoded();
}
@Value.Immutable @Value.Immutable
public interface AccountRangeData { public interface AccountRangeData {

@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.rlp.RLPInput;
import java.math.BigInteger; import java.math.BigInteger;
import java.util.Optional; import java.util.Optional;
import com.google.common.annotations.VisibleForTesting;
import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.bytes.Bytes32;
import org.immutables.value.Value; import org.immutables.value.Value;
@ -48,12 +49,21 @@ public final class GetAccountRangeMessage extends AbstractSnapMessageData {
public static GetAccountRangeMessage create( public static GetAccountRangeMessage create(
final Hash worldStateRootHash, final Bytes32 startKeyHash, final Bytes32 endKeyHash) { final Hash worldStateRootHash, final Bytes32 startKeyHash, final Bytes32 endKeyHash) {
return create(worldStateRootHash, startKeyHash, endKeyHash, SIZE_REQUEST);
}
@VisibleForTesting
public static GetAccountRangeMessage create(
final Hash worldStateRootHash,
final Bytes32 startKeyHash,
final Bytes32 endKeyHash,
final BigInteger sizeRequest) {
final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); final BytesValueRLPOutput tmp = new BytesValueRLPOutput();
tmp.startList(); tmp.startList();
tmp.writeBytes(worldStateRootHash); tmp.writeBytes(worldStateRootHash);
tmp.writeBytes(startKeyHash); tmp.writeBytes(startKeyHash);
tmp.writeBytes(endKeyHash); tmp.writeBytes(endKeyHash);
tmp.writeBigIntegerScalar(SIZE_REQUEST); tmp.writeBigIntegerScalar(sizeRequest);
tmp.endList(); tmp.endList();
return new GetAccountRangeMessage(tmp.encoded()); return new GetAccountRangeMessage(tmp.encoded());
} }

@ -0,0 +1,495 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.eth.manager.snap;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
import org.hyperledger.besu.ethereum.eth.manager.EthMessages;
import org.hyperledger.besu.ethereum.eth.messages.snap.AccountRangeMessage;
import org.hyperledger.besu.ethereum.eth.messages.snap.GetAccountRangeMessage;
import org.hyperledger.besu.ethereum.eth.sync.snapsync.ImmutableSnapSyncConfiguration;
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration;
import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode;
import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import java.math.BigInteger;
import java.util.NavigableMap;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
public class SnapServerGetAccountRangeTest {
private Hash rootHash;
public Bytes32 firstAccount;
public Bytes32 secondAccount;
private SnapServer snapServer;
private static ProtocolContext protocolContext;
@BeforeAll
public static void setup() {
// Setup local blockchain for testing
BlockchainSetupUtil localBlockchainSetup =
BlockchainSetupUtil.forSnapTesting(DataStorageFormat.BONSAI);
localBlockchainSetup.importAllBlocks(
HeaderValidationMode.LIGHT_DETACHED_ONLY, HeaderValidationMode.LIGHT);
protocolContext = localBlockchainSetup.getProtocolContext();
}
@BeforeEach
public void setupTest() {
WorldStateStorageCoordinator worldStateStorageCoordinator =
new WorldStateStorageCoordinator(
((DiffBasedWorldStateProvider) protocolContext.getWorldStateArchive())
.getWorldStateKeyValueStorage());
SnapSyncConfiguration snapSyncConfiguration =
ImmutableSnapSyncConfiguration.builder().isSnapServerEnabled(true).build();
snapServer =
new SnapServer(
snapSyncConfiguration,
new EthMessages(),
worldStateStorageCoordinator,
protocolContext)
.start();
initAccounts();
}
/**
* In this test, we request the entire state range, but limit the response to 4000 bytes.
* Expected: 86 accounts.
*/
@Test
public void test0_RequestEntireStateRangeWith4000BytesLimit() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.responseBytes(4000)
.expectedAccounts(86)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(
Bytes32.fromHexString(
"0x445cb5c1278fdce2f9cbdb681bdd76c52f8e50e41dbd9e220242a69ba99ac099"))
.build());
}
/**
* In this test, we request the entire state range, but limit the response to 3000 bytes.
* Expected: 65 accounts.
*/
@Test
public void test1_RequestEntireStateRangeWith3000BytesLimit() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.responseBytes(3000)
.expectedAccounts(65)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(
Bytes32.fromHexString(
"0x2e6fe1362b3e388184fd7bf08e99e74170b26361624ffd1c5f646da7067b58b6"))
.build());
}
/**
* In this test, we request the entire state range, but limit the response to 2000 bytes.
* Expected: 44 accounts.
*/
@Test
public void test2_RequestEntireStateRangeWith2000BytesLimit() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.responseBytes(2000)
.expectedAccounts(44)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(
Bytes32.fromHexString(
"0x1c3f74249a4892081ba0634a819aec9ed25f34c7653f5719b9098487e65ab595"))
.build());
}
/**
* In this test, we request the entire state range, but limit the response to 1 byte. The server
* should return the first account of the state. Expected: 1 account.
*/
@Test
public void test3_RequestEntireStateRangeWith1ByteLimit() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.responseBytes(1)
.expectedAccounts(1)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(firstAccount)
.build());
}
/**
* Here we request with a responseBytes limit of zero. The server should return one account.
* Expected: 1 account.
*/
@Test
public void test4_RequestEntireStateRangeWithZeroBytesLimit() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.responseBytes(0)
.expectedAccounts(1)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(firstAccount)
.build());
}
/**
* In this test, we request a range where startingHash is before the first available account key,
* and limitHash is after. The server should return the first and second account of the state
* (because the second account is the 'next available'). Expected: 2 accounts.
*/
@Test
public void test5_RequestRangeBeforeFirstAccountKey() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.startHash(hashAdd(firstAccount, -500))
.limitHash(hashAdd(firstAccount, 1))
.expectedAccounts(2)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(secondAccount)
.build());
}
/**
* Here we request range where both bounds are before the first available account key. This should
* return the first account (even though it's out of bounds). Expected: 1 account.
*/
@Test
public void test6_RequestRangeBothBoundsBeforeFirstAccountKey() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.startHash(hashAdd(firstAccount, -500))
.limitHash(hashAdd(firstAccount, -400))
.expectedAccounts(1)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(firstAccount)
.build());
}
/**
* In this test, both startingHash and limitHash are zero. The server should return the first
* available account. Expected: 1 account.
*/
@Test
public void test7_RequestBothBoundsZero() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.startHash(Hash.ZERO)
.limitHash(Hash.ZERO)
.expectedAccounts(1)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(firstAccount)
.build());
}
/**
* In this test, startingHash is exactly the first available account key. The server should return
* the first available account of the state as the first item. Expected: 86 accounts.
*/
@Test
public void test8_RequestStartingHashFirstAvailableAccountKey() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.startHash(firstAccount)
.responseBytes(4000)
.expectedAccounts(86)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(
Bytes32.fromHexString(
"0x445cb5c1278fdce2f9cbdb681bdd76c52f8e50e41dbd9e220242a69ba99ac099"))
.build());
}
/**
* In this test, startingHash is after the first available key. The server should return the
* second account of the state as the first item. Expected: 86 accounts.
*/
@Test
public void test9_RequestStartingHashAfterFirstAvailableKey() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.startHash(secondAccount)
.responseBytes(4000)
.expectedAccounts(86)
.expectedFirstAccount(secondAccount)
.expectedLastAccount(
Bytes32.fromHexString(
"0x4615e5f5df5b25349a00ad313c6cd0436b6c08ee5826e33a018661997f85ebaa"))
.build());
}
/** This test requests a non-existent state root. Expected: 0 accounts. */
@Test
public void test10_RequestNonExistentStateRoot() {
Hash rootHash =
Hash.fromHexString("1337000000000000000000000000000000000000000000000000000000000000");
testAccountRangeRequest(
new AccountRangeRequestParams.Builder().rootHash(rootHash).expectedAccounts(0).build());
}
/**
* This test requests data at a state root that is 127 blocks old. We expect the server to have
* this state available. Expected: 84 accounts.
*/
@Test
public void test12_RequestStateRoot127BlocksOld() {
Hash rootHash =
protocolContext
.getBlockchain()
.getBlockHeader((protocolContext.getBlockchain().getChainHeadBlockNumber() - 127))
.orElseThrow()
.getStateRoot();
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.expectedAccounts(84)
.responseBytes(4000)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(
Bytes32.fromHexString(
"0x580aa878e2f92d113a12c0a3ce3c21972b03dbe80786858d49a72097e2c491a3"))
.build());
}
/**
* This test requests data at a state root that is actually the storage root of an existing
* account. The server is supposed to ignore this request. Expected: 0 accounts.
*/
@Test
public void test13_RequestStateRootIsStorageRoot() {
Hash rootHash =
Hash.fromHexString("df97f94bc47471870606f626fb7a0b42eed2d45fcc84dc1200ce62f7831da990");
testAccountRangeRequest(
new AccountRangeRequestParams.Builder().rootHash(rootHash).expectedAccounts(0).build());
}
/**
* In this test, the startingHash is after limitHash (wrong order). The server should ignore this
* invalid request. Expected: 0 accounts.
*/
@Test
public void test14_RequestStartingHashAfterLimitHash() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.startHash(Hash.LAST)
.limitHash(Hash.ZERO)
.expectedAccounts(0)
.build());
}
/**
* In this test, the startingHash is the first available key, and limitHash is a key before
* startingHash (wrong order). The server should return the first available key. Expected: 1
* account.
*/
@Test
public void test15_RequestStartingHashFirstAvailableKeyAndLimitHashBefore() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.startHash(firstAccount)
.limitHash(hashAdd(firstAccount, -1))
.expectedAccounts(1)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(firstAccount)
.build());
}
/**
* In this test, the startingHash is the first available key and limitHash is zero. (wrong order).
* The server should return the first available key. Expected: 1 account.
*/
@Test
public void test16_RequestStartingHashFirstAvailableKeyAndLimitHashZero() {
testAccountRangeRequest(
new AccountRangeRequestParams.Builder()
.rootHash(rootHash)
.startHash(firstAccount)
.limitHash(Hash.ZERO)
.expectedAccounts(1)
.expectedFirstAccount(firstAccount)
.expectedLastAccount(firstAccount)
.build());
}
private void testAccountRangeRequest(final AccountRangeRequestParams params) {
NavigableMap<Bytes32, Bytes> accounts = getAccountRange(params);
assertThat(accounts.size()).isEqualTo(params.getExpectedAccounts());
if (params.getExpectedAccounts() > 0) {
assertThat(accounts.firstKey()).isEqualTo(params.getExpectedFirstAccount());
assertThat(accounts.lastKey()).isEqualTo(params.getExpectedLastAccount());
}
}
private NavigableMap<Bytes32, Bytes> getAccountRange(final AccountRangeRequestParams params) {
Hash rootHash = params.getRootHash();
Bytes32 startHash = params.getStartHash();
Bytes32 limitHash = params.getLimitHash();
BigInteger sizeRequest = BigInteger.valueOf(params.getResponseBytes());
GetAccountRangeMessage requestMessage =
GetAccountRangeMessage.create(rootHash, startHash, limitHash, sizeRequest);
AccountRangeMessage resultMessage =
AccountRangeMessage.readFrom(
snapServer.constructGetAccountRangeResponse(
requestMessage.wrapMessageData(BigInteger.ONE)));
NavigableMap<Bytes32, Bytes> accounts = resultMessage.accountData(false).accounts();
return accounts;
}
@SuppressWarnings("UnusedVariable")
private void initAccounts() {
rootHash = protocolContext.getWorldStateArchive().getMutable().rootHash();
GetAccountRangeMessage requestMessage =
GetAccountRangeMessage.create(rootHash, Hash.ZERO, Hash.LAST, BigInteger.valueOf(4000));
AccountRangeMessage resultMessage =
AccountRangeMessage.readFrom(
snapServer.constructGetAccountRangeResponse(
requestMessage.wrapMessageData(BigInteger.ONE)));
NavigableMap<Bytes32, Bytes> accounts = resultMessage.accountData(false).accounts();
firstAccount = accounts.firstEntry().getKey();
secondAccount =
accounts.entrySet().stream().skip(1).findFirst().orElse(accounts.firstEntry()).getKey();
}
private Bytes32 hashAdd(final Bytes32 hash, final int value) {
var result = Hash.wrap(hash).toBigInteger().add(BigInteger.valueOf(value));
Bytes resultBytes = Bytes.wrap(result.toByteArray());
return Bytes32.leftPad(resultBytes);
}
public static class AccountRangeRequestParams {
private final Hash rootHash;
private final Bytes32 startHash;
private final Bytes32 limitHash;
private final int responseBytes;
private final int expectedAccounts;
private final Bytes32 expectedFirstAccount;
private final Bytes32 expectedLastAccount;
private AccountRangeRequestParams(final Builder builder) {
this.rootHash = builder.rootHash;
this.startHash = builder.startHash;
this.limitHash = builder.limitHash;
this.responseBytes = builder.responseBytes;
this.expectedAccounts = builder.expectedAccounts;
this.expectedFirstAccount = builder.expectedFirstAccount;
this.expectedLastAccount = builder.expectedLastAccount;
}
public static class Builder {
private Hash rootHash = null;
private Bytes32 startHash = Bytes32.ZERO;
private Bytes32 limitHash = Hash.LAST;
private int responseBytes = Integer.MAX_VALUE;
private int expectedAccounts = 0;
private Bytes32 expectedFirstAccount = null;
private Bytes32 expectedLastAccount = null;
public Builder rootHash(final Hash rootHashHex) {
this.rootHash = rootHashHex;
return this;
}
public Builder startHash(final Bytes32 startHashHex) {
this.startHash = startHashHex;
return this;
}
public Builder limitHash(final Bytes32 limitHashHex) {
this.limitHash = limitHashHex;
return this;
}
public Builder responseBytes(final int responseBytes) {
this.responseBytes = responseBytes;
return this;
}
public Builder expectedAccounts(final int expectedAccounts) {
this.expectedAccounts = expectedAccounts;
return this;
}
public Builder expectedFirstAccount(final Bytes32 expectedFirstAccount) {
this.expectedFirstAccount = expectedFirstAccount;
return this;
}
public Builder expectedLastAccount(final Bytes32 expectedLastAccount) {
this.expectedLastAccount = expectedLastAccount;
return this;
}
public AccountRangeRequestParams build() {
return new AccountRangeRequestParams(this);
}
}
// Getters for each field
public Hash getRootHash() {
return rootHash;
}
public Bytes32 getStartHash() {
return startHash;
}
public Bytes32 getLimitHash() {
return limitHash;
}
public int getResponseBytes() {
return responseBytes;
}
public int getExpectedAccounts() {
return expectedAccounts;
}
public Bytes32 getExpectedFirstAccount() {
return expectedFirstAccount;
}
public Bytes32 getExpectedLastAccount() {
return expectedLastAccount;
}
}
}

@ -188,7 +188,7 @@ public class SnapServerTest {
public void assertAccountLimitRangeResponse() { public void assertAccountLimitRangeResponse() {
// assert we limit the range response according to size // assert we limit the range response according to size
final int acctCount = 2000; final int acctCount = 2000;
final long acctRLPSize = 105; final long acctRLPSize = 37;
List<Integer> randomLoad = IntStream.range(1, 4096).boxed().collect(Collectors.toList()); List<Integer> randomLoad = IntStream.range(1, 4096).boxed().collect(Collectors.toList());
Collections.shuffle(randomLoad); Collections.shuffle(randomLoad);

@ -14,11 +14,15 @@
*/ */
package org.hyperledger.besu.ethereum.eth.messages.snap; package org.hyperledger.besu.ethereum.eth.messages.snap;
import static org.assertj.core.api.Assertions.assertThat;
import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData;
import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.rlp.RLPInput;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import java.util.ArrayList; import java.util.ArrayList;
@ -28,7 +32,6 @@ import java.util.Map;
import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.bytes.Bytes32;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
public final class AccountRangeMessageTest { public final class AccountRangeMessageTest {
@ -51,7 +54,81 @@ public final class AccountRangeMessageTest {
// check match originals. // check match originals.
final AccountRangeMessage.AccountRangeData range = message.accountData(false); final AccountRangeMessage.AccountRangeData range = message.accountData(false);
Assertions.assertThat(range.accounts()).isEqualTo(keys); assertThat(range.accounts()).isEqualTo(keys);
Assertions.assertThat(range.proofs()).isEqualTo(proofs); assertThat(range.proofs()).isEqualTo(proofs);
}
@Test
public void toSlimAccountTest() {
// Initialize nonce and balance
long nonce = 1L;
Wei balance = Wei.of(2L);
// Create a StateTrieAccountValue with the given nonce and balance
final StateTrieAccountValue accountValue =
new StateTrieAccountValue(nonce, balance, Hash.EMPTY_TRIE_HASH, Hash.EMPTY);
// Encode the account value to RLP
final BytesValueRLPOutput rlpOut = new BytesValueRLPOutput();
accountValue.writeTo(rlpOut);
// Convert the encoded account value to a slim account representation
Bytes slimAccount = AccountRangeMessage.toSlimAccount(RLP.input(rlpOut.encoded()));
// Read the slim account RLP input
RLPInput in = RLP.input(slimAccount);
in.enterList();
// Verify the nonce and balance
final long expectedNonce = in.readLongScalar();
final Wei expectedWei = Wei.of(in.readUInt256Scalar());
assertThat(expectedNonce).isEqualTo(nonce);
assertThat(expectedWei).isEqualTo(balance);
// Check that the storageRoot is empty
assertThat(in.nextIsNull()).isTrue();
in.skipNext();
// Check that the codeHash is empty
assertThat(in.nextIsNull()).isTrue();
in.skipNext();
// Exit the list
in.leaveList();
}
@Test
public void toFullAccountTest() {
// Initialize nonce and balance
long nonce = 1L;
Wei balance = Wei.of(2L);
// Create a StateTrieAccountValue with the given nonce and balance
final StateTrieAccountValue accountValue =
new StateTrieAccountValue(nonce, balance, Hash.EMPTY_TRIE_HASH, Hash.EMPTY);
// Encode the account value to RLP
final BytesValueRLPOutput rlpOut = new BytesValueRLPOutput();
accountValue.writeTo(rlpOut);
// Convert the encoded account value to a full account representation
Bytes fullAccount = AccountRangeMessage.toFullAccount(RLP.input(rlpOut.encoded()));
// Read the full account RLP input
RLPInput in = RLP.input(fullAccount);
in.enterList();
// Verify the nonce and balance
final long expectedNonce = in.readLongScalar();
final Wei expectedWei = Wei.of(in.readUInt256Scalar());
assertThat(expectedNonce).isEqualTo(nonce);
assertThat(expectedWei).isEqualTo(balance);
// Verify the storageRoot and codeHash
assertThat(in.readBytes32()).isEqualTo(Hash.EMPTY_TRIE_HASH);
assertThat(in.readBytes32()).isEqualTo(Hash.EMPTY);
// Exit the list
in.leaveList();
} }
} }

@ -36,10 +36,4 @@ public class MetricsSystemModule {
MetricsSystem provideMetricsSystem(final MetricsConfiguration metricsConfig) { MetricsSystem provideMetricsSystem(final MetricsConfiguration metricsConfig) {
return MetricsSystemFactory.create(metricsConfig); return MetricsSystemFactory.create(metricsConfig);
} }
@Provides
@Singleton
ObservableMetricsSystem provideObservableMetricsSystem(final MetricsConfiguration metricsConfig) {
return MetricsSystemFactory.create(metricsConfig);
}
} }

@ -41,6 +41,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.DoubleSupplier; import java.util.function.DoubleSupplier;
import java.util.stream.Stream; import java.util.stream.Stream;
import javax.inject.Singleton;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.AttributeKey;
@ -67,6 +68,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** Metrics system relying on the native OpenTelemetry format. */ /** Metrics system relying on the native OpenTelemetry format. */
@Singleton
public class OpenTelemetrySystem implements ObservableMetricsSystem { public class OpenTelemetrySystem implements ObservableMetricsSystem {
private static final Logger LOG = LoggerFactory.getLogger(OpenTelemetrySystem.class); private static final Logger LOG = LoggerFactory.getLogger(OpenTelemetrySystem.class);

@ -52,6 +52,8 @@ public final class BlockTestUtil {
Suppliers.memoize(BlockTestUtil::supplyUpgradedForkResources); Suppliers.memoize(BlockTestUtil::supplyUpgradedForkResources);
private static final Supplier<ChainResources> testRpcCompactChainSupplier = private static final Supplier<ChainResources> testRpcCompactChainSupplier =
Suppliers.memoize(BlockTestUtil::supplyTestRpcCompactResources); Suppliers.memoize(BlockTestUtil::supplyTestRpcCompactResources);
private static final Supplier<ChainResources> snapTestChainSupplier =
Suppliers.memoize(BlockTestUtil::supplySnapTestChainResources);
/** /**
* Gets test blockchain url. * Gets test blockchain url.
@ -156,6 +158,15 @@ public final class BlockTestUtil {
return testRpcCompactChainSupplier.get(); return testRpcCompactChainSupplier.get();
} }
/**
* Gets test chain resources for Snap tests.
*
* @return the test chain resources
*/
public static ChainResources getSnapTestChainResources() {
return snapTestChainSupplier.get();
}
private static ChainResources supplyTestChainResources() { private static ChainResources supplyTestChainResources() {
final URL genesisURL = final URL genesisURL =
ensureFileUrl(BlockTestUtil.class.getClassLoader().getResource("testGenesis.json")); ensureFileUrl(BlockTestUtil.class.getClassLoader().getResource("testGenesis.json"));
@ -164,6 +175,15 @@ public final class BlockTestUtil {
return new ChainResources(genesisURL, blocksURL); return new ChainResources(genesisURL, blocksURL);
} }
private static ChainResources supplySnapTestChainResources() {
final URL genesisURL =
ensureFileUrl(BlockTestUtil.class.getClassLoader().getResource("snap/snapGenesis.json"));
final URL blocksURL =
ensureFileUrl(
BlockTestUtil.class.getClassLoader().getResource("snap/testBlockchain.blocks"));
return new ChainResources(genesisURL, blocksURL);
}
private static ChainResources supplyHiveTestChainResources() { private static ChainResources supplyHiveTestChainResources() {
final URL genesisURL = final URL genesisURL =
ensureFileUrl(BlockTestUtil.class.getClassLoader().getResource("hive/testGenesis.json")); ensureFileUrl(BlockTestUtil.class.getClassLoader().getResource("hive/testGenesis.json"));

@ -0,0 +1,111 @@
{
"config": {
"ethash": {},
"chainID": 3503995874084926,
"homesteadBlock": 0,
"eip150Block": 6,
"eip155Block": 12,
"eip158Block": 12,
"byzantiumBlock": 18,
"constantinopleBlock": 24,
"constantinopleFixBlock": 30,
"istanbulBlock": 36,
"muirGlacierBlock": 42,
"berlinBlock": 48,
"londonBlock": 54,
"arrowGlacierBlock": 60,
"grayGlacierBlock": 66,
"mergeNetsplitBlock": 72,
"terminalTotalDifficulty": 9454784,
"shanghaiTime": 780,
"cancunTime": 840
},
"nonce": "0x0",
"timestamp": "0x0",
"extraData": "0x68697665636861696e",
"gasLimit": "0x23f3e20",
"difficulty": "0x20000",
"mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"coinbase": "0x0000000000000000000000000000000000000000",
"alloc": {
"000f3df6d732807ef1319fb7b8bb8522d0beac02": {
"code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500",
"balance": "0x2a"
},
"0c2c51a0990aee1d73c1228de158688341557508": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"14e46043e63d0e3cdcf2530519f4cfaf35058cb2": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"16c57edf7fa9d9525378b0b81bf8a3ced0620c1c": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"1f4924b14f34e24159387c0a4cdbaa32f3ddb0cf": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"1f5bde34b4afc686f136c7a3cb6ec376f7357759": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"2d389075be5be9f2246ad654ce152cf05990b209": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"3ae75c08b4c907eb63a8960c45b86e1e9ab6123c": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"4340ee1b812acb40a1eb561c019c327b243b92df": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"4a0f1452281bcec5bd90c3dce6162a5995bfe9df": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"4dde844b71bcdf95512fb4dc94e84fb67b512ed8": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"5f552da00dfb4d3749d9e62dcee3c918855a86a0": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"654aa64f5fbefb84c270ec74211b81ca8c44a72e": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"717f8aa2b982bee0e29f573d31df288663e1ce16": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"7435ed30a8b4aeb0877cef0c6e8cffe834eb865f": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"83c7e323d189f18725ac510004fdc2941f8c4a78": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"84e75c28348fb86acea1a93a39426d7d60f4cc46": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"8bebc8ba651aee624937e7d897853ac30c95a067": {
"storage": {
"0x0000000000000000000000000000000000000000000000000000000000000001": "0x0000000000000000000000000000000000000000000000000000000000000001",
"0x0000000000000000000000000000000000000000000000000000000000000002": "0x0000000000000000000000000000000000000000000000000000000000000002",
"0x0000000000000000000000000000000000000000000000000000000000000003": "0x0000000000000000000000000000000000000000000000000000000000000003"
},
"balance": "0x1",
"nonce": "0x1"
},
"c7b99a164efd027a93f147376cc7da7c67c6bbe0": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"d803681e487e6ac18053afc5a6cd813c86ec3e4d": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"e7d13f7aa2a838d24c59b40186a0aca1e21cffcc": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
},
"eda8645ba6948855e3b3cd596bbb07596d59c603": {
"balance": "0xc097ce7bc90715b34b9f1000000000"
}
},
"number": "0x0",
"gasUsed": "0x0",
"parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"baseFeePerGas": null,
"excessBlobGas": null,
"blobGasUsed": null
}
Loading…
Cancel
Save