Add post-run allocation output to EVMTool (#4709)

* Add post-run allocation output to EVMTool

Add a CLI flag --json-alloc that will output the post-execution state of
the allocations the EVM Tool executed in. As well as post-execution
state for state-tests.

Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>

CHANGELOG.md

@@ -1,10 +1,13 @@
 # Changelog
-## 23.1
+## 23.1.0-beta
 
 ### Breaking Changes
 - GoQuorum-compatible privacy is deprecated and will be removed in 23.4
 - IBFT 1.0 is deprecated and will be removed in 23.4
 
+### Additions and Improvements
+- Added post-execution state logging option to EVM Tool [#4709](https://github.com/hyperledger/besu/pull/4709)
+
 ## 22.10.4
 
 ### Additions and Improvements
@@ -59,7 +62,7 @@ https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/22.10.2/besu-22.10.2
 - Support for ephemeral testnet Shandong, for EOF testing. [#4599](https://github.com/hyperledger/besu/pull/4599)
 - Improve performance of block processing by parallelizing some parts during the "commit" step [#4635](https://github.com/hyperledger/besu/pull/4635)
 - Upgrade RocksDB version from 7.6.0 to 7.7.3
-- Added new RPC endpoints `debug_setHead` & `debug_replayBlock  [4580](https://github.com/hyperledger/besu/pull/4580)
+- Added new RPC endpoints `debug_setHead` & `debug_replayBlock  [#4580](https://github.com/hyperledger/besu/pull/4580)
 - Upgrade OpenTelemetry to version 1.19.0 [#3675](https://github.com/hyperledger/besu/pull/3675)
 - Implement Eth/67 sub-protocol [#4596](https://github.com/hyperledger/besu/issues/4596)
 - Backward sync log UX improvements [#4655](https://github.com/hyperledger/besu/pull/4655)

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java

@@ -38,6 +38,7 @@ import org.hyperledger.besu.evm.precompile.PrecompileContractRegistry;
 import org.hyperledger.besu.evm.processor.MessageCallProcessor;
 import org.hyperledger.besu.evm.tracing.OperationTracer;
 import org.hyperledger.besu.evm.tracing.StandardJsonTracer;
+import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.util.Log4j2ConfiguratorUtil;
 
 import java.io.BufferedWriter;
@@ -47,13 +48,18 @@ import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.time.Instant;
 import java.util.ArrayDeque;
+import java.util.Comparator;
 import java.util.Deque;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
+import com.google.common.base.Joiner;
 import com.google.common.base.Stopwatch;
 import io.vertx.core.json.JsonObject;
 import org.apache.logging.log4j.Level;
 import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
+import org.apache.tuweni.units.bigints.UInt256;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
@@ -130,6 +136,12 @@ public class EvmToolCommand implements Runnable {
       scope = INHERIT)
   final Boolean showJsonResults = false;
 
+  @Option(
+      names = {"--json-alloc"},
+      description = "Output the final allocations after a run.",
+      scope = INHERIT)
+  final Boolean showJsonAlloc = false;
+
   @Option(
       names = {"--nomemory"},
       description = "Disable showing the full memory output for each op.",
@@ -151,6 +163,7 @@ public class EvmToolCommand implements Runnable {
       description = "Number of times to repeat for benchmarking.")
   private final Integer repeat = 0;
 
+  static final Joiner STORAGE_JOINER = Joiner.on(",\n");
   private final EvmToolCommandOptionsModule daggerOptions = new EvmToolCommandOptionsModule();
   private PrintWriter out =
       new PrintWriter(new BufferedWriter(new OutputStreamWriter(System.out, UTF_8)), true);
@@ -313,10 +326,62 @@ public class EvmToolCommand implements Runnable {
         }
         lastTime = stopwatch.elapsed().toNanos();
         stopwatch.reset();
+        if (showJsonAlloc && lastLoop) {
+          updater.commit();
+          WorldState worldState = component.getWorldState();
+          dumpWorldState(worldState, out);
+        }
       } while (remainingIters-- > 0);
 
     } catch (final IOException e) {
       LOG.error("Unable to create Genesis module", e);
     }
   }
+
+  public static void dumpWorldState(final WorldState worldState, final PrintWriter out) {
+    out.println("{");
+    worldState
+        .streamAccounts(Bytes32.ZERO, Integer.MAX_VALUE)
+        .sorted(Comparator.comparing(o -> o.getAddress().get().toHexString()))
+        .forEach(
+            account -> {
+              out.println(
+                  " \"" + account.getAddress().map(Address::toHexString).orElse("-") + "\": {");
+              if (account.getCode() != null && account.getCode().size() > 0) {
+                out.println("  \"code\": \"" + account.getCode().toHexString() + "\",");
+              }
+              var storageEntries = account.storageEntriesFrom(Bytes32.ZERO, Integer.MAX_VALUE);
+              if (!storageEntries.isEmpty()) {
+                out.println("  \"storage\": {");
+                out.println(
+                    STORAGE_JOINER.join(
+                        storageEntries.values().stream()
+                            .map(
+                                accountStorageEntry ->
+                                    "   \""
+                                        + accountStorageEntry
+                                            .getKey()
+                                            .map(UInt256::toHexString)
+                                            .orElse("-")
+                                        + "\": \""
+                                        + accountStorageEntry.getValue().toHexString()
+                                        + "\"")
+                            .collect(Collectors.toList())));
+                out.println("  },");
+              }
+              out.print("  \"balance\": \"" + account.getBalance().toShortHexString() + "\"");
+              if (account.getNonce() > 0) {
+                out.println(",");
+                out.println(
+                    "  \"nonce\": \""
+                        + Bytes.ofUnsignedLong(account.getNonce()).toShortHexString()
+                        + "\"");
+              } else {
+                out.println();
+              }
+              out.println(" },");
+            });
+    out.println("}");
+    out.flush();
+  }
 }

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommandOptionsModule.java

@@ -25,6 +25,7 @@ import org.hyperledger.besu.services.BesuConfigurationImpl;
 
 import java.nio.file.Path;
 import javax.inject.Named;
+import javax.inject.Singleton;
 
 import dagger.Module;
 import dagger.Provides;
@@ -71,6 +72,7 @@ public class EvmToolCommandOptionsModule {
   final Path dataPath = getDefaultBesuDataPath(this);
 
   @Provides
+  @Singleton
   BesuConfiguration provideBesuConfiguration() {
     return new BesuConfigurationImpl(dataPath, dataPath.resolve(BesuController.DATABASE_PATH));
   }
@@ -83,6 +85,7 @@ public class EvmToolCommandOptionsModule {
   private final BlockParameter blockParameter = BlockParameter.PENDING;
 
   @Provides
+  @Singleton
   BlockParameter provideBlockParameter() {
     return blockParameter;
   }

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolComponent.java

@@ -16,6 +16,7 @@
 package org.hyperledger.besu.evmtool;
 
 import org.hyperledger.besu.ethereum.chain.Blockchain;
+import org.hyperledger.besu.ethereum.core.MutableWorldState;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 
@@ -40,5 +41,7 @@ public interface EvmToolComponent {
 
   WorldUpdater getWorldUpdater();
 
+  MutableWorldState getWorldState();
+
   Blockchain getBlockchain();
 }

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/MetricsSystemModule.java

@@ -20,6 +20,8 @@ import org.hyperledger.besu.metrics.MetricsSystemFactory;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 
+import javax.inject.Singleton;
+
 import dagger.Module;
 import dagger.Provides;
 
@@ -28,6 +30,7 @@ import dagger.Provides;
 public class MetricsSystemModule {
 
   @Provides
+  @Singleton
   MetricsSystem getMetricsSystem() {
     return MetricsSystemFactory.create(MetricsConfiguration.builder().build());
   }

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/ProtocolModule.java

@@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 
 import java.util.function.Function;
+import javax.inject.Singleton;
 
 import dagger.Module;
 import dagger.Provides;
@@ -28,6 +29,7 @@ import dagger.Provides;
 public class ProtocolModule {
 
   @Provides
+  @Singleton
   Function<Integer, ProtocolSpec> getProtocolSpec(final ProtocolSchedule protocolSchedule) {
     return protocolSchedule::getByBlockNumber;
   }

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java

@@ -16,6 +16,7 @@
 
 package org.hyperledger.besu.evmtool;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules.shouldClearEmptyAccounts;
 import static org.hyperledger.besu.evmtool.StateTestSubCommand.COMMAND_NAME;
 
@@ -45,12 +46,14 @@ import org.hyperledger.besu.evmtool.exception.UnsupportedForkException;
 import org.hyperledger.besu.util.Log4j2ConfiguratorUtil;
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
 import java.io.PrintStream;
-import java.nio.charset.StandardCharsets;
+import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -124,8 +127,7 @@ public class StateTestSubCommand implements Runnable {
     try {
       if (stateTestFiles.isEmpty()) {
         // if no state tests were specified use standard input to get filenames
-        final BufferedReader in =
-            new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
+        final BufferedReader in = new BufferedReader(new InputStreamReader(input, UTF_8));
         while (true) {
           final String fileName = in.readLine();
           if (fileName == null) {
@@ -166,10 +168,10 @@ public class StateTestSubCommand implements Runnable {
 
   private void traceTestSpecs(final String test, final List<GeneralStateTestCaseEipSpec> specs) {
     Log4j2ConfiguratorUtil.setLevel(
-        "org.hyperledger.besu.ethereum.mainnet.ProtocolScheduleBuilder", Level.OFF);
+        "org.hyperledger.besu.ethereum.mainnet.AbstractProtocolScheduleBuilder", Level.OFF);
     final var referenceTestProtocolSchedules = ReferenceTestProtocolSchedules.create();
     Log4j2ConfiguratorUtil.setLevel(
-        "org.hyperledger.besu.ethereum.mainnet.ProtocolScheduleBuilder", null);
+        "org.hyperledger.besu.ethereum.mainnet.AbstractProtocolScheduleBuilder", null);
 
     final OperationTracer tracer = // You should have picked Mercy.
         parentCommand.showJsonResults
@@ -184,7 +186,9 @@ public class StateTestSubCommand implements Runnable {
 
       final ObjectNode summaryLine = objectMapper.createObjectNode();
       if (transaction == null) {
-        // Check the world state root hash.
+        if (parentCommand.showJsonAlloc || parentCommand.showJsonResults) {
+          output.println("{\"error\":\"Transaction was invalid, trace and alloc unavailable.\"}");
+        }
         summaryLine.put("test", test);
         summaryLine.put("fork", spec.getFork());
         summaryLine.put("d", spec.getDataIndex());
@@ -271,6 +275,12 @@ public class StateTestSubCommand implements Runnable {
               "validationError",
               "Exception '" + spec.getExpectException() + "' was expected but did not occur");
         }
+
+        if (parentCommand.showJsonAlloc) {
+          EvmToolCommand.dumpWorldState(
+              worldState,
+              new PrintWriter(new BufferedWriter(new OutputStreamWriter(output, UTF_8))));
+        }
       }
 
       output.println(summaryLine);