Updates for execution-spec-tests (#5638)

Updates needed to keep executing execution-spec-tests
* add withdrawals support
* add access lists to transactions
* accept t8n and b11r CLI args that are zero length or all whitespace
* Enumerate forks in t8n and b11r help
* remove JSON wrapping from t8n transaction RLP

Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java

@@ -33,6 +33,9 @@ import java.io.InputStreamReader;
 import java.io.PrintStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Stack;
 
 import com.fasterxml.jackson.core.JsonParser.Feature;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -47,7 +50,12 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
 import picocli.CommandLine.Command;
+import picocli.CommandLine.IParameterConsumer;
+import picocli.CommandLine.Model.ArgSpec;
+import picocli.CommandLine.Model.CommandSpec;
 import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+import picocli.CommandLine.ParentCommand;
 
 @Command(
     name = COMMAND_NAME,
@@ -81,6 +89,12 @@ public class B11rSubCommand implements Runnable {
       description = "The ommers for the block")
   private final Path ommers = stdinPath;
 
+  @Option(
+      names = {"--input.withdrawals"},
+      paramLabel = "full path",
+      description = "The withdrawals for the block")
+  private final Path withdrawals = stdinPath;
+
   @Option(
       names = {"--seal.clique"},
       paramLabel = "full path",
@@ -114,7 +128,25 @@ public class B11rSubCommand implements Runnable {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  @CommandLine.ParentCommand private final EvmToolCommand parentCommand;
+  @ParentCommand private final EvmToolCommand parentCommand;
+
+  @Parameters(parameterConsumer = OnlyEmptyParams.class)
+  @SuppressWarnings("UnusedVariable")
+  private final List<String> parameters = new ArrayList<>();
+
+  static class OnlyEmptyParams implements IParameterConsumer {
+    @Override
+    public void consumeParameters(
+        final Stack<String> args, final ArgSpec argSpec, final CommandSpec commandSpec) {
+      while (!args.isEmpty()) {
+        if (!args.pop().isEmpty()) {
+          throw new CommandLine.ParameterException(
+              argSpec.command().commandLine(),
+              "The block-builder command does not accept any non-empty parameters");
+        }
+      }
+    }
+  }
 
   @SuppressWarnings("unused")
   public B11rSubCommand() {
@@ -136,7 +168,7 @@ public class B11rSubCommand implements Runnable {
             .withSpacesInObjectEntries()
             .withObjectIndenter(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE.withIndent("  "))
             .withArrayIndenter(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE.withIndent("  ")));
-    final ObjectReader t8nReader = objectMapper.reader();
+    final ObjectReader b11rReader = objectMapper.reader();
     objectMapper.disable(Feature.AUTO_CLOSE_SOURCE);
 
     ObjectNode config;
@@ -144,32 +176,39 @@ public class B11rSubCommand implements Runnable {
       if (header.equals(stdinPath)
           || txs.equals(stdinPath)
           || ommers.equals(stdinPath)
-          || sealClique.equals(stdinPath)) {
+          || sealClique.equals(stdinPath)
+          || withdrawals.equals(stdinPath)) {
         config =
             (ObjectNode)
-                t8nReader.readTree(new InputStreamReader(parentCommand.in, StandardCharsets.UTF_8));
+                b11rReader.readTree(
+                    new InputStreamReader(parentCommand.in, StandardCharsets.UTF_8));
       } else {
         config = objectMapper.createObjectNode();
       }
 
       if (!header.equals(stdinPath)) {
         try (FileReader reader = new FileReader(header.toFile(), StandardCharsets.UTF_8)) {
-          config.set("header", t8nReader.readTree(reader));
+          config.set("header", b11rReader.readTree(reader));
         }
       }
       if (!txs.equals(stdinPath)) {
         try (FileReader reader = new FileReader(txs.toFile(), StandardCharsets.UTF_8)) {
-          config.set("txs", t8nReader.readTree(reader));
+          config.set("txs", b11rReader.readTree(reader));
+        }
+      }
+      if (!withdrawals.equals(stdinPath)) {
+        try (FileReader reader = new FileReader(withdrawals.toFile(), StandardCharsets.UTF_8)) {
+          config.set("withdrawals", b11rReader.readTree(reader));
         }
       }
       if (!ommers.equals(stdinPath)) {
         try (FileReader reader = new FileReader(ommers.toFile(), StandardCharsets.UTF_8)) {
-          config.set("ommers", t8nReader.readTree(reader));
+          config.set("ommers", b11rReader.readTree(reader));
         }
       }
       if (!sealClique.equals(stdinPath)) {
         try (FileReader reader = new FileReader(sealClique.toFile(), StandardCharsets.UTF_8)) {
-          config.set("clique", t8nReader.readTree(reader));
+          config.set("clique", b11rReader.readTree(reader));
         }
       }
     } catch (final JsonProcessingException jpe) {

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java

@@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup;
 import org.hyperledger.besu.evm.Code;
 import org.hyperledger.besu.evm.EVM;
+import org.hyperledger.besu.evm.EvmSpecVersion;
 import org.hyperledger.besu.evm.account.AccountStorageEntry;
 import org.hyperledger.besu.evm.code.CodeInvalid;
 import org.hyperledger.besu.evm.frame.MessageFrame;
@@ -53,10 +54,15 @@ import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.time.Instant;
 import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Deque;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.NavigableMap;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Stopwatch;
@@ -246,10 +252,40 @@ public class EvmToolCommand implements Runnable {
             .addPattern("^--trace.(\\w(-|\\w)*)$", "--trace.no$1", "--trace.[no]$1")
             .build());
 
+    // Enumerate forks to support execution-spec-tests
+    addForkHelp(commandLine.getSubcommands().get("t8n"));
+    addForkHelp(commandLine.getSubcommands().get("t8n-server"));
+
     commandLine.setExecutionStrategy(new CommandLine.RunLast());
     commandLine.execute(args);
   }
 
+  private static void addForkHelp(final CommandLine subCommandLine) {
+    subCommandLine
+        .getHelpSectionMap()
+        .put("forks_header", help -> help.createHeading("%nKnown Forks:%n"));
+    subCommandLine
+        .getHelpSectionMap()
+        .put(
+            "forks",
+            help ->
+                help.createTextTable(
+                        Arrays.stream(EvmSpecVersion.values())
+                            .collect(
+                                Collectors.toMap(
+                                    EvmSpecVersion::getName,
+                                    EvmSpecVersion::getDescription,
+                                    (a, b) -> b,
+                                    LinkedHashMap::new)))
+                    .toString());
+    List<String> keys = new ArrayList<>(subCommandLine.getHelpSectionKeys());
+    int index = keys.indexOf(CommandLine.Model.UsageMessageSpec.SECTION_KEY_FOOTER_HEADING);
+    keys.add(index, "forks_header");
+    keys.add(index + 1, "forks");
+
+    subCommandLine.setHelpSectionKeys(keys);
+  }
+
   @Override
   public void run() {
     LogConfigurator.setLevel("", "OFF");

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java

@@ -30,6 +30,7 @@ import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv;
 import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState;
 import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
+import org.hyperledger.besu.evm.AccessListEntry;
 import org.hyperledger.besu.evm.tracing.OperationTracer;
 import org.hyperledger.besu.evm.tracing.StandardJsonTracer;
 import org.hyperledger.besu.plugin.data.TransactionType;
@@ -49,6 +50,10 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.Stack;
+import java.util.stream.StreamSupport;
 
 import com.fasterxml.jackson.core.JsonParser.Feature;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -64,7 +69,12 @@ import org.apache.tuweni.bytes.Bytes32;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine.Command;
+import picocli.CommandLine.IParameterConsumer;
+import picocli.CommandLine.Model.ArgSpec;
+import picocli.CommandLine.Model.CommandSpec;
 import picocli.CommandLine.Option;
+import picocli.CommandLine.ParameterException;
+import picocli.CommandLine.Parameters;
 import picocli.CommandLine.ParentCommand;
 
 @Command(
@@ -147,6 +157,24 @@ public class T8nSubCommand implements Runnable {
 
   @ParentCommand private final EvmToolCommand parentCommand;
 
+  @Parameters(parameterConsumer = OnlyEmptyParams.class)
+  @SuppressWarnings("UnusedVariable")
+  private final List<String> parameters = new ArrayList<>();
+
+  static class OnlyEmptyParams implements IParameterConsumer {
+    @Override
+    public void consumeParameters(
+        final Stack<String> args, final ArgSpec argSpec, final CommandSpec commandSpec) {
+      while (!args.isEmpty()) {
+        if (!args.pop().isEmpty()) {
+          throw new ParameterException(
+              argSpec.command().commandLine(),
+              "The transition command does not accept any non-empty parameters");
+        }
+      }
+    }
+  }
+
   @SuppressWarnings("unused")
   public T8nSubCommand() {
     // PicoCLI requires this
@@ -270,7 +298,9 @@ public class T8nSubCommand implements Runnable {
             }
 
             @Override
-            public void disposeTracer(final OperationTracer tracer) {}
+            public void disposeTracer(final OperationTracer tracer) {
+              // single-test mode doesn't need to track tracers
+            }
           };
     }
     final T8nExecutor.T8nResult result =
@@ -302,7 +332,7 @@ public class T8nSubCommand implements Runnable {
       } else {
         try (PrintStream fileOut =
             new PrintStream(new FileOutputStream(outDir.resolve(outBody).toFile()))) {
-          fileOut.println(result.bodyBytes());
+          fileOut.print(result.bodyBytes().textValue());
         }
       }
 
@@ -364,6 +394,29 @@ public class T8nSubCommand implements Runnable {
                     Bytes.fromHexStringLenient(txNode.get("chainId").textValue()).toArrayUnsafe()));
           }
 
+          if (txNode.has("accessList")) {
+            JsonNode accessList = txNode.get("accessList");
+            if (!accessList.isArray()) {
+              parentCommand.out.printf(
+                  "TX json node unparseable: expected accessList to be an array - %s%n", txNode);
+              continue;
+            }
+            List<AccessListEntry> entries = new ArrayList<>(accessList.size());
+            for (JsonNode entryAsJson : accessList) {
+              Address address = Address.fromHexString(entryAsJson.get("address").textValue());
+              List<String> storageKeys =
+                  StreamSupport.stream(
+                          Spliterators.spliteratorUnknownSize(
+                              entryAsJson.get("storageKeys").elements(), Spliterator.ORDERED),
+                          false)
+                      .map(JsonNode::textValue)
+                      .toList();
+              var accessListEntry = AccessListEntry.createAccessListEntry(address, storageKeys);
+              entries.add(accessListEntry);
+            }
+            builder.accessList(entries);
+          }
+
           if (txNode.has("secretKey")) {
             SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance();
             KeyPair keys =

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java

@@ -75,6 +75,8 @@ public class ReferenceTestProtocolSchedules {
     builder.put("Merge", createSchedule(genesisStub.clone().mergeNetSplitBlock(0)));
     builder.put("Shanghai", createSchedule(genesisStub.clone().shanghaiTime(0)));
     builder.put("Cancun", createSchedule(genesisStub.clone().cancunTime(0)));
+    builder.put("Future_EIPs", createSchedule(genesisStub.clone().futureEipsTime(0)));
+    builder.put("Experimental_EIPs", createSchedule(genesisStub.clone().experimentalEipsTime(0)));
     return new ReferenceTestProtocolSchedules(builder.build());
   }
 

evm/src/main/java/org/hyperledger/besu/evm/EvmSpecVersion.java

@@ -21,33 +21,37 @@ import org.slf4j.LoggerFactory;
 /** The enum Evm spec version. */
 public enum EvmSpecVersion {
   /** Frontier evm spec version. */
-  FRONTIER(0, true),
+  FRONTIER(0, true, "Frontier", "Finalized"),
   /** Homestead evm spec version. */
-  HOMESTEAD(0, true),
+  HOMESTEAD(0, true, "Homestead", "Finalized"),
   /** Byzantium evm spec version. */
-  BYZANTIUM(0, true),
+  BYZANTIUM(0, true, "Byzantium", "Finalized"),
   /** Constantinople evm spec version. */
-  CONSTANTINOPLE(0, true),
+  CONSTANTINOPLE(0, true, "Constantinople", "Did not reach Mainnet"),
+  /** Petersburg / ConstantinopleFix evm spec version. */
+  PETERSBURG(0, true, "ConstantinopleFix", "Finalized (also called Petersburg)"),
   /** Istanbul evm spec version. */
-  ISTANBUL(0, true),
+  ISTANBUL(0, true, "Istanbul", "Finalized"),
+  /** Berlin evm spec version */
+  BERLIN(0, true, "Berlin", "Finalized"),
   /** London evm spec version. */
-  LONDON(0, true),
+  LONDON(0, true, "London", "Finalized"),
   /** Paris evm spec version. */
-  PARIS(0, true),
+  PARIS(0, true, "Merge", "Finalized (also called Paris)"),
   /** Shanghai evm spec version. */
-  SHANGHAI(0, true),
+  SHANGHAI(0, true, "Shanghai", "Finalized"),
   /** Cancun evm spec version. */
-  CANCUN(0, false),
+  CANCUN(0, false, "Cancun", "In Development"),
   /** Prague evm spec version. */
-  PRAGUE(0, false),
+  PRAGUE(0, false, "Prague", "Placeholder"),
   /** Osaka evm spec version. */
-  OSAKA(0, false),
+  OSAKA(0, false, "Osaka", "Placeholder"),
   /** Bogota evm spec version. */
-  BOGOTA(0, false),
+  BOGOTA(0, false, "Bogata", "Placeholder"),
   /** Development fork for unscheduled EIPs */
-  FUTURE_EIPS(1, false),
+  FUTURE_EIPS(1, false, "Future_EIPs", "Development, for accepted and unscheduled EIPs"),
   /** Development fork for EIPs not accepted to Mainnet */
-  EXPERIMENTAL_EIPS(1, false);
+  EXPERIMENTAL_EIPS(1, false, "Experimental_EIPs", "Development, for experimental EIPs");
 
   private static final Logger LOGGER = LoggerFactory.getLogger(EvmSpecVersion.class);
 
@@ -56,12 +60,23 @@ public enum EvmSpecVersion {
   /** The Max eof version. */
   final int maxEofVersion;
 
+  /** Public name matching execution-spec-tests name */
+  final String name;
+  /** A brief description of the state of the fork */
+  final String description;
+
   /** The Version warned. */
   boolean versionWarned = false;
 
-  EvmSpecVersion(final int maxEofVersion, final boolean specFinalized) {
+  EvmSpecVersion(
+      final int maxEofVersion,
+      final boolean specFinalized,
+      final String name,
+      final String description) {
     this.maxEofVersion = maxEofVersion;
     this.specFinalized = specFinalized;
+    this.name = name;
+    this.description = description;
   }
 
   /**
@@ -73,6 +88,24 @@ public enum EvmSpecVersion {
     return maxEofVersion;
   }
 
+  /**
+   * Name of the fork, in execution-spec-tests form
+   *
+   * @return name of the fork
+   */
+  public String getName() {
+    return name;
+  }
+
+  /**
+   * Description of the fork
+   *
+   * @return description
+   */
+  public String getDescription() {
+    return description;
+  }
+
   /** Maybe warn version. */
   @SuppressWarnings("AlreadyChecked") // false positive
   public void maybeWarnVersion() {

evm/src/main/java/org/hyperledger/besu/evm/MainnetEVMs.java

@@ -14,7 +14,6 @@
  */
 package org.hyperledger.besu.evm;
 
-import org.hyperledger.besu.evm.gascalculator.BerlinGasCalculator;
 import org.hyperledger.besu.evm.gascalculator.ByzantiumGasCalculator;
 import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator;
 import org.hyperledger.besu.evm.gascalculator.ConstantinopleGasCalculator;
@@ -547,7 +546,26 @@ public class MainnetEVMs {
    * @return the evm
    */
   public static EVM berlin(final BigInteger chainId, final EvmConfiguration evmConfiguration) {
-    return istanbul(new BerlinGasCalculator(), chainId, evmConfiguration);
+    return berlin(new IstanbulGasCalculator(), chainId, evmConfiguration);
+  }
+
+  /**
+   * Berlin evm.
+   *
+   * @param gasCalculator the gas calculator
+   * @param chainId the chain id
+   * @param evmConfiguration the evm configuration
+   * @return the evm
+   */
+  public static EVM berlin(
+      final GasCalculator gasCalculator,
+      final BigInteger chainId,
+      final EvmConfiguration evmConfiguration) {
+    return new EVM(
+        istanbulOperations(gasCalculator, chainId),
+        gasCalculator,
+        evmConfiguration,
+        EvmSpecVersion.BERLIN);
   }
 
   /**