Update Reference Tests to 11.3 (#4996)

* Do not check state on failed TXes
* Exclude EIPTests from test set
* Include withdrawals in block for blockchain reference test cases
* remove outdated test exclusions.

Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>
Signed-off-by: Jason Frame <jason.frame@consensys.net>
Co-authored-by: Jason Frame <jason.frame@consensys.net>

CHANGELOG.md

@@ -12,6 +12,7 @@
 - Improve get account performance by using the world state updater cache [#4897](https://github.com/hyperledger/besu/pull/4897)
 - Add new KZG precompile and option to override the trusted setup being used [#4822](https://github.com/hyperledger/besu/issues/4822)
 - Add implementation for eth_createAccessList RPC method [#4942](https://github.com/hyperledger/besu/pull/4942)
+- Updated reference tests to v11.3 [#4996](https://github.com/hyperledger/besu/pull/4996)
 
 ### Bug Fixes
 

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java

@@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
 import org.hyperledger.besu.ethereum.core.MutableWorldState;
 import org.hyperledger.besu.ethereum.core.ParsedExtraData;
 import org.hyperledger.besu.ethereum.core.Transaction;
+import org.hyperledger.besu.ethereum.core.Withdrawal;
 import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
 import org.hyperledger.besu.ethereum.rlp.RLPInput;
@@ -40,6 +41,7 @@ import org.hyperledger.besu.evm.log.LogsBloomFilter;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 
 import java.util.Map;
+import java.util.Optional;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -259,11 +261,10 @@ public class BlockchainReferenceTestCaseSpec {
       final BlockBody body =
           new BlockBody(
               input.readList(Transaction::readFrom),
-              input.readList(inputData -> BlockHeader.readFrom(inputData, blockHeaderFunctions)));
-      // input.readList(inputData -> BlockHeader.readFrom(inputData, blockHeaderFunctions),
-      // input.isEndOfCurrentList()
-      //         ? Optional.empty()
-      //           : Optional.of(input.readList(Withdrawal::readFrom))));
+              input.readList(inputData -> BlockHeader.readFrom(inputData, blockHeaderFunctions)),
+              input.isEndOfCurrentList()
+                  ? Optional.empty()
+                  : Optional.of(input.readList(Withdrawal::readFrom)));
       return new Block(header, body);
     }
   }

ethereum/referencetests/src/reference-test/external-resources

@@ -1 +1 @@
-Subproject commit 638da7ca7b2ca371ebb33d7d7d2562981aafd9b2
+Subproject commit 69c4c2a98dc8a712d4c6f5a817da4f21ff719006

ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTest.java

@@ -61,7 +61,7 @@ public class TransactionTest {
 
   public static Stream<Arguments> getTestParametersForConfig() {
     return JsonTestParameters.create(TransactionTestCaseSpec.class)
-        .generator((name, spec, collector) -> collector.add(name, spec, true))
+        .generator((name, fullPath, spec, collector) -> collector.add(name, fullPath, spec, true))
         .generate(TEST_CONFIG_FILE_DIR_PATH).stream().map(params -> Arguments.of(params[0], params[1]));
   }
 

ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java

@@ -56,9 +56,9 @@ public class BlockchainReferenceTestTools {
   private static final JsonTestParameters<?, ?> params =
       JsonTestParameters.create(BlockchainReferenceTestCaseSpec.class)
           .generator(
-              (testName, spec, collector) -> {
+              (testName, fullPath, spec, collector) -> {
                 final String eip = spec.getNetwork();
-                collector.add(testName + "[" + eip + "]", spec, NETWORKS_TO_RUN.contains(eip));
+                collector.add(testName + "[" + eip + "]", fullPath, spec, NETWORKS_TO_RUN.contains(eip));
               });
 
   static {
@@ -66,10 +66,6 @@ public class BlockchainReferenceTestTools {
       params.ignoreAll();
     }
 
-    // Known bad test.
-    params.ignore(
-        "RevertPrecompiledTouch(_storage)?_d(0|3)g0v0_(EIP158|Byzantium|Constantinople|ConstantinopleFix)");
-
     // Consumes a huge amount of memory
     params.ignore("static_Call1MB1024Calldepth_d1g0v0_\\w+");
     params.ignore("ShanghaiLove_.*");
@@ -85,6 +81,9 @@ public class BlockchainReferenceTestTools {
     // chain head.
     // Perfectly valid test pre-merge.
     params.ignore("UncleFromSideChain_Merge");
+
+    // EIP tests are explicitly meant to be works-in-progress with known failing tests
+    params.ignore("/EIPTests/");
   }
 
   private BlockchainReferenceTestTools() {

ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java

@@ -69,7 +69,7 @@ public class GeneralStateReferenceTestTools {
   private static final JsonTestParameters<?, ?> params =
       JsonTestParameters.create(GeneralStateTestCaseSpec.class, GeneralStateTestCaseEipSpec.class)
           .generator(
-              (testName, stateSpec, collector) -> {
+              (testName, fullPath, stateSpec, collector) -> {
                 final String prefix = testName + "-";
                 for (final Map.Entry<String, List<GeneralStateTestCaseEipSpec>> entry :
                     stateSpec.finalStateSpecs().entrySet()) {
@@ -77,10 +77,10 @@ public class GeneralStateReferenceTestTools {
                   final boolean runTest = EIPS_TO_RUN.contains(eip);
                   final List<GeneralStateTestCaseEipSpec> eipSpecs = entry.getValue();
                   if (eipSpecs.size() == 1) {
-                    collector.add(prefix + eip, eipSpecs.get(0), runTest);
+                    collector.add(prefix + eip, fullPath, eipSpecs.get(0), runTest);
                   } else {
                     for (int i = 0; i < eipSpecs.size(); i++) {
-                      collector.add(prefix + eip + '[' + i + ']', eipSpecs.get(i), runTest);
+                      collector.add(prefix + eip + '[' + i + ']', fullPath, eipSpecs.get(i), runTest);
                     }
                   }
                 }
@@ -91,13 +91,6 @@ public class GeneralStateReferenceTestTools {
       params.ignoreAll();
     }
 
-    // Known incorrect test.
-    params.ignore(
-        "RevertPrecompiledTouch(_storage)?-(EIP158|Byzantium|Constantinople|ConstantinopleFix)");
-
-    // Gas integer value is too large to construct a valid transaction.
-    params.ignore("OverflowGasRequire");
-
     // Consumes a huge amount of memory
     params.ignore("static_Call1MB1024Calldepth-\\w");
     params.ignore("ShanghaiLove_.*");
@@ -105,6 +98,9 @@ public class GeneralStateReferenceTestTools {
     // Don't do time consuming tests
     params.ignore("CALLBlake2f_MaxRounds.*");
     params.ignore("loopMul-.*");
+
+    // EIP tests are explicitly meant to be works-in-progress with known failing tests
+    params.ignore("/EIPTests/");
   }
 
   private GeneralStateReferenceTestTools() {
@@ -152,13 +148,17 @@ public class GeneralStateReferenceTestTools {
             new BlockHashLookup(blockHeader, blockchain),
             false,
             TransactionValidationParams.processingBlock());
-    if (!result.isInvalid()) {
-      final Account coinbase = worldStateUpdater.getOrCreate(spec.getBlockHeader().getCoinbase());
-      if (coinbase != null && coinbase.isEmpty() && shouldClearEmptyAccounts(spec.getFork())) {
-        worldStateUpdater.deleteAccount(coinbase.getAddress());
-      }
-      worldStateUpdater.commit();
+    if (result.isInvalid()) {
+      assertThat(spec.getExpectException()).isNotNull();
+      return;
+    }
+    assertThat(spec.getExpectException()).withFailMessage("Exception was expected - " + spec.getExpectException()).isNull();
+
+    final Account coinbase = worldStateUpdater.getOrCreate(spec.getBlockHeader().getCoinbase());
+    if (coinbase != null && coinbase.isEmpty() && shouldClearEmptyAccounts(spec.getFork())) {
+      worldStateUpdater.deleteAccount(coinbase.getAddress());
     }
+    worldStateUpdater.commit();
 
     // Check the world state root hash.
     final Hash expectedRootHash = spec.getExpectedRootHash();

testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java

@@ -76,11 +76,14 @@ public class JsonTestParameters<S, T> {
      * Add.
      *
      * @param name the name
+     * @param fullPath the full path of the test
      * @param value the value
      * @param runTest the run test
      */
-    public void add(final String name, final S value, final boolean runTest) {
-      testParameters.add(new Object[] {name, value, runTest && includes(name)});
+    public void add(
+        final String name, final String fullPath, final S value, final boolean runTest) {
+      testParameters.add(
+          new Object[] {name, value, runTest && includes(name) && includes(fullPath)});
     }
 
     private boolean includes(final String name) {
@@ -110,10 +113,11 @@ public class JsonTestParameters<S, T> {
      * Generate.
      *
      * @param name the name
+     * @param fullPath the full path of the test
      * @param mappedType the mapped type
      * @param collector the collector
      */
-    void generate(String name, S mappedType, Collector<T> collector);
+    void generate(String name, String fullPath, S mappedType, Collector<T> collector);
   }
 
   private static final ObjectMapper objectMapper =
@@ -152,7 +156,8 @@ public class JsonTestParameters<S, T> {
    */
   public static <T> JsonTestParameters<T, T> create(final Class<T> testCaseSpec) {
     return new JsonTestParameters<>(testCaseSpec, testCaseSpec)
-        .generator((name, testCase, collector) -> collector.add(name, testCase, true));
+        .generator(
+            (name, fullPath, testCase, collector) -> collector.add(name, fullPath, testCase, true));
   }
 
   /**
@@ -220,16 +225,6 @@ public class JsonTestParameters<S, T> {
     return this;
   }
 
-  /**
-   * Generate collection.
-   *
-   * @param path the path
-   * @return the collection
-   */
-  public Collection<Object[]> generate(final String path) {
-    return generate(new String[] {path});
-  }
-
   /**
    * Generate collection.
    *
@@ -253,7 +248,7 @@ public class JsonTestParameters<S, T> {
       for (final Map.Entry<String, S> entry : testCase.testCaseSpecs.entrySet()) {
         final String testName = entry.getKey();
         final S mappedType = entry.getValue();
-        generator.generate(testName, mappedType, collector);
+        generator.generate(testName, file.getPath(), mappedType, collector);
       }
     }
     return collector.getParameters();