Update reference tests to 12.4 (#5899)

* Update reference tests to 12.4

Update reference tests to 12.4
* Some in-memory storage changed to ConcurrentMap
* exclude cancun from all EIP tests, EIP-4788 still in flux
* Add new fields to ReferenceTestEnv, and re-order the reflected
  constructor for clarity and ease of development.

Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>

* spotless

Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>

---------

Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>
Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>

ethereum/referencetests/build.gradle

@@ -198,7 +198,7 @@ tasks.register('validateReferenceTestSubmodule') {
   description = "Checks that the reference tests submodule is not accidentally changed"
   doLast {
     def result = new ByteArrayOutputStream()
-    def expectedHash = '06e276776bc87817c38f6efb492bf6f4527fa904'
+    def expectedHash = '661356317ac6df52208d54187e692472a25a01f8'
     def submodulePath = java.nio.file.Path.of("${rootProject.projectDir}", "ethereum/referencetests/src/reference-test/external-resources").toAbsolutePath()
     try {
       exec {

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java

@@ -97,33 +97,35 @@ public class ReferenceTestEnv extends BlockHeader {
    */
   @JsonCreator
   public ReferenceTestEnv(
+      @JsonProperty("beaconRoot") final String beaconRoot,
+      @JsonProperty("blockHashes") final Map<String, String> blockHashes,
+      @JsonProperty("ommers") final List<String> _ommers,
+      @JsonProperty("previousHash") final String previousHash,
+      @JsonProperty("withdrawals") final List<EnvWithdrawal> withdrawals,
+      @JsonProperty("currentBaseFee") final String baseFee,
+      @JsonProperty("currentBeaconRoot") final String currentBeaconRoot,
+      @JsonProperty("currentBlobGasUsed") final String currentBlobGasUsed,
       @JsonProperty("currentCoinbase") final String coinbase,
+      @JsonProperty("currentDataGasUsed") final String currentDataGasUsed,
       @JsonProperty("currentDifficulty") final String difficulty,
+      @JsonProperty("currentExcessBlobGas") final String currentExcessBlobGas,
+      @JsonProperty("currentExcessDataGas") final String currentExcessDataGas,
       @JsonProperty("currentGasLimit") final String gasLimit,
       @JsonProperty("currentNumber") final String number,
-      @JsonProperty("currentBaseFee") final String baseFee,
-      @JsonProperty("currentTimestamp") final String timestamp,
       @JsonProperty("currentRandom") final String random,
       @JsonProperty("currentStateRoot") final String stateRoot,
-      @JsonProperty("previousHash") final String previousHash,
-      @JsonProperty("parentDifficulty") final String parentDifficulty,
+      @JsonProperty("currentTimestamp") final String timestamp,
+      @JsonProperty("currentWithdrawalsRoot") final String currentWithdrawalsRoot,
       @JsonProperty("parentBaseFee") final String parentBaseFee,
-      @JsonProperty("parentGasUsed") final String parentGasUsed,
-      @JsonProperty("parentGasLimit") final String parentGasLimit,
-      @JsonProperty("parentTimestamp") final String parentTimestamp,
-      @JsonProperty("ommers") final List<String> _ommers,
-      @JsonProperty("parentUncleHash") final String _parentUncleHash,
-      @JsonProperty("withdrawals") final List<EnvWithdrawal> withdrawals,
-      @JsonProperty("blockHashes") final Map<String, String> blockHashes,
-      @JsonProperty("currentExcessBlobGas") final String currentExcessBlobGas,
-      @JsonProperty("currentBlobGasUsed") final String currentBlobGasUsed,
-      @JsonProperty("currentExcessDataGas") final String currentExcessDataGas,
-      @JsonProperty("currentDataGasUsed") final String currentDataGasUsed,
-      @JsonProperty("parentExcessBlobGas") final String parentExcessBlobGas,
       @JsonProperty("parentBlobGasUsed") final String parentBlobGasUsed,
-      @JsonProperty("parentExcessDataGas") final String parentExcessDataGas,
       @JsonProperty("parentDataGasUsed") final String parentDataGasUsed,
-      @JsonProperty("beaconRoot") final String beaconRoot) {
+      @JsonProperty("parentDifficulty") final String parentDifficulty,
+      @JsonProperty("parentExcessBlobGas") final String parentExcessBlobGas,
+      @JsonProperty("parentExcessDataGas") final String parentExcessDataGas,
+      @JsonProperty("parentGasLimit") final String parentGasLimit,
+      @JsonProperty("parentGasUsed") final String parentGasUsed,
+      @JsonProperty("parentTimestamp") final String parentTimestamp,
+      @JsonProperty("parentUncleHash") final String _parentUncleHash) {
     super(
         generateTestBlockHash(previousHash, number),
         Hash.EMPTY_LIST_HASH, // ommersHash
@@ -141,7 +143,7 @@ public class ReferenceTestEnv extends BlockHeader {
         Optional.ofNullable(baseFee).map(Wei::fromHexString).orElse(null),
         Optional.ofNullable(random).map(Difficulty::fromHexString).orElse(Difficulty.ZERO),
         0L,
-        null, // withdrawalsRoot
+        currentWithdrawalsRoot == null ? null : Hash.fromHexString(currentWithdrawalsRoot),
         currentBlobGasUsed == null
             ? currentDataGasUsed == null ? null : Long.decode(currentDataGasUsed)
             : Long.decode(currentBlobGasUsed),
@@ -172,7 +174,10 @@ public class ReferenceTestEnv extends BlockHeader {
                         Map.entry(
                             Long.decode(entry.getKey()), Hash.fromHexString(entry.getValue())))
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-    this.beaconRoot = beaconRoot == null ? null : Hash.fromHexString(beaconRoot);
+    this.beaconRoot =
+        beaconRoot == null
+            ? (currentBeaconRoot == null ? null : Hash.fromHexString(currentBeaconRoot))
+            : Hash.fromHexString(beaconRoot);
   }
 
   @Override

ethereum/referencetests/src/reference-test/external-resources

@@ -1 +1 @@
-Subproject commit 06e276776bc87817c38f6efb492bf6f4527fa904
+Subproject commit 661356317ac6df52208d54187e692472a25a01f8

ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java

@@ -86,6 +86,9 @@ public class BlockchainReferenceTestTools {
     params.ignore("blobhashListBounds5");
     params.ignore("blockWithAllTransactionTypes");
 
+    // EIP-4788 is still in flux and the current fill is not against the final address
+    params.ignore("[Cancun]");
+
     // EOF tests are written against an older version of the spec
     params.ignore("/stEOF/");
   }

services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java

@@ -18,9 +18,10 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
 
 import java.io.PrintStream;
 import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.ReadWriteLock;
 
 import org.apache.tuweni.bytes.Bytes;
@@ -51,9 +52,10 @@ public class InMemoryKeyValueStorage extends SegmentedKeyValueStorageAdapter {
         }
       };
 
-  private static Map<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> asSegmentMap(
+  private static ConcurrentMap<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> asSegmentMap(
       final Map<Bytes, Optional<byte[]>> initialMap) {
-    final Map<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> segmentMap = new HashMap<>();
+    final ConcurrentMap<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> segmentMap =
+        new ConcurrentHashMap<>();
     segmentMap.put(SEGMENT_IDENTIFIER, initialMap);
     return segmentMap;
   }

services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java

@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.Lock;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -59,7 +60,7 @@ public class LayeredKeyValueStorage extends SegmentedInMemoryKeyValueStorage
    * @param parent the parent key value storage for this layered storage.
    */
   public LayeredKeyValueStorage(
-      final Map<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> map,
+      final ConcurrentMap<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> map,
       final SegmentedKeyValueStorage parent) {
     super(map);
     this.parent = parent;

services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java

@@ -30,6 +30,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -46,14 +48,14 @@ import org.apache.tuweni.bytes.Bytes;
 public class SegmentedInMemoryKeyValueStorage
     implements SnappedKeyValueStorage, SnappableKeyValueStorage, SegmentedKeyValueStorage {
   /** protected access for the backing hash map. */
-  final Map<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> hashValueStore;
+  final ConcurrentMap<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> hashValueStore;
 
   /** protected access to the rw lock. */
   protected final ReadWriteLock rwLock = new ReentrantReadWriteLock();
 
   /** Instantiates a new In memory key value storage. */
   public SegmentedInMemoryKeyValueStorage() {
-    this(new HashMap<>());
+    this(new ConcurrentHashMap<>());
   }
 
   /**
@@ -62,7 +64,7 @@ public class SegmentedInMemoryKeyValueStorage
    * @param hashValueStore the hash value store
    */
   protected SegmentedInMemoryKeyValueStorage(
-      final Map<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> hashValueStore) {
+      final ConcurrentMap<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> hashValueStore) {
     this.hashValueStore = hashValueStore;
   }
 
@@ -76,8 +78,8 @@ public class SegmentedInMemoryKeyValueStorage
         segments.stream()
             .collect(
                 Collectors
-                    .<SegmentIdentifier, SegmentIdentifier, Map<Bytes, Optional<byte[]>>>toMap(
-                        s -> s, s -> new HashMap<>())));
+                    .<SegmentIdentifier, SegmentIdentifier, Map<Bytes, Optional<byte[]>>>
+                        toConcurrentMap(s -> s, s -> new ConcurrentHashMap<>())));
   }
 
   @Override
@@ -214,7 +216,9 @@ public class SegmentedInMemoryKeyValueStorage
     // need to clone the submaps also:
     return new SegmentedInMemoryKeyValueStorage(
         hashValueStore.entrySet().stream()
-            .collect(Collectors.toMap(Map.Entry::getKey, e -> new HashMap<>(e.getValue()))));
+            .collect(
+                Collectors.toConcurrentMap(
+                    Map.Entry::getKey, e -> new ConcurrentHashMap<>(e.getValue()))));
   }
 
   @Override