Apply Reference tests v13.1 (#6574)

Update to new reference tests for cancun.
* Init KZG as part of ReferenceTestProtocolSchedules
* fix unsigned long errors
* block validation on borked blobGasUsed

Signed-off-by: Justin Florentine <justin+github@florentine.us>
Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>
Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com>
Co-authored-by: Justin Florentine <justin+github@florentine.us>
Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>

CHANGELOG.md

@@ -43,6 +43,7 @@
 - More accurate column size `storage rocksdb usage` subcommand [#6540](https://github.com/hyperledger/besu/pull/6540)
 - Adds `storage rocksdb x-stats` subcommand [#6540](https://github.com/hyperledger/besu/pull/6540)
 - New `eth_blobBaseFee`JSON-RPC method [#6581](https://github.com/hyperledger/besu/pull/6581)
+- Upgrade reference tests to version 13.1 [#6574](https://github.com/hyperledger/besu/pull/6574)
 
 ### Bug fixes
 - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225)

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/BlockHeader.java

@@ -102,11 +102,6 @@ public class BlockHeader extends SealableBlockHeader
     return Hash.wrap(mixHashOrPrevRandao);
   }
 
-  @Override
-  public Bytes32 getMixHashOrPrevRandao() {
-    return mixHashOrPrevRandao;
-  }
-
   /**
    * Returns the block nonce.
    *
@@ -207,7 +202,7 @@ public class BlockHeader extends SealableBlockHeader
             : null;
     final Long blobGasUsed = !input.isEndOfCurrentList() ? input.readLongScalar() : null;
     final BlobGas excessBlobGas =
-        !input.isEndOfCurrentList() ? BlobGas.of(input.readLongScalar()) : null;
+        !input.isEndOfCurrentList() ? BlobGas.of(input.readUInt64Scalar()) : null;
     final Bytes32 parentBeaconBlockRoot = !input.isEndOfCurrentList() ? input.readBytes32() : null;
     final Hash depositHashRoot =
         !input.isEndOfCurrentList() ? Hash.wrap(input.readBytes32()) : null;
@@ -242,10 +237,9 @@ public class BlockHeader extends SealableBlockHeader
     if (obj == this) {
       return true;
     }
-    if (!(obj instanceof BlockHeader)) {
+    if (!(obj instanceof BlockHeader other)) {
       return false;
     }
-    final BlockHeader other = (BlockHeader) obj;
     return getHash().equals(other.getHash());
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java

@@ -35,6 +35,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState;
 import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
 import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
 import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup;
+import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator;
 import org.hyperledger.besu.evm.tracing.OperationTracer;
 import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
@@ -101,6 +102,7 @@ public abstract class AbstractBlockProcessor implements BlockProcessor {
       final PrivateMetadataUpdater privateMetadataUpdater) {
     final List<TransactionReceipt> receipts = new ArrayList<>();
     long currentGasUsed = 0;
+    long currentBlobGasUsed = 0;
 
     final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(blockHeader);
 
@@ -127,7 +129,7 @@ public abstract class AbstractBlockProcessor implements BlockProcessor {
       Wei blobGasPrice =
           maybeParentHeader
               .map(
-                  (parentHeader) ->
+                  parentHeader ->
                       protocolSpec
                           .getFeeMarket()
                           .blobGasPricePerGas(
@@ -163,12 +165,25 @@ public abstract class AbstractBlockProcessor implements BlockProcessor {
       worldStateUpdater.commit();
 
       currentGasUsed += transaction.getGasLimit() - result.getGasRemaining();
+      if (transaction.getVersionedHashes().isPresent()) {
+        currentBlobGasUsed +=
+            (transaction.getVersionedHashes().get().size() * CancunGasCalculator.BLOB_GAS_PER_BLOB);
+      }
+
       final TransactionReceipt transactionReceipt =
           transactionReceiptFactory.create(
               transaction.getType(), result, worldState, currentGasUsed);
       receipts.add(transactionReceipt);
     }
-
+    if (blockHeader.getBlobGasUsed().isPresent()
+        && currentBlobGasUsed != blockHeader.getBlobGasUsed().get()) {
+      String errorMessage =
+          String.format(
+              "block did not consume expected blob gas: header %d, transactions %d",
+              blockHeader.getBlobGasUsed().get(), currentBlobGasUsed);
+      LOG.error(errorMessage);
+      return new BlockProcessingResult(Optional.empty(), errorMessage);
+    }
     final Optional<WithdrawalsProcessor> maybeWithdrawalsProcessor =
         protocolSpec.getWithdrawalsProcessor();
     if (maybeWithdrawalsProcessor.isPresent() && maybeWithdrawals.isPresent()) {

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/headervalidationrules/BlobGasValidationRule.java

@@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.mainnet.headervalidationrules;
 import org.hyperledger.besu.datatypes.BlobGas;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.mainnet.DetachedBlockHeaderValidationRule;
+import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator;
 import org.hyperledger.besu.evm.gascalculator.GasCalculator;
 
 import org.slf4j.Logger;
@@ -35,7 +36,7 @@ public class BlobGasValidationRule implements DetachedBlockHeaderValidationRule
 
   /**
    * Validates the block header by checking if the header's excess blob gas matches the calculated
-   * value based on the parent header.
+   * value based on the parent header, as well that the used blobGas is a multiple of GAS_PER_BLOB.
    */
   @Override
   public boolean validate(final BlockHeader header, final BlockHeader parent) {
@@ -53,6 +54,13 @@ public class BlobGasValidationRule implements DetachedBlockHeaderValidationRule
           calculatedExcessBlobGas);
       return false;
     }
+    long headerBlobGasUsed = header.getBlobGasUsed().orElse(0L);
+    if (headerBlobGasUsed % CancunGasCalculator.BLOB_GAS_PER_BLOB != 0) {
+      LOG.info(
+          "blob gas used must be multiple of GAS_PER_BLOB ({})",
+          CancunGasCalculator.BLOB_GAS_PER_BLOB);
+      return false;
+    }
     return true;
   }
 }

ethereum/referencetests/build.gradle

@@ -198,7 +198,7 @@ tasks.register('validateReferenceTestSubmodule') {
   description = "Checks that the reference tests submodule is not accidentally changed"
   doLast {
     def result = new ByteArrayOutputStream()
-    def expectedHash = '428f218d7d6f4a52544e12684afbfe6e2882ffbf'
+    def expectedHash = '853b1e03b1078d370614002851ba1ee9803d9fcf '
     def submodulePath = java.nio.file.Path.of("${rootProject.projectDir}", "ethereum/referencetests/src/reference-test/external-resources").toAbsolutePath()
     try {
       exec {

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java

@@ -17,6 +17,7 @@
 package org.hyperledger.besu.ethereum.referencetests;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.hyperledger.besu.evm.internal.Words.decodeUnsignedLong;
 
 import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.datatypes.BlobGas;
@@ -138,7 +139,7 @@ public class ReferenceTestEnv extends BlockHeader {
         number == null ? 0 : Long.decode(number),
         gasLimit == null ? 15_000_000L : Long.decode(gasLimit),
         0L,
-        timestamp == null ? 0L : Long.decode(timestamp),
+        timestamp == null ? 0L : decodeUnsignedLong(timestamp),
         Bytes.EMPTY,
         Optional.ofNullable(baseFee).map(Wei::fromHexString).orElse(null),
         Optional.ofNullable(random).map(Difficulty::fromHexString).orElse(Difficulty.ZERO),

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java

@@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolScheduleBuilder;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpecAdapters;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
+import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract;
 
 import java.math.BigInteger;
 import java.util.Arrays;
@@ -78,6 +79,8 @@ public class ReferenceTestProtocolSchedules {
         "ShanghaiToCancunAtTime15k",
         createSchedule(genesisStub.clone().shanghaiTime(0).cancunTime(15000)));
     builder.put("Cancun", createSchedule(genesisStub.clone().cancunTime(0)));
+    // also load KZG file for mainnet
+    KZGPointEvalPrecompiledContract.init();
     builder.put("Prague", createSchedule(genesisStub.clone().pragueTime(0)));
     builder.put("Future_EIPs", createSchedule(genesisStub.clone().futureEipsTime(0)));
     builder.put("Experimental_EIPs", createSchedule(genesisStub.clone().experimentalEipsTime(0)));

ethereum/referencetests/src/reference-test/external-resources

@@ -1 +1 @@
-Subproject commit 428f218d7d6f4a52544e12684afbfe6e2882ffbf
+Subproject commit 853b1e03b1078d370614002851ba1ee9803d9fcf

evm/src/main/java/org/hyperledger/besu/evm/internal/Words.java

@@ -16,6 +16,8 @@ package org.hyperledger.besu.evm.internal;
 
 import org.hyperledger.besu.datatypes.Address;
 
+import java.math.BigInteger;
+
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.bytes.MutableBytes;
@@ -213,4 +215,23 @@ public interface Words {
         (byte) (value >>> 8),
         (byte) value);
   }
+
+  static long decodeUnsignedLong(final String number) {
+    String parsable = number;
+    int radix = 10;
+    if (number.startsWith("0x")) {
+      radix = 16;
+      parsable = number.substring(2);
+    } else if (!number.matches("\\d+")) {
+      // presume naked hex
+      radix = 16;
+    }
+
+    BigInteger bi = new BigInteger(parsable, radix);
+    if (bi.bitCount() > 64) {
+      throw new NumberFormatException("Number larger than uint64");
+    }
+
+    return bi.longValue();
+  }
 }