Enable reference tests for the merge (#4608)

Enable the "Merge" network targeting the "paris" fork.

Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockHeaderValidator.java

@@ -17,14 +17,19 @@ package org.hyperledger.besu.ethereum.mainnet;
 import org.hyperledger.besu.config.MergeConfigOptions;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket;
+import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.AncestryValidationRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.AttachedComposedFromDetachedRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.BaseFeeMarketBlockHeaderGasPriceValidationRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.CalculatedDifficultyValidationRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.ConstantFieldValidationRule;
+import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.ConstantOmmersHashRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.ExtraDataMaxLengthValidationRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.GasLimitRangeAndDeltaValidationRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.GasUsageValidationRule;
+import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.IncrementalTimestampRule;
+import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.NoDifficultyRule;
+import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.NoNonceRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.ProofOfWorkValidationRule;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.TimestampBoundedByFutureParameter;
 import org.hyperledger.besu.ethereum.mainnet.headervalidationrules.TimestampMoreRecentThanParent;
@@ -44,6 +49,10 @@ public final class MainnetBlockHeaderValidator {
   public static final Bytes CLASSIC_FORK_BLOCK_HEADER =
       Bytes.fromHexString("0x94365e3a8c0b35089c1d1195081fe7489b528a84b22199c916180db8b28ade7f");
 
+  private MainnetBlockHeaderValidator() {
+    // utility class
+  }
+
   public static BlockHeaderValidator.Builder create() {
     return createPgaFeeMarketValidator(PoWHasher.ETHASH_LIGHT);
   }
@@ -85,11 +94,6 @@ public final class MainnetBlockHeaderValidator {
         new EpochCalculator.DefaultEpochCalculator(), PoWHasher.ETHASH_LIGHT);
   }
 
-  static BlockHeaderValidator.Builder createLegacyFeeMarketOmmerValidator(final PoWHasher hasher) {
-    return createLegacyFeeMarketOmmerValidator(
-        new EpochCalculator.DefaultEpochCalculator(), hasher);
-  }
-
   static BlockHeaderValidator.Builder createLegacyFeeMarketOmmerValidator(
       final EpochCalculator epochCalculator, final PoWHasher hasher) {
     return new BlockHeaderValidator.Builder()
@@ -177,4 +181,23 @@ public final class MainnetBlockHeaderValidator {
                 Optional.of(baseFeeMarket)))
         .addRule((new BaseFeeMarketBlockHeaderGasPriceValidationRule(baseFeeMarket)));
   }
+
+  public static BlockHeaderValidator.Builder mergeBlockHeaderValidator(final FeeMarket feeMarket) {
+
+    var baseFeeMarket = (BaseFeeMarket) feeMarket;
+
+    return new BlockHeaderValidator.Builder()
+        .addRule(new AncestryValidationRule())
+        .addRule(new GasUsageValidationRule())
+        .addRule(
+            new GasLimitRangeAndDeltaValidationRule(
+                MIN_GAS_LIMIT, Long.MAX_VALUE, Optional.of(baseFeeMarket)))
+        .addRule(new TimestampBoundedByFutureParameter(TIMESTAMP_TOLERANCE_S))
+        .addRule(new ExtraDataMaxLengthValidationRule(BlockHeader.MAX_EXTRA_DATA_BYTES))
+        .addRule((new BaseFeeMarketBlockHeaderGasPriceValidationRule(baseFeeMarket)))
+        .addRule(new ConstantOmmersHashRule())
+        .addRule(new NoNonceRule())
+        .addRule(new NoDifficultyRule())
+        .addRule(new IncrementalTimestampRule());
+  }
 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetDifficultyCalculators.java

@@ -98,6 +98,10 @@ public abstract class MainnetDifficultyCalculators {
       (time, parent, protocolContext) ->
           calculateThawedDifficulty(time, parent, GRAY_GLACIER_FAKE_BLOCK_OFFSET);
 
+  // Proof-of-Stake difficulty must not be altered
+  static final DifficultyCalculator PROOF_OF_STAKE_DIFFICULTY =
+      (time, parent, protocolContext) -> parent.getDifficulty().getAsBigInteger();
+
   private static BigInteger calculateThawedDifficulty(
       final long time, final BlockHeader parent, final long fakeBlockOffset) {
     final BigInteger parentDifficulty = difficulty(parent.getDifficulty());

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java

@@ -79,8 +79,6 @@ public abstract class MainnetProtocolSpecs {
 
   public static final int SPURIOUS_DRAGON_CONTRACT_SIZE_LIMIT = 24576;
 
-  public static final String LONDON_FORK_NAME = "London";
-
   private static final Address RIPEMD160_PRECOMPILE =
       Address.fromHexString("0x0000000000000000000000000000000000000003");
 
@@ -567,7 +565,7 @@ public abstract class MainnetProtocolSpecs {
             feeMarket ->
                 MainnetBlockHeaderValidator.createBaseFeeMarketOmmerValidator(londonFeeMarket))
         .blockBodyValidatorBuilder(BaseFeeBlockBodyValidator::new)
-        .name(LONDON_FORK_NAME);
+        .name("London");
   }
 
   static ProtocolSpecBuilder arrowGlacierDefinition(
@@ -630,6 +628,9 @@ public abstract class MainnetProtocolSpecs {
         .evmBuilder(
             (gasCalculator, jdCacheConfig) ->
                 MainnetEVMs.paris(gasCalculator, chainId.orElse(BigInteger.ZERO), evmConfiguration))
+        .difficultyCalculator(MainnetDifficultyCalculators.PROOF_OF_STAKE_DIFFICULTY)
+        .blockHeaderValidatorBuilder(MainnetBlockHeaderValidator::mergeBlockHeaderValidator)
+        .blockReward(Wei.ZERO)
         .name("ParisFork");
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/headervalidationrules/ConstantOmmersHashRule.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.hyperledger.besu.ethereum.mainnet.headervalidationrules;
+
+import org.hyperledger.besu.datatypes.Hash;
+import org.hyperledger.besu.ethereum.ProtocolContext;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.ethereum.mainnet.AttachedBlockHeaderValidationRule;
+
+public class ConstantOmmersHashRule implements AttachedBlockHeaderValidationRule {
+
+  private static final Hash mergeConstant = Hash.EMPTY_LIST_HASH;
+
+  @Override
+  public boolean validate(
+      final BlockHeader header, final BlockHeader parent, final ProtocolContext protocolContext) {
+    return header.getOmmersHash().equals(mergeConstant);
+  }
+
+  @Override
+  public boolean includeInLightValidation() {
+    return AttachedBlockHeaderValidationRule.super.includeInLightValidation();
+  }
+}

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/headervalidationrules/IncrementalTimestampRule.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.hyperledger.besu.ethereum.mainnet.headervalidationrules;
+
+import org.hyperledger.besu.ethereum.ProtocolContext;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.ethereum.mainnet.AttachedBlockHeaderValidationRule;
+
+public class IncrementalTimestampRule implements AttachedBlockHeaderValidationRule {
+
+  @Override
+  public boolean validate(
+      final BlockHeader header, final BlockHeader parent, final ProtocolContext protocolContext) {
+
+    final long blockTimestamp = header.getTimestamp();
+    final long parentTimestamp = parent.getTimestamp();
+
+    return blockTimestamp > parentTimestamp;
+  }
+}

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/headervalidationrules/NoDifficultyRule.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.hyperledger.besu.ethereum.mainnet.headervalidationrules;
+
+import org.hyperledger.besu.ethereum.ProtocolContext;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.ethereum.core.Difficulty;
+import org.hyperledger.besu.ethereum.mainnet.AttachedBlockHeaderValidationRule;
+
+public class NoDifficultyRule implements AttachedBlockHeaderValidationRule {
+
+  @Override
+  public boolean validate(
+      final BlockHeader header, final BlockHeader parent, final ProtocolContext protocolContext) {
+    return (header.getDifficulty() == null || header.getDifficulty().equals(Difficulty.ZERO));
+  }
+}

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/headervalidationrules/NoNonceRule.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.hyperledger.besu.ethereum.mainnet.headervalidationrules;
+
+import org.hyperledger.besu.ethereum.ProtocolContext;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.ethereum.mainnet.AttachedBlockHeaderValidationRule;
+
+public class NoNonceRule implements AttachedBlockHeaderValidationRule {
+
+  @Override
+  public boolean validate(
+      final BlockHeader header, final BlockHeader parent, final ProtocolContext protocolContext) {
+    return header.getNonce() == 0L;
+  }
+}

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/operations/PrevRanDaoOperationTest.java

@@ -49,7 +49,7 @@ public class PrevRanDaoOperationTest {
   }
 
   @Test
-  public void pushesDifficultyWhenPresent() {
+  public void pushesPrevRandDaoWhenDifficultyPresent() {
     PrevRanDaoOperation op = new PrevRanDaoOperation(new LondonGasCalculator());
     MessageFrame messageFrame = mock(MessageFrame.class);
     BlockValues blockHeader = mock(BlockValues.class);
@@ -61,6 +61,6 @@ public class PrevRanDaoOperationTest {
     EVM evm = mock(EVM.class);
     Operation.OperationResult r = op.executeFixedCostOperation(messageFrame, evm);
     assertThat(r.getHaltReason()).isNull();
-    verify(messageFrame).pushStackItem(difficulty);
+    verify(messageFrame).pushStackItem(prevRandao);
   }
 }

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseSpec.java

@@ -18,7 +18,6 @@ package org.hyperledger.besu.ethereum.referencetests;
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.Transaction;
-import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSpecs;
 
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
@@ -32,7 +31,6 @@ import javax.annotation.Nullable;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.tuweni.units.bigints.UInt256;
 
 /** A Transaction test case specification. */
 @JsonIgnoreProperties(ignoreUnknown = true)
@@ -56,25 +54,11 @@ public class GeneralStateTestCaseSpec {
       final Map<String, List<PostSection>> postSections,
       final StateTestVersionedTransaction versionedTransaction) {
 
-    BlockHeader safeHeader = blockHeader;
     initialWorldState.persist(null);
     final Map<String, List<GeneralStateTestCaseEipSpec>> res =
         new LinkedHashMap<>(postSections.size());
     for (final Map.Entry<String, List<PostSection>> entry : postSections.entrySet()) {
       final String eip = entry.getKey();
-      if (eip.equalsIgnoreCase(MainnetProtocolSpecs.LONDON_FORK_NAME)
-          && !blockHeader.getBaseFee().isPresent()) {
-        // for legacy state tests running in London, this value depends on
-        // the test filler's (retesteth's) default baseFee, currently 0x0a (10)
-        safeHeader =
-            new ReferenceTestEnv(
-                blockHeader.getCoinbase().toShortHexString(),
-                blockHeader.getDifficulty().toShortHexString(),
-                UInt256.valueOf(blockHeader.getGasLimit()).toShortHexString(),
-                UInt256.valueOf(blockHeader.getNumber()).toShortHexString(),
-                "0x0a",
-                UInt256.valueOf(blockHeader.getTimestamp()).toShortHexString());
-      }
       final List<PostSection> post = entry.getValue();
       final List<GeneralStateTestCaseEipSpec> specs = new ArrayList<>(post.size());
       for (final PostSection p : post) {
@@ -86,7 +70,7 @@ public class GeneralStateTestCaseSpec {
                 initialWorldState,
                 p.rootHash,
                 p.logsHash,
-                safeHeader,
+                blockHeader,
                 p.indexes.data,
                 p.indexes.gas,
                 p.indexes.value,

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java

@@ -46,15 +46,17 @@ public class ReferenceTestEnv extends BlockHeader {
    * @param number Block number for the mock block being tested.
    * @param baseFee Optional BaseFee for the mock block being tested.
    * @param timestamp Timestamp for the mock block being tested.
+   * @param random Optional RANDAO or the mock block being tested.
    */
   @JsonCreator
   public ReferenceTestEnv(
       @JsonProperty("currentCoinbase") final String coinbase,
-      @JsonProperty(value = "currentDifficulty", required = false) final String difficulty,
+      @JsonProperty("currentDifficulty") final String difficulty,
       @JsonProperty("currentGasLimit") final String gasLimit,
       @JsonProperty("currentNumber") final String number,
-      @JsonProperty(value = "currentBaseFee", required = false) final String baseFee,
+      @JsonProperty("currentBaseFee") final String baseFee,
-      @JsonProperty("currentTimestamp") final String timestamp) {
+      @JsonProperty("currentTimestamp") final String timestamp,
+      @JsonProperty("currentRandom") final String random) {
     super(
         generateTestBlockHash(Long.decode(number) - 1),
         Hash.EMPTY, // ommersHash
@@ -70,7 +72,7 @@ public class ReferenceTestEnv extends BlockHeader {
         Long.decode(timestamp),
         Bytes.EMPTY,
         Optional.ofNullable(baseFee).map(Wei::fromHexString).orElse(null),
-        Hash.ZERO,
+        Optional.ofNullable(random).map(Difficulty::fromHexString).orElse(Difficulty.ZERO),
         0L,
         new MainnetBlockHeaderFunctions());
   }

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java

@@ -64,12 +64,15 @@ public class ReferenceTestProtocolSchedules {
     builder.put("Istanbul", createSchedule(new StubGenesisConfigOptions().istanbulBlock(0)));
     builder.put("MuirGlacier", createSchedule(new StubGenesisConfigOptions().muirGlacierBlock(0)));
     builder.put("Berlin", createSchedule(new StubGenesisConfigOptions().berlinBlock(0)));
-    builder.put("London", createSchedule(new StubGenesisConfigOptions().londonBlock(0)));
+    builder.put(
+        "London",
+        createSchedule(new StubGenesisConfigOptions().londonBlock(0).baseFeePerGas(0x0a)));
     builder.put(
         "ArrowGlacier", createSchedule(new StubGenesisConfigOptions().arrowGlacierBlock(0)));
     builder.put("GrayGlacier", createSchedule(new StubGenesisConfigOptions().grayGlacierBlock(0)));
     builder.put(
-        "MergeNetSplit", createSchedule(new StubGenesisConfigOptions().mergeNetSplitBlock(0)));
+        "Merge",
+        createSchedule(new StubGenesisConfigOptions().mergeNetSplitBlock(0).baseFeePerGas(0x0a)));
     builder.put("Shandong", createSchedule(new StubGenesisConfigOptions().shandongBlock(0)));
     return new ReferenceTestProtocolSchedules(builder.build());
   }

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/StateTestVersionedTransaction.java

@@ -100,7 +100,7 @@ public class StateTestVersionedTransaction {
       @JsonDeserialize(using = StateTestAccessListDeserializer.class) @JsonProperty("accessLists")
           final List<List<AccessListEntry>> maybeAccessLists) {
 
-    this.nonce = Long.decode(nonce);
+    this.nonce = Bytes.fromHexString(nonce).toLong();
     this.gasPrice = Optional.ofNullable(gasPrice).map(Wei::fromHexString).orElse(null);
     this.maxFeePerGas = Optional.ofNullable(maxFeePerGas).map(Wei::fromHexString).orElse(null);
     this.maxPriorityFeePerGas =

ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java

@@ -49,7 +49,7 @@ public class BlockchainReferenceTestTools {
             "test.ethereum.blockchain.eips",
             "FrontierToHomesteadAt5,HomesteadToEIP150At5,HomesteadToDaoAt5,EIP158ToByzantiumAt5,"
                 + "Frontier,Homestead,EIP150,EIP158,Byzantium,Constantinople,ConstantinopleFix,Istanbul,Berlin,"
-                + "London,Shanghai,Cancun");
+                + "London,Merge,Shanghai,Cancun,Prague,Osaka,Bogota");
     NETWORKS_TO_RUN = Arrays.asList(networks.split(","));
   }
 
@@ -79,6 +79,15 @@ public class BlockchainReferenceTestTools {
 
     // Don't do time consuming tests
     params.ignore("CALLBlake2f_MaxRounds.*");
+    params.ignore("loopMul_*");
+
+    // Inconclusive fork choice rule, since in merge CL should be choosing forks and setting the chain head.
+    // Perfectly valid test pre-merge.
+    params.ignore("UncleFromSideChain_Merge");
+  }
+
+  private BlockchainReferenceTestTools() {
+    // utility class
   }
 
   public static Collection<Object[]> generateTestParametersForConfig(final String[] filePath) {

ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java

@@ -61,7 +61,7 @@ public class GeneralStateReferenceTestTools {
         System.getProperty(
             "test.ethereum.state.eips",
             "Frontier,Homestead,EIP150,EIP158,Byzantium,Constantinople,ConstantinopleFix,Istanbul,Berlin,"
-                + "London,Shanghai,Cancun");
+                + "London,Merge,Shanghai,Cancun,Prague,Osaka,Bogota");
     EIPS_TO_RUN = Arrays.asList(eips.split(","));
   }
 
@@ -103,6 +103,11 @@ public class GeneralStateReferenceTestTools {
 
     // Don't do time consuming tests
     params.ignore("CALLBlake2f_MaxRounds.*");
+    params.ignore("loopMul-.*");
+  }
+
+  private GeneralStateReferenceTestTools() {
+    // utility class
   }
 
   public static Collection<Object[]> generateTestParametersForConfig(final String[] filePath) {

evm/src/main/java/org/hyperledger/besu/evm/operation/PrevRanDaoOperation.java

@@ -18,8 +18,6 @@ import org.hyperledger.besu.evm.EVM;
 import org.hyperledger.besu.evm.frame.MessageFrame;
 import org.hyperledger.besu.evm.gascalculator.GasCalculator;
 
-import org.apache.tuweni.units.bigints.UInt256;
-
 public class PrevRanDaoOperation extends AbstractFixedCostOperation {
 
   public PrevRanDaoOperation(final GasCalculator gasCalculator) {
@@ -28,12 +26,7 @@ public class PrevRanDaoOperation extends AbstractFixedCostOperation {
 
   @Override
   public OperationResult executeFixedCostOperation(final MessageFrame frame, final EVM evm) {
-    if (frame.getBlockValues().getDifficultyBytes() == null
+    frame.pushStackItem(frame.getBlockValues().getMixHashOrPrevRandao());
-        || frame.getBlockValues().getDifficultyBytes().equals(UInt256.ZERO)) {
-      frame.pushStackItem(frame.getBlockValues().getMixHashOrPrevRandao());
-    } else {
-      frame.pushStackItem(frame.getBlockValues().getDifficultyBytes());
-    }
     return successResponse;
   }
 }