use reference tests to verify trielog generation (#6415)

use reference tests to validate trielog generation. Like that we will validate besu trielog generation for all of the tests we have in the references tests suite for past and future EIPs

---------

Signed-off-by: Karim Taam <karim.t2am@gmail.com>

.circleci/config.yml

@@ -204,7 +204,7 @@ jobs:
           at: ~/project
       - run:
           name: ReferenceTests
-          no_output_timeout: 30m
+          no_output_timeout: 40m
           command: |
             git submodule update --init --recursive
             ./gradlew --no-daemon referenceTest

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java

@@ -42,6 +42,17 @@ public class BonsaiValue<T> implements TrieLog.LogTuple<T> {
     this.clearedAtLeastOnce = lastStepCleared;
   }
 
+  public BonsaiValue(
+      final T prior,
+      final T updated,
+      final boolean lastStepCleared,
+      final boolean clearedAtLeastOnce) {
+    this.prior = prior;
+    this.updated = updated;
+    this.lastStepCleared = lastStepCleared;
+    this.clearedAtLeastOnce = clearedAtLeastOnce;
+  }
+
   @Override
   public T getPrior() {
     return prior;
@@ -117,4 +128,8 @@ public class BonsaiValue<T> implements TrieLog.LogTuple<T> {
         .append(lastStepCleared)
         .toHashCode();
   }
+
+  public BonsaiValue<T> copy() {
+    return new BonsaiValue<T>(prior, updated, lastStepCleared, clearedAtLeastOnce);
+  }
 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java

@@ -119,12 +119,12 @@ public class BonsaiWorldState
   }
 
   /**
-   * Having a protected method to override the accumulator solves the chicken-egg problem of needing
+   * Override the accumulator solves the chicken-egg problem of needing a worldstate reference
-   * a worldstate reference (this) when construction the Accumulator.
+   * (this) when construction the Accumulator.
    *
    * @param accumulator accumulator to use.
    */
-  protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) {
+  public void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) {
     this.accumulator = accumulator;
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java

@@ -60,13 +60,13 @@ public class BonsaiWorldStateUpdateAccumulator
     implements BonsaiWorldView, TrieLogAccumulator {
   private static final Logger LOG =
       LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class);
-  private final Consumer<BonsaiValue<BonsaiAccount>> accountPreloader;
+  protected final Consumer<BonsaiValue<BonsaiAccount>> accountPreloader;
-  private final Consumer<StorageSlotKey> storagePreloader;
+  protected final Consumer<StorageSlotKey> storagePreloader;
 
   private final AccountConsumingMap<BonsaiValue<BonsaiAccount>> accountsToUpdate;
   private final Map<Address, BonsaiValue<Bytes>> codeToUpdate = new ConcurrentHashMap<>();
   private final Set<Address> storageToClear = Collections.synchronizedSet(new HashSet<>());
-  private final EvmConfiguration evmConfiguration;
+  protected final EvmConfiguration evmConfiguration;
 
   // storage sub mapped by _hashed_ key.  This is because in self_destruct calls we need to
   // enumerate the old storage and delete it.  Those are trie stored by hashed key by spec and the
@@ -74,7 +74,7 @@ public class BonsaiWorldStateUpdateAccumulator
   private final Map<Address, StorageConsumingMap<StorageSlotKey, BonsaiValue<UInt256>>>
       storageToUpdate = new ConcurrentHashMap<>();
 
-  private boolean isAccumulatorStateChanged;
+  protected boolean isAccumulatorStateChanged;
 
   public BonsaiWorldStateUpdateAccumulator(
       final BonsaiWorldView world,

ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java

@@ -48,7 +48,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
-import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState;
+import org.hyperledger.besu.ethereum.referencetests.ForestReferenceTestWorldState;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
 
@@ -154,7 +154,7 @@ public class BackwardSyncContextTest {
                       new BlockProcessingOutputs(
                           // use forest-based worldstate since it does not require
                           // blockheader stateroot to match actual worldstate root
-                          DefaultReferenceTestWorldState.create(Collections.emptyMap()),
+                          ForestReferenceTestWorldState.create(Collections.emptyMap()),
                           blockDataGenerator.receipts(block))));
             });
 

ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java

@@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer;
 import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
-import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState;
+import org.hyperledger.besu.ethereum.referencetests.ForestReferenceTestWorldState;
 import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
 
 import java.nio.charset.StandardCharsets;
@@ -138,7 +138,7 @@ public class ForwardSyncStepTest {
               return new BlockProcessingResult(
                   Optional.of(
                       new BlockProcessingOutputs(
-                          DefaultReferenceTestWorldState.create(Collections.emptyMap()),
+                          ForestReferenceTestWorldState.create(Collections.emptyMap()),
                           blockDataGenerator.receipts(block))));
             });
   }

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java

@@ -23,7 +23,10 @@ import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdat
 import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 
+import java.util.concurrent.ConcurrentHashMap;
+
 import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.units.bigints.UInt256;
 
 public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdateAccumulator {
   private final BonsaiPreImageProxy preImageProxy;
@@ -43,4 +46,29 @@ public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdate
     // by default do not save hash preImages
     return preImageProxy.hashAndSavePreImage(bytes);
   }
+
+  public BonsaiReferenceTestUpdateAccumulator createDetachedAccumulator() {
+    final BonsaiReferenceTestUpdateAccumulator copy =
+        new BonsaiReferenceTestUpdateAccumulator(
+            wrappedWorldView(),
+            accountPreloader,
+            storagePreloader,
+            preImageProxy,
+            evmConfiguration);
+    getAccountsToUpdate().forEach((k, v) -> copy.getAccountsToUpdate().put(k, v.copy()));
+    getCodeToUpdate().forEach((k, v) -> copy.getCodeToUpdate().put(k, v.copy()));
+    copy.getStorageToClear().addAll(getStorageToClear());
+    getStorageToUpdate()
+        .forEach(
+            (k, v) -> {
+              StorageConsumingMap<StorageSlotKey, BonsaiValue<UInt256>> newMap =
+                  new StorageConsumingMap<>(k, new ConcurrentHashMap<>(), v.getConsumer());
+              v.forEach((key, value) -> newMap.put(key, value.copy()));
+              copy.getStorageToUpdate().put(k, newMap);
+            });
+    copy.updatedAccounts.putAll(updatedAccounts);
+    copy.deletedAccounts.addAll(deletedAccounts);
+    copy.isAccumulatorStateChanged = true;
+    return copy;
+  }
 }

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java

@@ -23,19 +23,25 @@ import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager
 import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager;
 import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy;
 import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
-import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager;
+import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage;
+import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent;
 import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager;
 import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState;
+import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
+import org.hyperledger.besu.plugin.services.trielogs.TrieLog;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Stream;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 
@@ -98,6 +104,99 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState
     // The test harness validates the root hash, no need to validate in-line for reference test
   }
 
+  @Override
+  public void processExtraStateStorageFormatValidation(final BlockHeader blockHeader) {
+    if (blockHeader != null) {
+      final Hash parentStateRoot = getWorldStateRootHash();
+      final BonsaiReferenceTestUpdateAccumulator originalUpdater =
+          ((BonsaiReferenceTestUpdateAccumulator) updater()).createDetachedAccumulator();
+
+      // validate trielog generation with persisted state
+      validateStateRolling(parentStateRoot, originalUpdater, blockHeader, false);
+      // validate trielog generation with frozen state
+      validateStateRolling(parentStateRoot, originalUpdater, blockHeader, true);
+    }
+  }
+
+  /**
+   * TrieLog is an important part of Bonsai, so it's important to verify the generation of the
+   * TrieLog by performing rollbacks and rollforwards.
+   *
+   * @param blockHeader header of the block to import
+   */
+  private void validateStateRolling(
+      final Hash parentStateRoot,
+      final BonsaiReferenceTestUpdateAccumulator originalUpdater,
+      final BlockHeader blockHeader,
+      final boolean isFrozenState) {
+    // With Bonsai, a TrieLog is generated when the state is persisted. Therefore, we generate the
+    // TrieLog by triggering a state persist in order to closely match the real case scenario.
+    generateTrieLogFromState(blockHeader, originalUpdater, isFrozenState);
+    final TrieLog trieLogFromFrozenState =
+        trieLogManager
+            .getTrieLogLayer(blockHeader.getBlockHash())
+            .orElseThrow(() -> new RuntimeException("trielog not found during test"));
+    // trying rollback rollfoward with frozen state
+    validateTrieLog(parentStateRoot, blockHeader, trieLogFromFrozenState);
+  }
+
+  private void validateTrieLog(
+      final Hash parentStateRoot, final BlockHeader blockHeader, final TrieLog trieLog) {
+
+    try (var bonsaiWorldState = createBonsaiWorldState(false)) {
+      BonsaiWorldStateUpdateAccumulator updaterForState =
+          (BonsaiWorldStateUpdateAccumulator) bonsaiWorldState.updater();
+      updaterForState.rollForward(trieLog);
+      updaterForState.commit();
+      bonsaiWorldState.persist(blockHeader);
+      Hash generatedRootHash = bonsaiWorldState.rootHash();
+      if (!bonsaiWorldState.rootHash().equals(blockHeader.getStateRoot())) {
+        throw new RuntimeException(
+            "state root becomes invalid following a rollForward %s != %s"
+                .formatted(blockHeader.getStateRoot(), generatedRootHash));
+      }
+
+      updaterForState = (BonsaiWorldStateUpdateAccumulator) bonsaiWorldState.updater();
+      updaterForState.rollBack(trieLog);
+      updaterForState.commit();
+      bonsaiWorldState.persist(null);
+      generatedRootHash = bonsaiWorldState.rootHash();
+      if (!bonsaiWorldState.rootHash().equals(parentStateRoot)) {
+        throw new RuntimeException(
+            "state root becomes invalid following a rollBackward %s != %s"
+                .formatted(parentStateRoot, generatedRootHash));
+      }
+    }
+  }
+
+  private void generateTrieLogFromState(
+      final BlockHeader blockHeader,
+      final BonsaiReferenceTestUpdateAccumulator originalUpdater,
+      final boolean isFrozen) {
+    // generate trielog
+    BonsaiReferenceTestUpdateAccumulator updaterForState =
+        originalUpdater.createDetachedAccumulator();
+    try (var bonsaiWorldState = createBonsaiWorldState(isFrozen)) {
+      bonsaiWorldState.setAccumulator(updaterForState);
+      updaterForState.commit();
+      bonsaiWorldState.persist(blockHeader);
+    }
+  }
+
+  private BonsaiWorldState createBonsaiWorldState(final boolean isFrozen) {
+    BonsaiWorldState bonsaiWorldState =
+        new BonsaiWorldState(
+            new BonsaiWorldStateLayerStorage(worldStateStorage),
+            cachedMerkleTrieLoader,
+            cachedWorldStorageManager,
+            trieLogManager,
+            evmConfiguration);
+    if (isFrozen) {
+      bonsaiWorldState.freeze(); // freeze state
+    }
+    return bonsaiWorldState;
+  }
+
   @JsonCreator
   public static BonsaiReferenceTestWorldState create(
       final Map<String, ReferenceTestWorldState.AccountMock> accounts) {
@@ -110,7 +209,7 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState
       final EvmConfiguration evmConfiguration) {
     final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem();
     final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem);
-    final TrieLogManager trieLogManager = new NoOpTrieLogManager();
+    final TrieLogManager trieLogManager = new ReferenceTestsInMemoryTrieLogManager();
     final BonsaiPreImageProxy preImageProxy =
         new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy();
 
@@ -149,6 +248,40 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState
     return this.refTestStorage.streamAccounts(this, startKeyHash, limit);
   }
 
+  static class ReferenceTestsInMemoryTrieLogManager extends TrieLogManager {
+
+    private final Cache<Hash, byte[]> trieLogCache =
+        CacheBuilder.newBuilder().maximumSize(5).build();
+
+    public ReferenceTestsInMemoryTrieLogManager() {
+      super(null, null, 0, null);
+    }
+
+    @Override
+    public synchronized void saveTrieLog(
+        final BonsaiWorldStateUpdateAccumulator localUpdater,
+        final Hash forWorldStateRootHash,
+        final BlockHeader forBlockHeader,
+        final BonsaiWorldState forWorldState) {
+      // notify trie log added observers, synchronously
+      TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader);
+      trieLogCache.put(forBlockHeader.getHash(), trieLogFactory.serialize(trieLog));
+      trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog)));
+    }
+
+    @Override
+    public long getMaxLayersToLoad() {
+      return 0;
+    }
+
+    @Override
+    public Optional<TrieLog> getTrieLogLayer(final Hash blockHash) {
+      final byte[] trielog = trieLogCache.getIfPresent(blockHash);
+      trieLogCache.invalidate(blockHash); // remove trielog from the cache
+      return Optional.ofNullable(trieLogFactory.deserialize(trielog));
+    }
+  }
+
   @Override
   protected Hash hashAndSavePreImage(final Bytes value) {
     // by default do not save has preImages

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ForestReferenceTestWorldState.java

@@ -15,6 +15,7 @@
 package org.hyperledger.besu.ethereum.referencetests;
 
 import org.hyperledger.besu.datatypes.Address;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage;
 import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage;
 import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState;
@@ -27,28 +28,41 @@ import java.util.Map;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 
-public class DefaultReferenceTestWorldState extends ForestMutableWorldState
+public class ForestReferenceTestWorldState extends ForestMutableWorldState
     implements ReferenceTestWorldState {
 
-  DefaultReferenceTestWorldState() {
+  ForestReferenceTestWorldState() {
     super(
         new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()),
         new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()),
         EvmConfiguration.DEFAULT);
   }
 
-  public DefaultReferenceTestWorldState(final WorldState worldState) {
+  public ForestReferenceTestWorldState(final WorldState worldState) {
     super(worldState, EvmConfiguration.DEFAULT);
   }
 
   @Override
   public ReferenceTestWorldState copy() {
-    return new DefaultReferenceTestWorldState(this);
+    return new ForestReferenceTestWorldState(this);
+  }
+
+  /**
+   * Executes additional validation checks that are specific to the storage format.
+   *
+   * <p>Depending on the storage format (e.g., Bonsai, etc.), this method performs additional checks
+   * to validate the state. This could include validating the TrieLog and rolling for Bonsai, or
+   * potentially other checks for other modes. This method is intended to be used before the state
+   * root has been validated, to ensure the integrity of other aspects of the state.
+   */
+  @Override
+  public void processExtraStateStorageFormatValidation(final BlockHeader blockHeader) {
+    // nothing more to verify with forest
   }
 
   @JsonCreator
   public static ReferenceTestWorldState create(final Map<String, AccountMock> accounts) {
-    final ReferenceTestWorldState worldState = new DefaultReferenceTestWorldState();
+    final ReferenceTestWorldState worldState = new ForestReferenceTestWorldState();
     final WorldUpdater updater = worldState.updater();
 
     for (final Map.Entry<String, AccountMock> entry : accounts.entrySet()) {

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java

@@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.referencetests;
 
 import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.datatypes.Wei;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.MutableWorldState;
 import org.hyperledger.besu.evm.account.MutableAccount;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
@@ -90,6 +91,8 @@ public interface ReferenceTestWorldState extends MutableWorldState {
 
   ReferenceTestWorldState copy();
 
+  void processExtraStateStorageFormatValidation(final BlockHeader blockHeader);
+
   @JsonCreator
   static ReferenceTestWorldState create(final Map<String, AccountMock> accounts) {
     // delegate to a Bonsai reference test world state:

ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java

@@ -28,7 +28,6 @@ import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder;
-import org.hyperledger.besu.ethereum.core.MutableWorldState;
 import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
@@ -151,7 +150,7 @@ public class GeneralStateReferenceTestTools {
       return;
     }
 
-    final MutableWorldState worldState = initialWorldState.copy();
+    final ReferenceTestWorldState worldState = initialWorldState.copy();
     // Several of the GeneralStateTests check if the transaction could potentially
     // consume more gas than is left for the block it's attempted to be included in.
     // This check is performed within the `BlockImporter` rather than inside the
@@ -193,6 +192,7 @@ public class GeneralStateReferenceTestTools {
       worldStateUpdater.deleteAccount(coinbase.getAddress());
     }
     worldStateUpdater.commit();
+    worldState.processExtraStateStorageFormatValidation(blockHeader);
     worldState.persist(blockHeader);
 
     // Check the world state root hash.