Bonsai Tries Rolling Head and Layered Reads (#1750)

Two updates to Bonsai Tries

* Log Rolling is implemented on top of the existing Persisted head. When 
  Besu is at chain head and the new best head makes the current head an 
  orphan branch, the Bonsai TrieLogs are used to roll back to a common 
  block and roll forward to the needed base block. Goerli is known to 
  maintain sync. There are still some issues with frontier era block 
  receipts.
* Non-mutable reads can be done off of the persisted block. These are 
  accurate for all reads that were performed in the block. If a read is 
  not known it proceeds through a fallback series of calls to prior 
  layers until it hits the persisted block. These layered reads are 
  driven off of the TrieLogs.

Signed-off-by: Danno Ferrin <danno.ferrin@gmail.com>

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -26,6 +26,8 @@ import org.hyperledger.besu.ethereum.blockcreation.GasLimitCalculator;
 import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator;
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
+import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
+import org.hyperledger.besu.ethereum.chain.DefaultBlockchain;
 import org.hyperledger.besu.ethereum.chain.GenesisState;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
 import org.hyperledger.besu.ethereum.core.Hash;
@@ -230,23 +232,24 @@ public abstract class BesuControllerBuilder {
     final ProtocolSchedule protocolSchedule = createProtocolSchedule();
     final GenesisState genesisState = GenesisState.fromConfig(genesisConfig, protocolSchedule);
     final WorldStateStorage worldStateStorage = storageProvider.createWorldStateStorage();
-    final WorldStateArchive worldStateArchive = createWorldStateArchive(worldStateStorage);
+
+    final BlockchainStorage blockchainStorage =
+        storageProvider.createBlockchainStorage(protocolSchedule);
+
+    final MutableBlockchain blockchain =
+        DefaultBlockchain.createMutable(
+            genesisState.getBlock(), blockchainStorage, metricsSystem, reorgLoggingThreshold);
+
+    final WorldStateArchive worldStateArchive =
+        createWorldStateArchive(worldStateStorage, blockchain);
     final ProtocolContext protocolContext =
         ProtocolContext.init(
-            storageProvider,
-            worldStateArchive,
-            genesisState,
-            protocolSchedule,
-            metricsSystem,
-            this::createConsensusContext,
-            reorgLoggingThreshold);
+            blockchain, worldStateArchive, genesisState, this::createConsensusContext);
     validateContext(protocolContext);
 
     protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor(
         protocolContext.getWorldStateArchive());
 
-    final MutableBlockchain blockchain = protocolContext.getBlockchain();
-
     Optional<Pruner> maybePruner = Optional.empty();
     if (isPruningEnabled) {
       if (!storageProvider.isWorldStateIterable()) {
@@ -427,10 +430,11 @@ public abstract class BesuControllerBuilder {
         genesisConfig.getForks());
   }
 
-  public WorldStateArchive createWorldStateArchive(final WorldStateStorage worldStateStorage) {
+  private WorldStateArchive createWorldStateArchive(
+      final WorldStateStorage worldStateStorage, final Blockchain blockchain) {
     switch (dataStorageConfiguration.getDataStorageFormat()) {
       case BONSAI:
-        return new BonsaiWorldStateArchive(storageProvider);
+        return new BonsaiWorldStateArchive(storageProvider, blockchain);
       case FOREST:
       default:
         final WorldStatePreimageStorage preimageStorage =

ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/vm/TraceTransactionIntegrationTest.java

@@ -163,7 +163,7 @@ public class TraceTransactionIntegrationTest {
     final Transaction transaction =
         Transaction.readFrom(
             new BytesValueRLPInput(Bytes.fromHexString(CONTRACT_CREATION_TX), false));
-    BlockHeader genesisBlockHeader = genesisBlock.getHeader();
+    final BlockHeader genesisBlockHeader = genesisBlock.getHeader();
     transactionProcessor.processTransaction(
         blockchain,
         worldStateArchive

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java

@@ -15,14 +15,9 @@
 package org.hyperledger.besu.ethereum;
 
 import org.hyperledger.besu.ethereum.chain.Blockchain;
-import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
-import org.hyperledger.besu.ethereum.chain.DefaultBlockchain;
 import org.hyperledger.besu.ethereum.chain.GenesisState;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
-import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
-import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
-import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 import java.util.function.BiFunction;
 
@@ -46,20 +41,10 @@ public class ProtocolContext {
   }
 
   public static ProtocolContext init(
-      final StorageProvider storageProvider,
+      final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
       final GenesisState genesisState,
-      final ProtocolSchedule protocolSchedule,
-      final MetricsSystem metricsSystem,
-      final BiFunction<Blockchain, WorldStateArchive, Object> consensusContextFactory,
-      final long reorgLoggingThreshold) {
-    final BlockchainStorage blockchainStorage =
-        storageProvider.createBlockchainStorage(protocolSchedule);
-
-    final MutableBlockchain blockchain =
-        DefaultBlockchain.createMutable(
-            genesisState.getBlock(), blockchainStorage, metricsSystem, reorgLoggingThreshold);
-
+      final BiFunction<Blockchain, WorldStateArchive, Object> consensusContextFactory) {
     if (blockchain.getChainHeadBlockNumber() < 1) {
       genesisState.writeStateTo(worldStateArchive.getMutable());
     }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java

@@ -41,7 +41,7 @@ import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
 
 public class BonsaiAccount implements MutableAccount, EvmAccount {
-  private final BonsaiWorldState context;
+  private final BonsaiWorldView context;
   private final boolean mutable;
 
   private final Address address;
@@ -56,7 +56,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
   private final Map<UInt256, UInt256> updatedStorage = new HashMap<>();
 
   BonsaiAccount(
-      final BonsaiWorldState context,
+      final BonsaiWorldView context,
       final Address address,
       final Hash addressHash,
       final long nonce,
@@ -78,7 +78,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
   }
 
   BonsaiAccount(
-      final BonsaiWorldState context,
+      final BonsaiWorldView context,
       final Address address,
       final StateTrieAccountValue stateTrieAccount,
       final boolean mutable) {
@@ -98,7 +98,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
     this(toCopy, toCopy.context, false);
   }
 
-  BonsaiAccount(final BonsaiAccount toCopy, final BonsaiWorldState context, final boolean mutable) {
+  BonsaiAccount(final BonsaiAccount toCopy, final BonsaiWorldView context, final boolean mutable) {
     this.context = context;
     this.address = toCopy.address;
     this.addressHash = toCopy.addressHash;
@@ -113,8 +113,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
     this.mutable = mutable;
   }
 
-  BonsaiAccount(
-      final BonsaiWorldState context, final UpdateTrackingAccount<BonsaiAccount> tracked) {
+  BonsaiAccount(final BonsaiWorldView context, final UpdateTrackingAccount<BonsaiAccount> tracked) {
     this.context = context;
     this.address = tracked.getAddress();
     this.addressHash = tracked.getAddressHash();
@@ -130,7 +129,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
   }
 
   static BonsaiAccount fromRLP(
-      final BonsaiWorldState context,
+      final BonsaiWorldView context,
       final Address address,
       final Bytes encoded,
       final boolean mutable)
@@ -202,7 +201,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
   @Override
   public Bytes getCode() {
     if (code == null) {
-      code = context.getCode(address);
+      code = context.getCode(address).orElse(Bytes.EMPTY);
     }
     return code;
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java

@@ -16,12 +16,11 @@
 
 package org.hyperledger.besu.ethereum.bonsai;
 
-import static com.google.common.base.Preconditions.checkArgument;
-
 import org.hyperledger.besu.ethereum.core.Account;
 import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.ethereum.core.Hash;
 import org.hyperledger.besu.ethereum.core.WorldState;
+import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -33,21 +32,73 @@ import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
 
 /** A World State backed first by trie log layer and then by another world state. */
-public class BonsaiLayeredWorldState implements BonsaiWorldState, WorldState {
+public class BonsaiLayeredWorldState implements BonsaiWorldView, WorldState {
+
+  private final BonsaiWorldView parent;
+  protected final long height;
+  protected final TrieLogLayer trieLog;
 
-  private final BonsaiWorldState parent;
-  private final TrieLogLayer trieLog;
-  private final Map<Address, Account> cachedAccounts = new HashMap<>();
+  private final Hash worldStateRootHash;
 
-  public BonsaiLayeredWorldState(final BonsaiWorldState parent, final TrieLogLayer trieLog) {
-    checkArgument(trieLog.isFrozen(), "TrieLogs must be frozen to be used as world state.");
+  BonsaiLayeredWorldState(
+      final BonsaiWorldView parent,
+      final long height,
+      final Hash worldStateRootHash,
+      final TrieLogLayer trieLog) {
     this.parent = parent;
+    this.height = height;
+    this.worldStateRootHash = worldStateRootHash;
     this.trieLog = trieLog;
   }
 
+  public BonsaiWorldView getParent() {
+    return parent;
+  }
+
+  public TrieLogLayer getTrieLog() {
+    return trieLog;
+  }
+
+  public long getHeight() {
+    return height;
+  }
+
+  @Override
+  public Optional<Bytes> getCode(final Address address) {
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      final Optional<Bytes> maybeCode = currentLayer.trieLog.getCode(address);
+      if (maybeCode.isPresent()) {
+        return maybeCode;
+      }
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        return currentLayer.parent.getCode(address);
+      }
+    }
+    return Optional.empty();
+  }
+
   @Override
-  public Bytes getCode(final Address address) {
-    return trieLog.getCode(address).orElseGet(() -> parent.getCode(address));
+  public Optional<Bytes> getStateTrieNode(final Bytes location) {
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        return currentLayer.parent.getStateTrieNode(location);
+      }
+    }
+    return Optional.empty();
   }
 
   @Override
@@ -57,9 +108,24 @@ public class BonsaiLayeredWorldState implements BonsaiWorldState, WorldState {
 
   @Override
   public Optional<UInt256> getStorageValueBySlotHash(final Address address, final Hash slotHash) {
-    return trieLog
-        .getStorageBySlotHash(address, slotHash)
-        .or(() -> parent.getStorageValueBySlotHash(address, slotHash));
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      final Optional<UInt256> maybeValue =
+          currentLayer.trieLog.getStorageBySlotHash(address, slotHash);
+      if (maybeValue.isPresent()) {
+        return maybeValue;
+      }
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        return currentLayer.parent.getStorageValueBySlotHash(address, slotHash);
+      }
+    }
+    return Optional.empty();
   }
 
   @Override
@@ -70,30 +136,75 @@ public class BonsaiLayeredWorldState implements BonsaiWorldState, WorldState {
 
   @Override
   public Map<Bytes32, Bytes> getAllAccountStorage(final Address address, final Hash rootHash) {
-    final Map<Bytes32, Bytes> results = parent.getAllAccountStorage(address, rootHash);
-    trieLog
-        .streamStorageChanges(address)
-        .forEach(entry -> results.put(entry.getKey(), entry.getValue().getUpdated().toBytes()));
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    final Map<Bytes32, Bytes> results = new HashMap<>();
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      if (currentLayer.trieLog.hasStorageChanges(address)) {
+        currentLayer
+            .trieLog
+            .streamStorageChanges(address)
+            .forEach(
+                entry -> {
+                  if (!results.containsKey(entry.getKey())) {
+                    final UInt256 value = entry.getValue().getUpdated();
+                    // yes, store the nulls.  If it was deleted it should stay deleted
+                    results.put(entry.getKey(), value == null ? null : value.toBytes());
+                  }
+                });
+      }
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        final Account account = currentLayer.parent.get(address);
+        if (account != null) {
+          account
+              .storageEntriesFrom(Hash.ZERO, Integer.MAX_VALUE)
+              .forEach(
+                  (k, v) -> {
+                    if (!results.containsKey(k)) {
+                      results.put(k, v.getValue().toBytes());
+                    }
+                  });
+        }
+        currentLayer = null;
+      }
+    }
     return results;
   }
 
   @Override
   public Account get(final Address address) {
-    return cachedAccounts.computeIfAbsent(
-        address,
-        addr ->
-            trieLog
-                .getAccount(addr)
-                .map(
-                    stateTrieAccountValue ->
-                        (Account)
-                            new BonsaiAccount(
-                                BonsaiLayeredWorldState.this, addr, stateTrieAccountValue, false))
-                .orElseGet(() -> parent.get(address)));
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      final Optional<StateTrieAccountValue> maybeStateTrieAccount =
+          currentLayer.trieLog.getAccount(address);
+      if (maybeStateTrieAccount.isPresent()) {
+        return new BonsaiAccount(
+            BonsaiLayeredWorldState.this, address, maybeStateTrieAccount.get(), false);
+      }
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        return currentLayer.parent.get(address);
+      }
+    }
+    return null;
   }
 
   @Override
   public Hash rootHash() {
+    return worldStateRootHash;
+  }
+
+  public Hash blockHash() {
     return trieLog.getBlockHash();
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java

@@ -37,13 +37,22 @@ import java.util.function.Function;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
 
-public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorldState {
+public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorldView {
 
-  private static final byte[] WORLD_ROOT_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8);
+  private static final Logger LOG = LogManager.getLogger();
+
+  private static final byte[] WORLD_ROOT_HASH_KEY =
+      "worldRootHash".getBytes(StandardCharsets.UTF_8);
+
+  @VisibleForTesting
+  static final byte[] WORLD_BLOCK_HASH_KEY = "worldBlockHash".getBytes(StandardCharsets.UTF_8);
 
   private final KeyValueStorage accountStorage;
   private final KeyValueStorage codeStorage;
@@ -51,10 +60,11 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
   private final KeyValueStorage trieBranchStorage;
   private final KeyValueStorage trieLogStorage;
 
-  private Bytes32 worldStateRootHash;
-
   private final BonsaiWorldStateArchive archive;
-  private BonsaiWorldStateUpdater updater;
+  private final BonsaiWorldStateUpdater updater;
+
+  private Hash worldStateRootHash;
+  private Hash worldStateBlockHash;
 
   public BonsaiPersistedWorldState(
       final BonsaiWorldStateArchive archive,
@@ -70,8 +80,17 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
     this.trieBranchStorage = trieBranchStorage;
     this.trieLogStorage = trieLogStorage;
     worldStateRootHash =
-        Bytes32.wrap(
-            trieBranchStorage.get(WORLD_ROOT_KEY).map(Bytes::wrap).orElse(Hash.EMPTY_TRIE_HASH));
+        Hash.wrap(
+            Bytes32.wrap(
+                trieBranchStorage
+                    .get(WORLD_ROOT_HASH_KEY)
+                    .map(Bytes::wrap)
+                    .orElse(Hash.EMPTY_TRIE_HASH)));
+    worldStateBlockHash =
+        Hash.wrap(
+            Bytes32.wrap(
+                trieBranchStorage.get(WORLD_BLOCK_HASH_KEY).map(Bytes::wrap).orElse(Hash.ZERO)));
+    updater = new BonsaiWorldStateUpdater(this);
   }
 
   public BonsaiWorldStateArchive getArchive() {
@@ -85,14 +104,15 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
   }
 
   @Override
-  public Bytes getCode(@Nonnull final Address address) {
-    return codeStorage.get(address.toArrayUnsafe()).map(Bytes::wrap).orElse(Bytes.EMPTY);
+  public Optional<Bytes> getCode(@Nonnull final Address address) {
+    return codeStorage.get(address.toArrayUnsafe()).map(Bytes::wrap);
   }
 
   @Override
   public void persist(final BlockHeader blockHeader) {
-    final Hash blockHash = blockHeader == null ? null : blockHeader.getHash();
     boolean success = false;
+    final Hash originalBlockHash = worldStateBlockHash;
+    final Hash originalRootHash = worldStateRootHash;
     final KeyValueStorageTransaction accountTx = accountStorage.startTransaction();
     final KeyValueStorageTransaction codeTx = codeStorage.startTransaction();
     final KeyValueStorageTransaction storageTx = storageStorage.startTransaction();
@@ -165,7 +185,7 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
           } else {
             final Bytes32 updatedStorageBytes = updatedStorage.toBytes();
             storageTx.put(writeAddress, updatedStorageBytes.toArrayUnsafe());
-            storageTrie.put(keyHash, rlpEncode(updatedStorageBytes));
+            storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorageBytes));
           }
         }
 
@@ -216,21 +236,41 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
         }
       }
 
+      // TODO write to a cache and then generate a layer update from that and the
+      // DB tx updates.  Right now it is just DB updates.
       accountTrie.commit((location, hash, value) -> writeTrieNode(trieBranchTx, location, value));
-      worldStateRootHash = accountTrie.getRootHash();
-      trieBranchTx.put(WORLD_ROOT_KEY, worldStateRootHash.toArrayUnsafe());
+      worldStateRootHash = Hash.wrap(accountTrie.getRootHash());
+      trieBranchTx.put(WORLD_ROOT_HASH_KEY, worldStateRootHash.toArrayUnsafe());
 
       // for manicured tries and composting, trim and compost branches here
 
-      if (blockHash != null) {
-        final TrieLogLayer trieLog = updater.generateTrieLog(blockHash);
-        trieLog.freeze();
-        // TODO add to archive here, but only once we get persisted follow distance implemented
-        // archive.addLayeredWorldState(new BonsaiLayeredWorldState(this, trieLog));
-
-        final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput();
-        trieLog.writeTo(rlpLog);
-        trieLogTx.put(blockHash.toArrayUnsafe(), rlpLog.encoded().toArrayUnsafe());
+      // if we are persisted with a block header, and the prior state is the parent
+      // then persist the TrieLog for that transition.  If specified but not a direct
+      // descendant simply store the new block hash.
+      if (blockHeader != null) {
+        if (!worldStateRootHash.equals(blockHeader.getStateRoot())) {
+          throw new RuntimeException(
+              "World State Root does not match expected value, header "
+                  + blockHeader.getStateRoot().toHexString()
+                  + " calcualted "
+                  + worldStateRootHash.toHexString());
+        }
+        worldStateBlockHash = blockHeader.getHash();
+        trieBranchTx.put(WORLD_BLOCK_HASH_KEY, worldStateBlockHash.toArrayUnsafe());
+        if (originalBlockHash.equals(blockHeader.getParentHash())) {
+          LOG.debug("Writing Trie Log for {}", worldStateBlockHash);
+          final TrieLogLayer trieLog = updater.generateTrieLog(worldStateBlockHash);
+          trieLog.freeze();
+          archive.addLayeredWorldState(
+              new BonsaiLayeredWorldState(
+                  this, blockHeader.getNumber(), worldStateRootHash, trieLog));
+          final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput();
+          trieLog.writeTo(rlpLog);
+          trieLogTx.put(worldStateBlockHash.toArrayUnsafe(), rlpLog.encoded().toArrayUnsafe());
+        }
+      } else {
+        trieBranchTx.remove(WORLD_BLOCK_HASH_KEY);
+        worldStateBlockHash = null;
       }
 
       success = true;
@@ -248,21 +288,17 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
         storageTx.rollback();
         trieBranchTx.rollback();
         trieLogTx.rollback();
+        worldStateBlockHash = originalBlockHash;
+        worldStateRootHash = originalRootHash;
       }
     }
-  }
-
-  private static Bytes rlpEncode(final Bytes bytes) {
-    final BytesValueRLPOutput out = new BytesValueRLPOutput();
-    out.writeBytes(bytes.trimLeadingZeros());
-    return out.encoded();
+    if (blockHeader != null) {
+      archive.scrubLayeredCache(blockHeader.getNumber());
+    }
   }
 
   @Override
   public WorldUpdater updater() {
-    if (updater == null) {
-      updater = new BonsaiWorldStateUpdater(this);
-    }
     return updater;
   }
 
@@ -271,6 +307,10 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
     return Hash.wrap(worldStateRootHash);
   }
 
+  public Hash blockHash() {
+    return worldStateBlockHash;
+  }
+
   @Override
   public Stream<StreamableAccount> streamAccounts(final Bytes32 startKeyHash, final int limit) {
     throw new RuntimeException("Bonsai Tries do not provide account streaming.");
@@ -288,10 +328,15 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
     if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) {
       return Optional.of(MerklePatriciaTrie.EMPTY_TRIE_NODE);
     } else {
-      return trieBranchStorage.get(location.toArrayUnsafe()).map(Bytes::wrap);
+      return getStateTrieNode(location);
     }
   }
 
+  @Override
+  public Optional<Bytes> getStateTrieNode(final Bytes location) {
+    return trieBranchStorage.get(location.toArrayUnsafe()).map(Bytes::wrap);
+  }
+
   private void writeTrieNode(
       final KeyValueStorageTransaction tx, final Bytes location, final Bytes value) {
     tx.put(location.toArrayUnsafe(), value.toArrayUnsafe());
@@ -302,9 +347,7 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
     if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) {
       return Optional.of(MerklePatriciaTrie.EMPTY_TRIE_NODE);
     } else {
-      return trieBranchStorage
-          .get(Bytes.concatenate(address, location).toArrayUnsafe())
-          .map(Bytes::wrap);
+      return getStateTrieNode(Bytes.concatenate(address, location));
     }
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java

@@ -16,7 +16,9 @@
 
 package org.hyperledger.besu.ethereum.bonsai;
 
+import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.core.Address;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.Hash;
 import org.hyperledger.besu.ethereum.core.MutableWorldState;
 import org.hyperledger.besu.ethereum.core.WorldState;
@@ -24,22 +26,36 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProof;
 import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
+import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
 
 public class BonsaiWorldStateArchive implements WorldStateArchive {
 
+  private static final Logger LOG = LogManager.getLogger();
+
+  static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks
+
+  private final Blockchain blockchain;
+
   private final BonsaiPersistedWorldState persistedState;
   private final Map<Bytes32, BonsaiLayeredWorldState> layeredWorldStates;
+  private final KeyValueStorage trieLogStorage;
 
-  public BonsaiWorldStateArchive(final StorageProvider provider) {
+  public BonsaiWorldStateArchive(final StorageProvider provider, final Blockchain blockchain) {
+    this.blockchain = blockchain;
+    trieLogStorage =
+        provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
     persistedState =
         new BonsaiPersistedWorldState(
             this,
@@ -48,37 +64,98 @@ public class BonsaiWorldStateArchive implements WorldStateArchive {
             provider.getStorageBySegmentIdentifier(
                 KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE),
             provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE),
-            provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE));
+            trieLogStorage);
     layeredWorldStates = new HashMap<>();
   }
 
   @Override
   public Optional<WorldState> get(final Hash rootHash, final Hash blockHash) {
     if (layeredWorldStates.containsKey(rootHash)) {
-      return Optional.of(layeredWorldStates.get(rootHash));
-    } else if (rootHash.equals(persistedState.rootHash())) {
+      return Optional.of(layeredWorldStates.get(blockHash));
+    } else if (rootHash.equals(persistedState.blockHash())) {
       return Optional.of(persistedState);
     } else {
       return Optional.empty();
     }
   }
 
-  public void addLayeredWorldState(final BonsaiLayeredWorldState worldState) {
-    layeredWorldStates.put(worldState.rootHash(), worldState);
+  void addLayeredWorldState(final BonsaiLayeredWorldState worldState) {
+    layeredWorldStates.put(worldState.blockHash(), worldState);
+  }
+
+  private Optional<TrieLogLayer> getTrieLogLayer(final Hash blockHash) {
+    if (layeredWorldStates.containsKey(blockHash)) {
+      return Optional.of(layeredWorldStates.get(blockHash).getTrieLog());
+    } else {
+      return trieLogStorage.get(blockHash.toArrayUnsafe()).map(TrieLogLayer::fromBytes);
+    }
   }
 
   @Override
   public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) {
-    return layeredWorldStates.containsKey(rootHash)
-        || persistedState.rootHash().equals(rootHash) /* || check disk storage */;
+    return layeredWorldStates.containsKey(blockHash)
+        || persistedState.blockHash().equals(blockHash)
+        || trieLogStorage.containsKey(blockHash.toArrayUnsafe());
   }
 
   @Override
   public Optional<MutableWorldState> getMutable(final Hash rootHash, final Hash blockHash) {
-    if (rootHash.equals(persistedState.rootHash())) {
+    if (blockHash.equals(persistedState.blockHash())) {
       return Optional.of(persistedState);
     } else {
-      return Optional.empty();
+      try {
+        BlockHeader persistedHeader = blockchain.getBlockHeader(persistedState.blockHash()).get();
+        BlockHeader targetHeader = blockchain.getBlockHeader(blockHash).get();
+
+        final List<TrieLogLayer> rollBacks = new ArrayList<>();
+        final List<TrieLogLayer> rollForwards = new ArrayList<>();
+
+        // roll back from persisted to even with target
+        while (persistedHeader.getNumber() > targetHeader.getNumber()) {
+          LOG.debug("Rollback {}", persistedHeader.getHash());
+          rollBacks.add(getTrieLogLayer(persistedHeader.getHash()).get());
+          persistedHeader = blockchain.getBlockHeader(persistedHeader.getParentHash()).get();
+        }
+        // roll forward to target
+        while (persistedHeader.getNumber() < targetHeader.getNumber()) {
+          LOG.debug("Rollforward {}", targetHeader.getHash());
+          rollForwards.add(getTrieLogLayer(targetHeader.getHash()).get());
+          targetHeader = blockchain.getBlockHeader(targetHeader.getParentHash()).get();
+        }
+
+        // roll back in tandem until we hit a shared state
+        while (!persistedHeader.getHash().equals(targetHeader.getHash())) {
+          LOG.debug("Paired Rollback {}", persistedHeader.getHash());
+          LOG.debug("Paired Rollforward {}", targetHeader.getHash());
+          rollForwards.add(getTrieLogLayer(targetHeader.getHash()).get());
+          targetHeader = blockchain.getBlockHeader(targetHeader.getParentHash()).get();
+
+          rollBacks.add(getTrieLogLayer(persistedHeader.getHash()).get());
+          persistedHeader = blockchain.getBlockHeader(persistedHeader.getParentHash()).get();
+        }
+
+        // attempt the state rolling
+        final BonsaiWorldStateUpdater bonsaiUpdater =
+            (BonsaiWorldStateUpdater) persistedState.updater();
+        try {
+          for (final TrieLogLayer rollBack : rollBacks) {
+            bonsaiUpdater.rollBack(rollBack);
+          }
+          for (int i = rollForwards.size() - 1; i >= 0; i--) {
+            bonsaiUpdater.rollForward(rollForwards.get(i));
+          }
+          bonsaiUpdater.commit();
+          persistedState.persist(blockchain.getBlockHeader(blockHash).get());
+          return Optional.of(persistedState);
+        } catch (final Exception e) {
+          // if we fail we must clean up the updater
+          bonsaiUpdater.reset();
+          throw new RuntimeException(e);
+        }
+      } catch (final RuntimeException re) {
+        re.printStackTrace(System.out);
+        return Optional.empty();
+      }
     }
   }
 
@@ -97,6 +174,12 @@ public class BonsaiWorldStateArchive implements WorldStateArchive {
       final Hash worldStateRoot,
       final Address accountAddress,
       final List<UInt256> accountStorageKeys) {
+    // FIXME we can do proofs for layered tries and the persisted trie
     return Optional.empty();
   }
+
+  void scrubLayeredCache(final long newMaxHeight) {
+    final long waterline = newMaxHeight - RETAINED_LAYERS;
+    layeredWorldStates.entrySet().removeIf(entry -> entry.getValue().getHeight() < waterline);
+  }
 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java

@@ -43,9 +43,8 @@ import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
 
-public class BonsaiWorldStateUpdater
-    extends AbstractWorldUpdater<BonsaiPersistedWorldState, BonsaiAccount>
-    implements BonsaiWorldState {
+public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldView, BonsaiAccount>
+    implements BonsaiWorldView {
 
   private final Map<Address, BonsaiValue<BonsaiAccount>> accountsToUpdate = new HashMap<>();
   private final Map<Address, BonsaiValue<Bytes>> codeToUpdate = new HashMap<>();
@@ -56,7 +55,7 @@ public class BonsaiWorldStateUpdater
   // alternative was to keep a giant pre-image cache of the entire trie.
   private final Map<Address, Map<Hash, BonsaiValue<UInt256>>> storageToUpdate = new HashMap<>();
 
-  BonsaiWorldStateUpdater(final BonsaiPersistedWorldState world) {
+  BonsaiWorldStateUpdater(final BonsaiWorldView world) {
     super(world);
   }
 
@@ -156,10 +155,11 @@ public class BonsaiWorldStateUpdater
       if (codeValue != null) {
         codeValue.setUpdated(null);
       } else {
-        final Bytes deletedCode = wrappedWorldView().getCode(deletedAddress);
-        if (deletedCode != null) {
-          codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null));
-        }
+        wrappedWorldView()
+            .getCode(deletedAddress)
+            .ifPresent(
+                deletedCode ->
+                    codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null)));
       }
       final BonsaiValue<BonsaiAccount> accountValue =
           accountsToUpdate.computeIfAbsent(
@@ -227,7 +227,8 @@ public class BonsaiWorldStateUpdater
       if (tracked.codeWasUpdated()) {
         final BonsaiValue<Bytes> pendingCode =
             codeToUpdate.computeIfAbsent(
-                updatedAddress, addr -> new BonsaiValue<>(wrappedWorldView().getCode(addr), null));
+                updatedAddress,
+                addr -> new BonsaiValue<>(wrappedWorldView().getCode(addr).orElse(null), null));
         pendingCode.setUpdated(updatedAccount.getCode());
       }
 
@@ -263,15 +264,21 @@ public class BonsaiWorldStateUpdater
   }
 
   @Override
-  public Bytes getCode(final Address address) {
+  public Optional<Bytes> getCode(final Address address) {
     final BonsaiValue<Bytes> localCode = codeToUpdate.get(address);
     if (localCode == null) {
       return wrappedWorldView().getCode(address);
     } else {
-      return localCode.getUpdated();
+      return Optional.of(localCode.getUpdated());
     }
   }
 
+  @Override
+  public Optional<Bytes> getStateTrieNode(final Bytes location) {
+    // updater doesn't track trie nodes.  Always a miss.
+    return Optional.empty();
+  }
+
   @Override
   public UInt256 getStorageValue(final Address address, final UInt256 storageKey) {
     // TODO maybe log the read into the trie layer?
@@ -325,6 +332,11 @@ public class BonsaiWorldStateUpdater
 
   public TrieLogLayer generateTrieLog(final Hash blockHash) {
     final TrieLogLayer layer = new TrieLogLayer();
+    importIntoTrieLog(layer, blockHash);
+    return layer;
+  }
+
+  private void importIntoTrieLog(final TrieLogLayer layer, final Hash blockHash) {
     layer.setBlockHash(blockHash);
     for (final Map.Entry<Address, BonsaiValue<BonsaiAccount>> updatedAccount :
         accountsToUpdate.entrySet()) {
@@ -371,8 +383,6 @@ public class BonsaiWorldStateUpdater
             slotUpdate.getValue().getUpdated());
       }
     }
-
-    return layer;
   }
 
   public void rollForward(final TrieLogLayer layer) {
@@ -452,12 +462,17 @@ public class BonsaiWorldStateUpdater
       }
     } else {
       if (expectedValue == null) {
-        throw new IllegalStateException(
-            String.format(
-                "Expected to create account, but the account exists.  Address=%s", address));
+        if (accountValue.getUpdated() != null) {
+          throw new IllegalStateException(
+              String.format(
+                  "Expected to create account, but the account exists.  Address=%s", address));
+        }
+      } else {
+        BonsaiAccount.assertCloseEnoughForDiffing(
+            accountValue.getUpdated(),
+            expectedValue,
+            "Address=" + address + " Prior Value in Rolling Change");
       }
-      BonsaiAccount.assertCloseEnoughForDiffing(
-          accountValue.getUpdated(), expectedValue, "Prior Value in Rolling Change");
       if (replacementValue == null) {
         if (accountValue.getOriginal() == null) {
           accountsToUpdate.remove(address);
@@ -465,12 +480,8 @@ public class BonsaiWorldStateUpdater
           accountValue.setUpdated(null);
         }
       } else {
-        final BonsaiAccount existingAccount = accountValue.getUpdated();
-        existingAccount.setNonce(replacementValue.getNonce());
-        existingAccount.setBalance(replacementValue.getBalance());
-        existingAccount.setStorageRoot(replacementValue.getStorageRoot());
-        // depend on correctly structured layers to set code hash
-        existingAccount.setVersion(replacementValue.getVersion());
+        accountValue.setUpdated(
+            new BonsaiAccount(wrappedWorldView(), address, replacementValue, true));
       }
     }
   }
@@ -497,7 +508,7 @@ public class BonsaiWorldStateUpdater
     }
     BonsaiValue<Bytes> codeValue = codeToUpdate.get(address);
     if (codeValue == null) {
-      final Bytes storedCode = wrappedWorldView().getCode(address);
+      final Bytes storedCode = wrappedWorldView().getCode(address).orElse(Bytes.EMPTY);
       if (!storedCode.isEmpty()) {
         codeValue = new BonsaiValue<>(storedCode, storedCode);
         codeToUpdate.put(address, codeValue);
@@ -505,7 +516,7 @@ public class BonsaiWorldStateUpdater
     }
 
     if (codeValue == null) {
-      if (expectedCode == null && replacementCode != null) {
+      if ((expectedCode == null || expectedCode.size() == 0) && replacementCode != null) {
         codeToUpdate.put(address, new BonsaiValue<>(null, replacementCode));
       } else {
         throw new IllegalStateException(
@@ -513,22 +524,23 @@ public class BonsaiWorldStateUpdater
                 "Expected to update code, but the code does not exist.  Address=%s", address));
       }
     } else {
-      if (expectedCode == null) {
+      final Bytes existingCode = codeValue.getUpdated();
+      if ((expectedCode == null || expectedCode.isEmpty())
+          && existingCode != null
+          && !existingCode.isEmpty()) {
         throw new IllegalStateException(
             String.format("Expected to create code, but the code exists.  Address=%s", address));
       }
-      if (!codeValue.getUpdated().equals(expectedCode)) {
+      if (!Objects.equals(expectedCode, existingCode)) {
         throw new IllegalStateException(
             String.format(
                 "Old value of code does not match expected value.  Address=%s ExpectedHash=%s ActualHash=%s",
-                address, Hash.hash(expectedCode), Hash.hash(codeValue.getUpdated())));
+                address,
+                expectedCode == null ? "null" : Hash.hash(expectedCode),
+                Hash.hash(codeValue.getUpdated())));
       }
-      if (replacementCode == null) {
-        if (codeValue.getOriginal() == null) {
-          codeToUpdate.remove(address);
-        } else {
-          codeValue.setUpdated(null);
-        }
+      if (replacementCode == null && codeValue.getOriginal() == null) {
+        codeToUpdate.remove(address);
       } else {
         codeValue.setUpdated(replacementCode);
       }
@@ -555,6 +567,10 @@ public class BonsaiWorldStateUpdater
       // non-change, a cached read.
       return;
     }
+    if (replacementValue == null && expectedValue != null && expectedValue.isZero()) {
+      // corner case on deletes, non-change
+      return;
+    }
     final Map<Hash, BonsaiValue<UInt256>> storageMap = storageToUpdate.get(address);
     BonsaiValue<UInt256> slotValue = storageMap == null ? null : storageMap.get(slotHash);
     if (slotValue == null) {
@@ -566,7 +582,7 @@ public class BonsaiWorldStateUpdater
       }
     }
     if (slotValue == null) {
-      if (expectedValue == null && replacementValue != null) {
+      if ((expectedValue == null || expectedValue.isZero()) && replacementValue != null) {
         maybeCreateStorageMap(storageMap, address)
             .put(slotHash, new BonsaiValue<>(null, replacementValue));
       } else {
@@ -576,32 +592,30 @@ public class BonsaiWorldStateUpdater
                 address, slotHash));
       }
     } else {
-      if (expectedValue == null) {
+      final UInt256 existingSlotValue = slotValue.getUpdated();
+      if ((expectedValue == null || expectedValue.isZero())
+          && existingSlotValue != null
+          && !existingSlotValue.isZero()) {
         throw new IllegalStateException(
             String.format(
-                "Expected to create slot, but the slot exists. Account=%s SlotHash=%s",
-                address, slotHash));
+                "Expected to create slot, but the slot exists. Account=%s SlotHash=%s expectedValue=%s existingValue=%s",
+                address, slotHash, expectedValue, existingSlotValue));
       }
-      final UInt256 existingSlotValue = slotValue.getUpdated();
-      if (!existingSlotValue.equals(expectedValue)) {
+      if (!Objects.equals(expectedValue, existingSlotValue)) {
         throw new IllegalStateException(
             String.format(
                 "Old value of slot does not match expected value. Account=%s SlotHash=%s Expected=%s Actual=%s",
                 address,
                 slotHash,
-                expectedValue.toShortHexString(),
-                existingSlotValue.toShortHexString()));
+                expectedValue == null ? "null" : expectedValue.toShortHexString(),
+                existingSlotValue == null ? "null" : existingSlotValue.toShortHexString()));
       }
-      if (replacementValue == null) {
-        if (slotValue.getOriginal() == null) {
-          final Map<Hash, BonsaiValue<UInt256>> thisStorageUpdate =
-              maybeCreateStorageMap(storageMap, address);
-          thisStorageUpdate.remove(slotHash);
-          if (thisStorageUpdate.isEmpty()) {
-            storageToUpdate.remove(address);
-          }
-        } else {
-          slotValue.setUpdated(null);
+      if (replacementValue == null && slotValue.getOriginal() == null) {
+        final Map<Hash, BonsaiValue<UInt256>> thisStorageUpdate =
+            maybeCreateStorageMap(storageMap, address);
+        thisStorageUpdate.remove(slotHash);
+        if (thisStorageUpdate.isEmpty()) {
+          storageToUpdate.remove(address);
         }
       } else {
         slotValue.setUpdated(replacementValue);

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldView.java

@@ -19,6 +19,7 @@ package org.hyperledger.besu.ethereum.bonsai;
 import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.ethereum.core.Hash;
 import org.hyperledger.besu.ethereum.core.WorldView;
+import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
 
 import java.util.Map;
 import java.util.Optional;
@@ -27,9 +28,11 @@ import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
 
-public interface BonsaiWorldState extends WorldView {
+public interface BonsaiWorldView extends WorldView {
 
-  Bytes getCode(Address address);
+  Optional<Bytes> getCode(Address address);
+
+  Optional<Bytes> getStateTrieNode(Bytes location);
 
   UInt256 getStorageValue(Address address, UInt256 key);
 
@@ -38,7 +41,7 @@ public interface BonsaiWorldState extends WorldView {
   UInt256 getOriginalStorageValue(Address address, UInt256 key);
 
   /**
-   * Stream all the storage values of a account.
+   * Retrieve all the storage values of a account.
    *
    * @param address the account to stream
    * @param rootHash the root hash of the account storage trie
@@ -46,4 +49,10 @@ public interface BonsaiWorldState extends WorldView {
    *     is the Bytes representation of the storage value.
    */
   Map<Bytes32, Bytes> getAllAccountStorage(Address address, Hash rootHash);
+
+  static Bytes encodeTrieValue(final Bytes bytes) {
+    final BytesValueRLPOutput out = new BytesValueRLPOutput();
+    out.writeBytes(bytes.trimLeadingZeros());
+    return out.encoded();
+  }
 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogLayer.java

@@ -20,11 +20,14 @@ import static com.google.common.base.Preconditions.checkState;
 
 import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.ethereum.core.Hash;
+import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
 import org.hyperledger.besu.ethereum.rlp.RLPInput;
 import org.hyperledger.besu.ethereum.rlp.RLPOutput;
 import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
@@ -45,11 +48,18 @@ import org.apache.tuweni.units.bigints.UInt256;
 public class TrieLogLayer {
 
   private Hash blockHash;
-  private final Map<Address, BonsaiValue<StateTrieAccountValue>> accounts = new TreeMap<>();
-  private final Map<Address, BonsaiValue<Bytes>> code = new TreeMap<>();
-  private final Map<Address, Map<Hash, BonsaiValue<UInt256>>> storage = new TreeMap<>();
+  private final Map<Address, BonsaiValue<StateTrieAccountValue>> accounts;
+  private final Map<Address, BonsaiValue<Bytes>> code;
+  private final Map<Address, Map<Hash, BonsaiValue<UInt256>>> storage;
   private boolean frozen = false;
 
+  TrieLogLayer() {
+    // TODO when tuweni fixes zero length byte comparison consider TreeMap
+    accounts = new HashMap<>();
+    code = new HashMap<>();
+    storage = new HashMap<>();
+  }
+
   /** Locks the layer so no new changes can be added; */
   void freeze() {
     frozen = true; // The code never bothered me anyway
@@ -64,7 +74,7 @@ public class TrieLogLayer {
     this.blockHash = blockHash;
   }
 
-  public void addAccountChange(
+  void addAccountChange(
       final Address address,
       final StateTrieAccountValue oldValue,
       final StateTrieAccountValue newValue) {
@@ -88,6 +98,10 @@ public class TrieLogLayer {
         .put(slotHash, new BonsaiValue<>(oldValue, newValue));
   }
 
+  static TrieLogLayer fromBytes(final byte[] bytes) {
+    return readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false));
+  }
+
   static TrieLogLayer readFrom(final RLPInput input) {
     final TrieLogLayer newLayer = new TrieLogLayer();
 
@@ -134,6 +148,9 @@ public class TrieLogLayer {
         input.leaveList();
         newLayer.storage.put(address, storageChanges);
       }
+
+      // TODO add trie nodes
+
       // lenient leave list for forward compatible additions.
       input.leaveListLenient();
     }
@@ -186,6 +203,9 @@ public class TrieLogLayer {
         }
         output.endList();
       }
+
+      // TODO write trie nodes
+
       output.endList(); // this change
     }
     output.endList(); // container
@@ -203,6 +223,10 @@ public class TrieLogLayer {
     return storage.entrySet().stream();
   }
 
+  boolean hasStorageChanges(final Address address) {
+    return storage.containsKey(address);
+  }
+
   Stream<Map.Entry<Hash, BonsaiValue<UInt256>>> streamStorageChanges(final Address address) {
     return storage.getOrDefault(address, Map.of()).entrySet().stream();
   }
@@ -216,10 +240,6 @@ public class TrieLogLayer {
     }
   }
 
-  boolean isFrozen() {
-    return frozen;
-  }
-
   public Optional<Bytes> getCode(final Address address) {
     return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated);
   }
@@ -233,4 +253,52 @@ public class TrieLogLayer {
   public Optional<StateTrieAccountValue> getAccount(final Address address) {
     return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated);
   }
+
+  public String dump() {
+    final StringBuilder sb = new StringBuilder();
+    sb.append("TrieLogLayer{" + "blockHash=").append(blockHash).append(frozen).append('}');
+    sb.append("accounts\n");
+    for (final Map.Entry<Address, BonsaiValue<StateTrieAccountValue>> account :
+        accounts.entrySet()) {
+      sb.append(" : ").append(account.getKey()).append("\n");
+      if (Objects.equals(account.getValue().getOriginal(), account.getValue().getUpdated())) {
+        sb.append("   = ").append(account.getValue().getUpdated()).append("\n");
+      } else {
+        sb.append("   - ").append(account.getValue().getOriginal()).append("\n");
+        sb.append("   + ").append(account.getValue().getUpdated()).append("\n");
+      }
+    }
+    sb.append("code").append("\n");
+    for (final Map.Entry<Address, BonsaiValue<Bytes>> code : code.entrySet()) {
+      sb.append(" : ").append(code.getKey()).append("\n");
+      if (Objects.equals(code.getValue().getOriginal(), code.getValue().getUpdated())) {
+        sb.append("   = ").append(code.getValue().getOriginal()).append("\n");
+      } else {
+        sb.append("   - ").append(code.getValue().getOriginal()).append("\n");
+        sb.append("   + ").append(code.getValue().getUpdated()).append("\n");
+      }
+    }
+    sb.append("Storage").append("\n");
+    for (final Map.Entry<Address, Map<Hash, BonsaiValue<UInt256>>> storage : storage.entrySet()) {
+      sb.append(" : ").append(storage.getKey()).append("\n");
+      for (final Map.Entry<Hash, BonsaiValue<UInt256>> slot : storage.getValue().entrySet()) {
+        final UInt256 originalValue = slot.getValue().getOriginal();
+        final UInt256 updatedValue = slot.getValue().getUpdated();
+        sb.append("   : ").append(slot.getKey()).append("\n");
+        if (Objects.equals(originalValue, updatedValue)) {
+          sb.append("     = ")
+              .append((originalValue == null) ? "null" : originalValue.toShortHexString())
+              .append("\n");
+        } else {
+          sb.append("     - ")
+              .append((originalValue == null) ? "null" : originalValue.toShortHexString())
+              .append("\n");
+          sb.append("     + ")
+              .append((updatedValue == null) ? "null" : updatedValue.toShortHexString())
+              .append("\n");
+        }
+      }
+    }
+    return sb.toString();
+  }
 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java

@@ -108,7 +108,7 @@ public final class GenesisState {
   private static void writeAccountsTo(
       final MutableWorldState target,
       final List<GenesisAccount> genesisAccounts,
-      final BlockHeader rootHash) {
+      final BlockHeader rootHeader) {
     final WorldUpdater updater = target.updater();
     genesisAccounts.forEach(
         genesisAccount -> {
@@ -120,7 +120,7 @@ public final class GenesisState {
           genesisAccount.storage.forEach(account::setStorageValue);
         });
     updater.commit();
-    target.persist(rootHash);
+    target.persist(rootHeader);
   }
 
   private static Hash calculateGenesisStateHash(final List<GenesisAccount> genesisAccounts) {

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java

@@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
+import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
 import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
 
 import java.util.Optional;
@@ -65,7 +66,7 @@ public class LogRollingTests {
           Hash.ZERO,
           Hash.EMPTY_LIST_HASH,
           Address.ZERO,
-          Hash.EMPTY_TRIE_HASH,
+          Hash.fromHexString("0x0ecfa454ddfe6b740f4af7b7f4c61b5c6bac2854efd2b07b27b1f53dba9bb46c"),
           Hash.EMPTY_TRIE_HASH,
           Hash.EMPTY_LIST_HASH,
           LogsBloomFilter.builder().build(),
@@ -84,7 +85,7 @@ public class LogRollingTests {
           headerOne.getHash(),
           Hash.EMPTY_LIST_HASH,
           Address.ZERO,
-          Hash.EMPTY_TRIE_HASH,
+          Hash.fromHexString("0x5b675f79cd11ba67266161d79a8d5be3ac330dfbb76300a4f15d76b610b18193"),
           Hash.EMPTY_TRIE_HASH,
           Hash.EMPTY_LIST_HASH,
           LogsBloomFilter.builder().build(),
@@ -102,7 +103,7 @@ public class LogRollingTests {
   @Before
   public void createStorage() {
     final InMemoryStorageProvider provider = new InMemoryStorageProvider();
-    archive = new BonsaiWorldStateArchive(provider);
+    archive = new BonsaiWorldStateArchive(provider, null);
     accountStorage =
         (InMemoryKeyValueStorage)
             provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
@@ -121,7 +122,7 @@ public class LogRollingTests {
             provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
 
     final InMemoryStorageProvider secondProvider = new InMemoryStorageProvider();
-    secondArchive = new BonsaiWorldStateArchive(secondProvider);
+    secondArchive = new BonsaiWorldStateArchive(secondProvider, null);
     secondAccountStorage =
         (InMemoryKeyValueStorage)
             provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
@@ -182,6 +183,9 @@ public class LogRollingTests {
     assertKeyValueStorageEqual(accountStorage, secondAccountStorage);
     assertKeyValueStorageEqual(codeStorage, secondCodeStorage);
     assertKeyValueStorageEqual(storageStorage, secondStorageStorage);
+    final KeyValueStorageTransaction tx = trieBranchStorage.startTransaction();
+    tx.remove(BonsaiPersistedWorldState.WORLD_BLOCK_HASH_KEY);
+    tx.commit();
     assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage);
     // trie logs won't be the same, we shouldn't generate logs on rolls.
     assertKeyValueSubset(trieLogStorage, secondTrieLogStorage);
@@ -239,6 +243,9 @@ public class LogRollingTests {
     assertKeyValueStorageEqual(accountStorage, secondAccountStorage);
     assertKeyValueStorageEqual(codeStorage, secondCodeStorage);
     assertKeyValueStorageEqual(storageStorage, secondStorageStorage);
+    final KeyValueStorageTransaction tx = trieBranchStorage.startTransaction();
+    tx.remove(BonsaiPersistedWorldState.WORLD_BLOCK_HASH_KEY);
+    tx.commit();
     assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage);
     // trie logs won't be the same, we shouldn't generate logs on rolls.
     assertKeyValueSubset(trieLogStorage, secondTrieLogStorage);
@@ -277,7 +284,7 @@ public class LogRollingTests {
     final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, headerTwo.getHash());
     firstRollbackUpdater.rollBack(layerTwo);
 
-    worldState.persist(headerTwo);
+    worldState.persist(headerOne);
 
     final BonsaiPersistedWorldState secondWorldState =
         new BonsaiPersistedWorldState(
@@ -300,6 +307,9 @@ public class LogRollingTests {
     assertKeyValueStorageEqual(accountStorage, secondAccountStorage);
     assertKeyValueStorageEqual(codeStorage, secondCodeStorage);
     assertKeyValueStorageEqual(storageStorage, secondStorageStorage);
+    final KeyValueStorageTransaction tx = trieBranchStorage.startTransaction();
+    tx.remove(BonsaiPersistedWorldState.WORLD_BLOCK_HASH_KEY);
+    tx.commit();
     assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage);
     // trie logs won't be the same, we don't delete the roll back log
     assertKeyValueSubset(trieLogStorage, secondTrieLogStorage);

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java

@@ -38,7 +38,7 @@ public class RollingImport {
         new RollingFileReader((i, c) -> Path.of(String.format(arg[0] + "-%04d.rdat", i)), false);
 
     final InMemoryStorageProvider provider = new InMemoryStorageProvider();
-    final BonsaiWorldStateArchive archive = new BonsaiWorldStateArchive(provider);
+    final BonsaiWorldStateArchive archive = new BonsaiWorldStateArchive(provider, null);
     final InMemoryKeyValueStorage accountStorage =
         (InMemoryKeyValueStorage)
             provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java

@@ -121,7 +121,6 @@ public class PrivacyBlockProcessorTest {
             any());
   }
 
-  @SuppressWarnings({"rawtypes", "unchecked"})
   @Test
   public void mustPerformRehydration() {
     final BlockDataGenerator blockDataGenerator = new BlockDataGenerator();
@@ -189,7 +188,6 @@ public class PrivacyBlockProcessorTest {
     return mockPrivateState;
   }
 
-  @SuppressWarnings("rawtypes")
   private ProtocolSpec mockProtocolSpec() {
     final ProtocolSpec protocolSpec = mock(ProtocolSpec.class);
     final MainnetTransactionProcessor mockPublicTransactionProcessor =

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java

@@ -60,9 +60,10 @@ class FullSyncTargetManager extends SyncTargetManager {
       return Optional.of(syncTarget);
     } else {
       LOG.warn(
-          "Disconnecting {} because world state is not available at common ancestor at block {}",
+          "Disconnecting {} because world state is not available at common ancestor at block {} ({})",
           syncTarget.peer(),
-          commonAncestor.getNumber());
+          commonAncestor.getNumber(),
+          commonAncestor.getHash());
       syncTarget.peer().disconnect(DisconnectReason.USELESS_PEER);
       return Optional.empty();
     }