Add metrics for accounts and storage reads (Flat databasae vs MPT) (#5315)

Add 8 metrics to evaluate the state of the flat database for accounts and storage slots
* Total number of calls (for both accounts and storage)
* Number of account or storage entries found in the flat database
* Number of storage slots located in the merkle trie but not found in the flat database
* Number of account or storage entries not found in either the flat database or the merkle trie

Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net>
Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>

CHANGELOG.md

@@ -7,6 +7,7 @@
 
 ### Additions and Improvements
 - Update most dependencies to latest version [#5269](https://github.com/hyperledger/besu/pull/5269)
+- Add metrics for accounts and storage reads (Flat database vs Merkle Patricia Trie) [#5315](https://github.com/hyperledger/besu/pull/5315)
 
 ### Bug Fixes
 

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -945,7 +945,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
             (BonsaiWorldStateKeyValueStorage) worldStateStorage,
             blockchain,
             Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
-            cachedMerkleTrieLoader);
+            cachedMerkleTrieLoader,
+            metricsSystem);
 
       case FOREST:
       default:

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java

@@ -39,6 +39,7 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.evm.worldstate.WorldState;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -68,25 +69,32 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
   public BonsaiWorldStateProvider(
       final StorageProvider provider,
       final Blockchain blockchain,
-      final CachedMerkleTrieLoader cachedMerkleTrieLoader) {
+      final CachedMerkleTrieLoader cachedMerkleTrieLoader,
+      final ObservableMetricsSystem metricsSystem) {
     this(
         (BonsaiWorldStateKeyValueStorage)
             provider.createWorldStateStorage(DataStorageFormat.BONSAI),
         blockchain,
         Optional.empty(),
-        cachedMerkleTrieLoader);
+        cachedMerkleTrieLoader,
+        metricsSystem);
   }
 
   public BonsaiWorldStateProvider(
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
       final Blockchain blockchain,
       final Optional<Long> maxLayersToLoad,
-      final CachedMerkleTrieLoader cachedMerkleTrieLoader) {
+      final CachedMerkleTrieLoader cachedMerkleTrieLoader,
+      final ObservableMetricsSystem metricsSystem) {
 
     // TODO: de-dup constructors
     this.trieLogManager =
         new CachedWorldStorageManager(
-            this, blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS));
+            this,
+            blockchain,
+            worldStateStorage,
+            metricsSystem,
+            maxLayersToLoad.orElse(RETAINED_LAYERS));
     this.blockchain = blockchain;
     this.worldStateStorage = worldStateStorage;
     this.persistedState = new BonsaiWorldState(this, worldStateStorage);

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java

@@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager;
 import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -41,24 +42,34 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
     implements BonsaiStorageSubscriber {
   private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class);
   private final BonsaiWorldStateProvider archive;
+  private final ObservableMetricsSystem metricsSystem;
 
   CachedWorldStorageManager(
       final BonsaiWorldStateProvider archive,
       final Blockchain blockchain,
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
       final long maxLayersToLoad,
-      final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash) {
+      final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash,
+      final ObservableMetricsSystem metricsSystem) {
     super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash);
     worldStateStorage.subscribe(this);
     this.archive = archive;
+    this.metricsSystem = metricsSystem;
   }
 
   public CachedWorldStorageManager(
       final BonsaiWorldStateProvider archive,
       final Blockchain blockchain,
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
+      final ObservableMetricsSystem metricsSystem,
       final long maxLayersToLoad) {
-    this(archive, blockchain, worldStateStorage, maxLayersToLoad, new ConcurrentHashMap<>());
+    this(
+        archive,
+        blockchain,
+        worldStateStorage,
+        maxLayersToLoad,
+        new ConcurrentHashMap<>(),
+        metricsSystem);
   }
 
   @Override
@@ -81,7 +92,8 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
         cachedBonsaiWorldView
             .get()
             .updateWorldStateStorage(
-                new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.worldStateStorage));
+                new BonsaiSnapshotWorldStateKeyValueStorage(
+                    forWorldState.worldStateStorage, metricsSystem));
       }
     } else {
       LOG.atDebug()
@@ -94,7 +106,8 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
             blockHeader.getHash(),
             new CachedBonsaiWorldView(
                 blockHeader,
-                new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.worldStateStorage)));
+                new BonsaiSnapshotWorldStateKeyValueStorage(
+                    forWorldState.worldStateStorage, metricsSystem)));
       } else {
         // otherwise, add the layer to the cache
         cachedWorldStatesByHash.put(

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java

@@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.bonsai.storage;
 
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.exception.StorageException;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage;
@@ -41,21 +42,30 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey
       final SnappedKeyValueStorage codeStorage,
       final SnappedKeyValueStorage storageStorage,
       final SnappedKeyValueStorage trieBranchStorage,
-      final KeyValueStorage trieLogStorage) {
-    super(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage);
+      final KeyValueStorage trieLogStorage,
+      final ObservableMetricsSystem metricsSystem) {
+    super(
+        accountStorage,
+        codeStorage,
+        storageStorage,
+        trieBranchStorage,
+        trieLogStorage,
+        metricsSystem);
     this.parentWorldStateStorage = parentWorldStateStorage;
     this.subscribeParentId = parentWorldStateStorage.subscribe(this);
   }
 
   public BonsaiSnapshotWorldStateKeyValueStorage(
-      final BonsaiWorldStateKeyValueStorage worldStateStorage) {
+      final BonsaiWorldStateKeyValueStorage worldStateStorage,
+      final ObservableMetricsSystem metricsSystem) {
     this(
         worldStateStorage,
         ((SnappableKeyValueStorage) worldStateStorage.accountStorage).takeSnapshot(),
         ((SnappableKeyValueStorage) worldStateStorage.codeStorage).takeSnapshot(),
         ((SnappableKeyValueStorage) worldStateStorage.storageStorage).takeSnapshot(),
         ((SnappableKeyValueStorage) worldStateStorage.trieBranchStorage).takeSnapshot(),
-        worldStateStorage.trieLogStorage);
+        worldStateStorage.trieLogStorage,
+        metricsSystem);
   }
 
   @Override

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java

@@ -22,6 +22,9 @@ import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie;
 import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory;
 import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
+import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
+import org.hyperledger.besu.plugin.services.metrics.Counter;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
 import org.hyperledger.besu.util.Subscribers;
@@ -55,19 +58,33 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
   protected final KeyValueStorage trieBranchStorage;
   protected final KeyValueStorage trieLogStorage;
 
+  private final Counter getAccountCounter;
+  private final Counter getAccountFlatDatabaseCounter;
+  private final Counter getAccountMerkleTrieCounter;
+  private final Counter getAccountMissingMerkleTrieCounter;
+
+  private final Counter getStorageValueCounter;
+  private final Counter getStorageValueFlatDatabaseCounter;
+  private final Counter getStorageValueMerkleTrieCounter;
+  private final Counter getStorageValueMissingMerkleTrieCounter;
+
+  protected final ObservableMetricsSystem metricsSystem;
+
   private final AtomicBoolean shouldClose = new AtomicBoolean(false);
 
   protected final AtomicBoolean isClosed = new AtomicBoolean(false);
 
   protected final Subscribers<BonsaiStorageSubscriber> subscribers = Subscribers.create();
 
-  public BonsaiWorldStateKeyValueStorage(final StorageProvider provider) {
+  public BonsaiWorldStateKeyValueStorage(
+      final StorageProvider provider, final ObservableMetricsSystem metricsSystem) {
     this(
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE),
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE),
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE),
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE),
-        provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE));
+        provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE),
+        metricsSystem);
   }
 
   public BonsaiWorldStateKeyValueStorage(
@@ -75,12 +92,62 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
       final KeyValueStorage codeStorage,
       final KeyValueStorage storageStorage,
       final KeyValueStorage trieBranchStorage,
-      final KeyValueStorage trieLogStorage) {
+      final KeyValueStorage trieLogStorage,
+      final ObservableMetricsSystem metricsSystem) {
     this.accountStorage = accountStorage;
     this.codeStorage = codeStorage;
     this.storageStorage = storageStorage;
     this.trieBranchStorage = trieBranchStorage;
     this.trieLogStorage = trieLogStorage;
+    this.metricsSystem = metricsSystem;
+
+    getAccountCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_account_total",
+            "Total number of calls to getAccount");
+
+    getAccountFlatDatabaseCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_account_flat_database",
+            "Number of accounts found in the flat database");
+
+    getAccountMerkleTrieCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_account_merkle_trie",
+            "Number of accounts not found in the flat database, but found in the merkle trie");
+
+    getAccountMissingMerkleTrieCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_account_missing_merkle_trie",
+            "Number of accounts not found (either in the flat database or the merkle trie)");
+
+    getStorageValueCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_storagevalue_total",
+            "Total number of calls to getStorageValueBySlotHash");
+
+    getStorageValueFlatDatabaseCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_storagevalue_flat_database",
+            "Number of storage slots found in the flat database");
+
+    getStorageValueMerkleTrieCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_storagevalue_merkle_trie",
+            "Number of storage slots not found in the flat database, but found in the merkle trie");
+
+    getStorageValueMissingMerkleTrieCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_storagevalue_missing_merkle_trie",
+            "Number of storage slots not found (either in the flat database or in the merkle trie)");
   }
 
   @Override
@@ -96,6 +163,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
   }
 
   public Optional<Bytes> getAccount(final Hash accountHash) {
+    getAccountCounter.inc();
     Optional<Bytes> response = accountStorage.get(accountHash.toArrayUnsafe()).map(Bytes::wrap);
     if (response.isEmpty()) {
       // after a snapsync/fastsync we only have the trie branches.
@@ -107,8 +175,13 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
                         this::getAccountStateTrieNode, Function.identity(), Function.identity()),
                     Bytes32.wrap(worldStateRootHash.get()))
                 .get(accountHash);
+        if (response.isEmpty()) getAccountMissingMerkleTrieCounter.inc();
+        else getAccountMerkleTrieCounter.inc();
       }
+    } else {
+      getAccountFlatDatabaseCounter.inc();
     }
+
     return response;
   }
 
@@ -176,6 +249,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
       final Supplier<Optional<Hash>> storageRootSupplier,
       final Hash accountHash,
       final Hash slotHash) {
+    getStorageValueCounter.inc();
     Optional<Bytes> response =
         storageStorage
             .get(Bytes.concatenate(accountHash, slotHash).toArrayUnsafe())
@@ -193,7 +267,11 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
                     storageRoot.get())
                 .get(slotHash)
                 .map(bytes -> Bytes32.leftPad(RLP.decodeValue(bytes)));
+        if (response.isEmpty()) getStorageValueMissingMerkleTrieCounter.inc();
+        else getStorageValueMerkleTrieCounter.inc();
       }
+    } else {
+      getStorageValueFlatDatabaseCounter.inc();
     }
     return response;
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java

@@ -16,6 +16,7 @@
 package org.hyperledger.besu.ethereum.bonsai.storage;
 
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage;
 import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage;
@@ -30,7 +31,8 @@ public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyVal
         new LayeredKeyValueStorage(parent.storageStorage),
         new LayeredKeyValueStorage(parent.trieBranchStorage),
         parent.trieLogStorage,
-        parent);
+        parent,
+        parent.metricsSystem);
   }
 
   public BonsaiWorldStateLayerStorage(
@@ -39,8 +41,16 @@ public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyVal
       final SnappedKeyValueStorage storageStorage,
       final SnappedKeyValueStorage trieBranchStorage,
       final KeyValueStorage trieLogStorage,
-      final BonsaiWorldStateKeyValueStorage parent) {
-    super(parent, accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage);
+      final BonsaiWorldStateKeyValueStorage parent,
+      final ObservableMetricsSystem metricsSystem) {
+    super(
+        parent,
+        accountStorage,
+        codeStorage,
+        storageStorage,
+        trieBranchStorage,
+        trieLogStorage,
+        metricsSystem);
   }
 
   @Override
@@ -51,6 +61,7 @@ public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyVal
         ((LayeredKeyValueStorage) storageStorage).clone(),
         ((LayeredKeyValueStorage) trieBranchStorage).clone(),
         trieLogStorage,
-        parentWorldStateStorage);
+        parentWorldStateStorage,
+        metricsSystem);
   }
 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/GoQuorumKeyValueStorageProvider.java

@@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.storage.keyvalue;
 import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
 
@@ -27,21 +28,24 @@ public class GoQuorumKeyValueStorageProvider extends KeyValueStorageProvider {
   public GoQuorumKeyValueStorageProvider(
       final Function<SegmentIdentifier, KeyValueStorage> storageCreator,
       final KeyValueStorage worldStatePreimageStorage,
-      final boolean segmentIsolationSupported) {
-    super(storageCreator, worldStatePreimageStorage, segmentIsolationSupported);
+      final boolean segmentIsolationSupported,
+      final ObservableMetricsSystem metricsSystem) {
+    super(storageCreator, worldStatePreimageStorage, segmentIsolationSupported, metricsSystem);
   }
 
   public GoQuorumKeyValueStorageProvider(
       final Function<SegmentIdentifier, KeyValueStorage> storageCreator,
       final KeyValueStorage worldStatePreimageStorage,
       final KeyValueStorage privateWorldStatePreimageStorage,
-      final boolean segmentIsolationSupported) {
+      final boolean segmentIsolationSupported,
+      final ObservableMetricsSystem metricsSystem) {
     super(
         storageCreator,
         worldStatePreimageStorage,
         privateWorldStatePreimageStorage,
         segmentIsolationSupported,
-        SNAPSHOT_ISOLATION_UNSUPPORTED);
+        SNAPSHOT_ISOLATION_UNSUPPORTED,
+        metricsSystem);
   }
 
   @Override

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java

@@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
 import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage;
@@ -44,16 +45,19 @@ public class KeyValueStorageProvider implements StorageProvider {
   private final boolean isWorldStateIterable;
   private final boolean isWorldStateSnappable;
   protected final Map<SegmentIdentifier, KeyValueStorage> storageInstances = new HashMap<>();
+  private final ObservableMetricsSystem metricsSystem;
 
   public KeyValueStorageProvider(
       final Function<SegmentIdentifier, KeyValueStorage> storageCreator,
       final KeyValueStorage worldStatePreimageStorage,
-      final boolean segmentIsolationSupported) {
+      final boolean segmentIsolationSupported,
+      final ObservableMetricsSystem metricsSystem) {
     this.storageCreator = storageCreator;
     this.worldStatePreimageStorage = worldStatePreimageStorage;
     this.privateWorldStatePreimageStorage = null;
     this.isWorldStateIterable = segmentIsolationSupported;
     this.isWorldStateSnappable = SNAPSHOT_ISOLATION_UNSUPPORTED;
+    this.metricsSystem = metricsSystem;
   }
 
   public KeyValueStorageProvider(
@@ -61,12 +65,14 @@ public class KeyValueStorageProvider implements StorageProvider {
       final KeyValueStorage worldStatePreimageStorage,
       final KeyValueStorage privateWorldStatePreimageStorage,
       final boolean segmentIsolationSupported,
-      final boolean storageSnapshotIsolationSupported) {
+      final boolean storageSnapshotIsolationSupported,
+      final ObservableMetricsSystem metricsSystem) {
     this.storageCreator = storageCreator;
     this.worldStatePreimageStorage = worldStatePreimageStorage;
     this.privateWorldStatePreimageStorage = privateWorldStatePreimageStorage;
     this.isWorldStateIterable = segmentIsolationSupported;
     this.isWorldStateSnappable = storageSnapshotIsolationSupported;
+    this.metricsSystem = metricsSystem;
   }
 
   @Override
@@ -79,7 +85,7 @@ public class KeyValueStorageProvider implements StorageProvider {
   @Override
   public WorldStateStorage createWorldStateStorage(final DataStorageFormat dataStorageFormat) {
     if (dataStorageFormat.equals(DataStorageFormat.BONSAI)) {
-      return new BonsaiWorldStateKeyValueStorage(this);
+      return new BonsaiWorldStateKeyValueStorage(this, metricsSystem);
     } else {
       return new WorldStateKeyValueStorage(
           getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.WORLD_STATE));

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProviderBuilder.java

@@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.storage.keyvalue;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.BesuConfiguration;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
@@ -74,14 +75,16 @@ public class KeyValueStorageProviderBuilder {
           segment -> storageFactory.create(segment, commonConfiguration, metricsSystem),
           worldStatePreImageStorage,
           privateWorldStatePreImageStorage,
-          storageFactory.isSegmentIsolationSupported());
+          storageFactory.isSegmentIsolationSupported(),
+          (ObservableMetricsSystem) metricsSystem);
     } else {
       return new KeyValueStorageProvider(
           segment -> storageFactory.create(segment, commonConfiguration, metricsSystem),
           worldStatePreImageStorage,
           privateWorldStatePreImageStorage,
           storageFactory.isSegmentIsolationSupported(),
-          storageFactory.isSnapshotIsolationSupported());
+          storageFactory.isSnapshotIsolationSupported(),
+          (ObservableMetricsSystem) metricsSystem);
     }
   }
 }

ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java

@@ -40,7 +40,8 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider {
         new InMemoryKeyValueStorage(),
         new InMemoryKeyValueStorage(),
         SEGMENT_ISOLATION_SUPPORTED,
-        SNAPSHOT_ISOLATION_UNSUPPORTED);
+        SNAPSHOT_ISOLATION_UNSUPPORTED,
+        new NoOpMetricsSystem());
   }
 
   public static MutableBlockchain createInMemoryBlockchain(final Block genesisBlock) {
@@ -70,7 +71,10 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider {
     final CachedMerkleTrieLoader cachedMerkleTrieLoader =
         new CachedMerkleTrieLoader(new NoOpMetricsSystem());
     return new BonsaiWorldStateProvider(
-        inMemoryKeyValueStorageProvider, blockchain, cachedMerkleTrieLoader);
+        inMemoryKeyValueStorageProvider,
+        blockchain,
+        cachedMerkleTrieLoader,
+        new NoOpMetricsSystem());
   }
 
   public static MutableWorldState createInMemoryWorldState() {

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java

@@ -82,14 +82,16 @@ public class BlockImportExceptionHandlingTest {
   private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider();
 
   private final WorldStateStorage worldStateStorage =
-      new BonsaiWorldStateKeyValueStorage(storageProvider);
+      new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem());
 
   private CachedMerkleTrieLoader cachedMerkleTrieLoader;
 
   private final WorldStateArchive worldStateArchive =
       // contains a BonsaiWorldState which we need to spy on.
       // do we need to also test with a DefaultWorldStateArchive?
-      spy(new BonsaiWorldStateProvider(storageProvider, blockchain, cachedMerkleTrieLoader));
+      spy(
+          new BonsaiWorldStateProvider(
+              storageProvider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem()));
 
   private final BonsaiWorldState persisted =
       spy(

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java

@@ -114,7 +114,8 @@ public abstract class AbstractIsolationTests {
             bonsaiWorldStateStorage,
             blockchain,
             Optional.of(16L),
-            new CachedMerkleTrieLoader(new NoOpMetricsSystem()));
+            new CachedMerkleTrieLoader(new NoOpMetricsSystem()),
+            new NoOpMetricsSystem());
     var ws = archive.getMutable();
     genesisState.writeStateTo(ws);
     protocolContext = new ProtocolContext(blockchain, archive, null);

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java

@@ -88,7 +88,7 @@ public class BonsaiWorldStateArchiveTest {
     bonsaiWorldStateArchive =
         new BonsaiWorldStateProvider(
             trieLogManager,
-            new BonsaiWorldStateKeyValueStorage(storageProvider),
+            new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()),
             blockchain,
             new CachedMerkleTrieLoader(new NoOpMetricsSystem()));
 
@@ -100,10 +100,11 @@ public class BonsaiWorldStateArchiveTest {
   public void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() {
     bonsaiWorldStateArchive =
         new BonsaiWorldStateProvider(
-            new BonsaiWorldStateKeyValueStorage(storageProvider),
+            new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()),
             blockchain,
             Optional.of(512L),
-            new CachedMerkleTrieLoader(new NoOpMetricsSystem()));
+            new CachedMerkleTrieLoader(new NoOpMetricsSystem()),
+            new NoOpMetricsSystem());
     final BlockHeader blockHeader = blockBuilder.number(0).buildHeader();
     final BlockHeader chainHead = blockBuilder.number(512).buildHeader();
     when(blockchain.getChainHeadHeader()).thenReturn(chainHead);
@@ -117,7 +118,7 @@ public class BonsaiWorldStateArchiveTest {
     bonsaiWorldStateArchive =
         new BonsaiWorldStateProvider(
             trieLogManager,
-            new BonsaiWorldStateKeyValueStorage(storageProvider),
+            new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()),
             blockchain,
             new CachedMerkleTrieLoader(new NoOpMetricsSystem()));
     final BlockHeader blockHeader = blockBuilder.number(0).buildHeader();
@@ -141,7 +142,8 @@ public class BonsaiWorldStateArchiveTest {
     when(keyValueStorage.startTransaction()).thenReturn(mock(KeyValueStorageTransaction.class));
     when(trieLogManager.getTrieLogLayer(any())).thenReturn(Optional.of(mock(TrieLogLayer.class)));
 
-    var worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider);
+    var worldStateStorage =
+        new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem());
     bonsaiWorldStateArchive =
         spy(
             new BonsaiWorldStateProvider(
@@ -166,7 +168,8 @@ public class BonsaiWorldStateArchiveTest {
 
     when(keyValueStorage.startTransaction()).thenReturn(mock(KeyValueStorageTransaction.class));
 
-    var worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider);
+    var worldStateStorage =
+        new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem());
     bonsaiWorldStateArchive =
         spy(
             new BonsaiWorldStateProvider(
@@ -199,7 +202,8 @@ public class BonsaiWorldStateArchiveTest {
 
     when(trieLogManager.getTrieLogLayer(any())).thenReturn(Optional.of(mock(TrieLogLayer.class)));
 
-    var worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider);
+    var worldStateStorage =
+        new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem());
 
     bonsaiWorldStateArchive =
         spy(
@@ -244,7 +248,7 @@ public class BonsaiWorldStateArchiveTest {
         spy(
             new BonsaiWorldStateProvider(
                 trieLogManager,
-                new BonsaiWorldStateKeyValueStorage(storageProvider),
+                new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()),
                 blockchain,
                 new CachedMerkleTrieLoader(new NoOpMetricsSystem())));
 

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java

@@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie;
 import org.hyperledger.besu.ethereum.trie.StorageEntriesCollector;
 import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie;
 import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
+import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 
 import java.util.TreeMap;
 
@@ -289,6 +290,7 @@ public class BonsaiWorldStateKeyValueStorageTest {
   }
 
   private BonsaiWorldStateKeyValueStorage emptyStorage() {
-    return new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider());
+    return new BonsaiWorldStateKeyValueStorage(
+        new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem());
   }
 }

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java

@@ -46,7 +46,7 @@ public class CachedMerkleTrieLoaderTest {
   private CachedMerkleTrieLoader merkleTrieLoader;
   private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider();
   private final BonsaiWorldStateKeyValueStorage inMemoryWorldState =
-      Mockito.spy(new BonsaiWorldStateKeyValueStorage(storageProvider));
+      Mockito.spy(new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()));
 
   final List<Address> accounts =
       List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee"));
@@ -67,7 +67,8 @@ public class CachedMerkleTrieLoaderTest {
         inMemoryWorldState, Hash.wrap(trie.getRootHash()), accounts.get(0));
 
     final BonsaiWorldStateKeyValueStorage emptyStorage =
-        new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider());
+        new BonsaiWorldStateKeyValueStorage(
+            new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem());
     StoredMerklePatriciaTrie<Bytes, Bytes> cachedTrie =
         new StoredMerklePatriciaTrie<>(
             (location, hash) ->
@@ -103,7 +104,8 @@ public class CachedMerkleTrieLoaderTest {
 
     final List<Bytes> cachedSlots = new ArrayList<>();
     final BonsaiWorldStateKeyValueStorage emptyStorage =
-        new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider());
+        new BonsaiWorldStateKeyValueStorage(
+            new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem());
     final StoredMerklePatriciaTrie<Bytes, Bytes> cachedTrie =
         new StoredMerklePatriciaTrie<>(
             (location, hash) ->

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java

@@ -123,7 +123,9 @@ public class LogRollingTests {
     final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider();
     final CachedMerkleTrieLoader cachedMerkleTrieLoader =
         new CachedMerkleTrieLoader(new NoOpMetricsSystem());
-    archive = new BonsaiWorldStateProvider(provider, blockchain, cachedMerkleTrieLoader);
+    archive =
+        new BonsaiWorldStateProvider(
+            provider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem());
     accountStorage =
         (InMemoryKeyValueStorage)
             provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
@@ -145,7 +147,8 @@ public class LogRollingTests {
     final CachedMerkleTrieLoader secondOptimizedMerkleTrieLoader =
         new CachedMerkleTrieLoader(new NoOpMetricsSystem());
     secondArchive =
-        new BonsaiWorldStateProvider(secondProvider, blockchain, secondOptimizedMerkleTrieLoader);
+        new BonsaiWorldStateProvider(
+            secondProvider, blockchain, secondOptimizedMerkleTrieLoader, new NoOpMetricsSystem());
     secondAccountStorage =
         (InMemoryKeyValueStorage)
             secondProvider.getStorageBySegmentIdentifier(
@@ -174,7 +177,12 @@ public class LogRollingTests {
         new BonsaiWorldState(
             archive,
             new BonsaiWorldStateKeyValueStorage(
-                accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage));
+                accountStorage,
+                codeStorage,
+                storageStorage,
+                trieBranchStorage,
+                trieLogStorage,
+                new NoOpMetricsSystem()));
     final WorldUpdater updater = worldState.updater();
 
     final MutableAccount mutableAccount =
@@ -192,7 +200,8 @@ public class LogRollingTests {
                 secondCodeStorage,
                 secondStorageStorage,
                 secondTrieBranchStorage,
-                secondTrieLogStorage));
+                secondTrieLogStorage,
+                new NoOpMetricsSystem()));
     final BonsaiWorldStateUpdateAccumulator secondUpdater =
         (BonsaiWorldStateUpdateAccumulator) secondWorldState.updater();
 
@@ -223,7 +232,12 @@ public class LogRollingTests {
         new BonsaiWorldState(
             archive,
             new BonsaiWorldStateKeyValueStorage(
-                accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage));
+                accountStorage,
+                codeStorage,
+                storageStorage,
+                trieBranchStorage,
+                trieLogStorage,
+                new NoOpMetricsSystem()));
 
     final WorldUpdater updater = worldState.updater();
     final MutableAccount mutableAccount =
@@ -249,7 +263,8 @@ public class LogRollingTests {
                 secondCodeStorage,
                 secondStorageStorage,
                 secondTrieBranchStorage,
-                secondTrieLogStorage));
+                secondTrieLogStorage,
+                new NoOpMetricsSystem()));
     final BonsaiWorldStateUpdateAccumulator secondUpdater =
         (BonsaiWorldStateUpdateAccumulator) secondWorldState.updater();
 
@@ -281,7 +296,12 @@ public class LogRollingTests {
         new BonsaiWorldState(
             archive,
             new BonsaiWorldStateKeyValueStorage(
-                accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage));
+                accountStorage,
+                codeStorage,
+                storageStorage,
+                trieBranchStorage,
+                trieLogStorage,
+                new NoOpMetricsSystem()));
 
     final WorldUpdater updater = worldState.updater();
     final MutableAccount mutableAccount =
@@ -314,7 +334,8 @@ public class LogRollingTests {
                 secondCodeStorage,
                 secondStorageStorage,
                 secondTrieBranchStorage,
-                secondTrieLogStorage));
+                secondTrieLogStorage,
+                new NoOpMetricsSystem()));
 
     final WorldUpdater secondUpdater = secondWorldState.updater();
     final MutableAccount secondMutableAccount =

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java

@@ -47,7 +47,8 @@ public class RollingImport {
     final CachedMerkleTrieLoader cachedMerkleTrieLoader =
         new CachedMerkleTrieLoader(new NoOpMetricsSystem());
     final BonsaiWorldStateProvider archive =
-        new BonsaiWorldStateProvider(provider, null, cachedMerkleTrieLoader);
+        new BonsaiWorldStateProvider(
+            provider, null, cachedMerkleTrieLoader, new NoOpMetricsSystem());
     final InMemoryKeyValueStorage accountStorage =
         (InMemoryKeyValueStorage)
             provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
@@ -68,7 +69,12 @@ public class RollingImport {
         new BonsaiWorldState(
             archive,
             new BonsaiWorldStateKeyValueStorage(
-                accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage));
+                accountStorage,
+                codeStorage,
+                storageStorage,
+                trieBranchStorage,
+                trieLogStorage,
+                new NoOpMetricsSystem()));
 
     int count = 0;
     while (!reader.isDone()) {

ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java

@@ -30,6 +30,7 @@ import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProce
 import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
+import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
 import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues;
 import org.hyperledger.besu.testutil.TestClock;
@@ -84,7 +85,8 @@ public class FastWorldDownloadStateTest {
   public void setUp() {
     if (storageFormat == DataStorageFormat.BONSAI) {
       worldStateStorage =
-          new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider());
+          new BonsaiWorldStateKeyValueStorage(
+              new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem());
     } else {
       worldStateStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
     }

ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java

@@ -107,7 +107,8 @@ public class SnapWorldDownloadStateTest {
 
     if (storageFormat == DataStorageFormat.BONSAI) {
       worldStateStorage =
-          new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider());
+          new BonsaiWorldStateKeyValueStorage(
+              new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem());
     } else {
       worldStateStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
     }