Retry mechanism for block creation (#4407)

* Retry mechanism for block creation

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Update CHANGELOG

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Always keep 1 thread active in the computation executors

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Surface StorageException when building a block for finer filtering

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Log successful block created at info

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Reformat block proposal logs

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Remove test code

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Upgrade besu-native to 0.6.0 and use Blake2bf native implementation if available by default [#4264](https://github.com/hyperledger/besu/pull/4264)
 
 ### Bug Fixes
+- Retry block creation if there is a transient error and we still have time, to mitigate empty block issue [#4407](https://github.com/hyperledger/besu/pull/4407)
 - Fix StacklessClosedChannelException in Besu and resulted timeout errors in CL clients ([#4398](https://github.com/hyperledger/besu/issues/4398), [#4400](https://github.com/hyperledger/besu/issues/4400))
 
 ## 22.7.2

besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java

@@ -66,6 +66,7 @@ public class MergeBesuControllerBuilder extends BesuControllerBuilder {
     return createTransitionMiningCoordinator(
         protocolSchedule,
         protocolContext,
+        ethProtocolManager.ethContext(),
         transactionPool,
         miningParameters,
         syncState,
@@ -131,6 +132,7 @@ public class MergeBesuControllerBuilder extends BesuControllerBuilder {
   protected MiningCoordinator createTransitionMiningCoordinator(
       final ProtocolSchedule protocolSchedule,
       final ProtocolContext protocolContext,
+      final EthContext ethContext,
       final TransactionPool transactionPool,
       final MiningParameters miningParameters,
       final SyncState syncState,
@@ -141,6 +143,7 @@ public class MergeBesuControllerBuilder extends BesuControllerBuilder {
     return new MergeCoordinator(
         protocolContext,
         protocolSchedule,
+        ethContext,
         transactionPool.getPendingTransactions(),
         miningParameters,
         backwardSyncContext);

besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java

@@ -131,6 +131,7 @@ public class TransitionBesuControllerBuilder extends BesuControllerBuilder {
             mergeBesuControllerBuilder.createTransitionMiningCoordinator(
                 transitionProtocolSchedule,
                 protocolContext,
+                ethProtocolManager.ethContext(),
                 transactionPool,
                 transitionMiningParameters,
                 syncState,

consensus/merge/build.gradle

@@ -48,8 +48,11 @@ dependencies {
   implementation 'org.apache.tuweni:tuweni-units'
 
   testImplementation project(path: ':consensus:common', configuration: 'testArtifacts')
+  testImplementation project(':crypto')
   testImplementation project(path: ':crypto', configuration: 'testSupportArtifacts')
   testImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts')
+  testImplementation project(':metrics:core')
+  testImplementation project(path: ':metrics:core', configuration: 'testSupportArtifacts')
   testImplementation project(':testutil')
 
   testImplementation 'junit:junit'

consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java

@@ -17,6 +17,7 @@ package org.hyperledger.besu.consensus.merge.blockcreation;
 import static org.hyperledger.besu.consensus.merge.TransitionUtils.isTerminalProofOfWorkBlock;
 import static org.hyperledger.besu.consensus.merge.blockcreation.MergeMiningCoordinator.ForkchoiceResult.Status.INVALID;
 import static org.hyperledger.besu.consensus.merge.blockcreation.MergeMiningCoordinator.ForkchoiceResult.Status.INVALID_PAYLOAD_ATTRIBUTES;
+import static org.hyperledger.besu.util.FutureUtils.exceptionallyCompose;
 import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda;
 
 import org.hyperledger.besu.consensus.merge.MergeContext;
@@ -33,13 +34,17 @@ import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.Difficulty;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.Transaction;
+import org.hyperledger.besu.ethereum.eth.manager.EthContext;
 import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncContext;
 import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BadChainListener;
 import org.hyperledger.besu.ethereum.eth.transactions.sorter.AbstractPendingTransactionsSorter;
 import org.hyperledger.besu.ethereum.mainnet.AbstractGasLimitSpecification;
 import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
+import org.hyperledger.besu.plugin.services.exception.StorageException;
 
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -47,6 +52,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
 
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
@@ -64,17 +70,20 @@ public class MergeCoordinator implements MergeMiningCoordinator, BadChainListene
   protected final AtomicReference<BlockHeader> latestDescendsFromTerminal = new AtomicReference<>();
   protected final MergeContext mergeContext;
   protected final ProtocolContext protocolContext;
+  protected final EthContext ethContext;
   protected final BackwardSyncContext backwardSyncContext;
   protected final ProtocolSchedule protocolSchedule;
 
   public MergeCoordinator(
       final ProtocolContext protocolContext,
       final ProtocolSchedule protocolSchedule,
+      final EthContext ethContext,
       final AbstractPendingTransactionsSorter pendingTransactions,
       final MiningParameters miningParams,
       final BackwardSyncContext backwardSyncContext) {
     this.protocolContext = protocolContext;
     this.protocolSchedule = protocolSchedule;
+    this.ethContext = ethContext;
     this.mergeContext = protocolContext.getConsensusContext(MergeContext.class);
     this.miningParameters = miningParams;
     this.backwardSyncContext = backwardSyncContext;
@@ -188,28 +197,87 @@ public class MergeCoordinator implements MergeMiningCoordinator, BadChainListene
           result.errorMessage);
     }
 
+    tryToBuildBetterBlock(timestamp, random, payloadIdentifier, mergeBlockCreator);
+
+    return payloadIdentifier;
+  }
+
+  private void tryToBuildBetterBlock(
+      final Long timestamp,
+      final Bytes32 random,
+      final PayloadIdentifier payloadIdentifier,
+      final MergeBlockCreator mergeBlockCreator) {
+
+    long remainingTime = miningParameters.getPosBlockCreationTimeout();
+    final Supplier<Block> blockCreator =
+        () -> mergeBlockCreator.createBlock(Optional.empty(), random, timestamp);
     // start working on a full block and update the payload value and candidate when it's ready
-    CompletableFuture.supplyAsync(
-            () -> mergeBlockCreator.createBlock(Optional.empty(), random, timestamp))
-        .orTimeout(12, TimeUnit.SECONDS)
-        .whenComplete(
-            (bestBlock, throwable) -> {
-              if (throwable != null) {
-                LOG.debug("something went wrong creating block", throwable);
+    final long startedAt = System.currentTimeMillis();
+    retryBlockCreation(payloadIdentifier, blockCreator, remainingTime)
+        .thenAccept(
+            bestBlock -> {
+              final var resultBest = validateBlock(bestBlock);
+              if (resultBest.blockProcessingOutputs.isPresent()) {
+                mergeContext.putPayloadById(payloadIdentifier, bestBlock);
+                LOG.info(
+                    "Successfully built block {} for proposal identified by {}, with {} transactions, in {}ms",
+                    bestBlock.toLogString(),
+                    payloadIdentifier.toHexString(),
+                    bestBlock.getBody().getTransactions().size(),
+                    System.currentTimeMillis() - startedAt);
               } else {
-                final var resultBest = validateBlock(bestBlock);
-                if (resultBest.blockProcessingOutputs.isPresent()) {
-                  mergeContext.putPayloadById(payloadIdentifier, bestBlock);
-                } else {
-                  LOG.warn(
-                      "failed to execute block proposal {}, reason {}",
-                      bestBlock.getHash(),
-                      resultBest.errorMessage);
-                }
+                LOG.warn(
+                    "Block {} built for proposal identified by {}, is not valid reason {}",
+                    bestBlock.getHash(),
+                    payloadIdentifier.toHexString(),
+                    resultBest.errorMessage);
               }
             });
+  }
 
-    return payloadIdentifier;
+  private CompletableFuture<Block> retryBlockCreation(
+      final PayloadIdentifier payloadIdentifier,
+      final Supplier<Block> blockCreator,
+      final long remainingTime) {
+    final long startedAt = System.currentTimeMillis();
+
+    debugLambda(
+        LOG,
+        "Block creation started for payload id {}, remaining time is {}ms",
+        payloadIdentifier::toShortHexString,
+        () -> remainingTime);
+
+    return exceptionallyCompose(
+        ethContext
+            .getScheduler()
+            .scheduleComputationTask(blockCreator)
+            .orTimeout(remainingTime, TimeUnit.MILLISECONDS),
+        throwable -> {
+          if (canRetryBlockCreation(throwable)) {
+            final long newRemainingTime = remainingTime - (System.currentTimeMillis() - startedAt);
+            debugLambda(
+                LOG,
+                "Retrying block creation for payload id {}, remaining time is {}ms, last error {}",
+                payloadIdentifier::toShortHexString,
+                () -> newRemainingTime,
+                () -> logException(throwable));
+            return retryBlockCreation(payloadIdentifier, blockCreator, newRemainingTime);
+          } else {
+            debugLambda(
+                LOG,
+                "Something went wrong creating block for payload id {}, error {}",
+                payloadIdentifier::toShortHexString,
+                () -> logException(throwable));
+          }
+          return CompletableFuture.failedFuture(throwable);
+        });
+  }
+
+  private boolean canRetryBlockCreation(final Throwable throwable) {
+    if (throwable instanceof StorageException) {
+      return true;
+    }
+    return false;
   }
 
   @Override
@@ -633,4 +701,12 @@ public class MergeCoordinator implements MergeMiningCoordinator, BadChainListene
   private boolean isPoSHeader(final BlockHeader header) {
     return header.getDifficulty().equals(Difficulty.ZERO);
   }
+
+  private String logException(final Throwable throwable) {
+    final StringWriter sw = new StringWriter();
+    final PrintWriter pw = new PrintWriter(sw);
+    throwable.printStackTrace(pw);
+    pw.flush();
+    return sw.toString();
+  }
 }

consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java

@@ -15,20 +15,28 @@
 package org.hyperledger.besu.consensus.merge.blockcreation;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
 import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain;
 import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryWorldStateArchive;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import org.hyperledger.besu.config.MergeConfigOptions;
 import org.hyperledger.besu.consensus.merge.MergeContext;
 import org.hyperledger.besu.consensus.merge.blockcreation.MergeMiningCoordinator.ForkchoiceResult;
+import org.hyperledger.besu.crypto.KeyPair;
+import org.hyperledger.besu.crypto.SECPPrivateKey;
+import org.hyperledger.besu.crypto.SignatureAlgorithm;
+import org.hyperledger.besu.crypto.SignatureAlgorithmFactory;
 import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.datatypes.Wei;
@@ -41,17 +49,28 @@ import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture;
 import org.hyperledger.besu.ethereum.core.Difficulty;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
+import org.hyperledger.besu.ethereum.core.Transaction;
+import org.hyperledger.besu.ethereum.core.TransactionTestFixture;
+import org.hyperledger.besu.ethereum.eth.manager.EthContext;
+import org.hyperledger.besu.ethereum.eth.manager.EthScheduler;
 import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncContext;
-import org.hyperledger.besu.ethereum.eth.transactions.sorter.AbstractPendingTransactionsSorter;
+import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;
+import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.LondonFeeMarket;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
+import org.hyperledger.besu.metrics.StubMetricsSystem;
+import org.hyperledger.besu.plugin.services.exception.StorageException;
+import org.hyperledger.besu.testutil.TestClock;
 
+import java.time.ZoneId;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
 
+import com.google.common.base.Suppliers;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
 import org.junit.Before;
@@ -59,14 +78,31 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mock;
+import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
 @RunWith(MockitoJUnitRunner.class)
 public class MergeCoordinatorTest implements MergeGenesisConfigHelper {
 
-  @Mock AbstractPendingTransactionsSorter mockSorter;
+  private static final com.google.common.base.Supplier<SignatureAlgorithm> SIGNATURE_ALGORITHM =
+      Suppliers.memoize(SignatureAlgorithmFactory::getInstance);
+
+  private static final SECPPrivateKey PRIVATE_KEY1 =
+      SIGNATURE_ALGORITHM
+          .get()
+          .createPrivateKey(
+              Bytes32.fromHexString(
+                  "ae6ae8e5ccbfb04590405997ee2d52d2b330726137b875053c36d94e974d162f"));
+  private static final KeyPair KEYS1 =
+      new KeyPair(PRIVATE_KEY1, SIGNATURE_ALGORITHM.get().createPublicKey(PRIVATE_KEY1));
   @Mock MergeContext mergeContext;
   @Mock BackwardSyncContext backwardSyncContext;
+  @Mock EthContext ethContext;
+  @Mock EthScheduler ethScheduler;
+  private final Address coinbase = genesisAllocations(getPosGenesisConfigFile()).findFirst().get();
+
+  @Spy
+  MiningParameters miningParameters = new MiningParameters.Builder().coinbase(coinbase).build();
 
   private MergeCoordinator coordinator;
   private ProtocolContext protocolContext;
@@ -81,30 +117,45 @@ public class MergeCoordinatorTest implements MergeGenesisConfigHelper {
       spy(createInMemoryBlockchain(genesisState.getBlock()));
 
   private final Address suggestedFeeRecipient = Address.ZERO;
-  private final Address coinbase = genesisAllocations(getPosGenesisConfigFile()).findFirst().get();
   private final BlockHeaderTestFixture headerGenerator = new BlockHeaderTestFixture();
   private final BaseFeeMarket feeMarket =
       new LondonFeeMarket(0, genesisState.getBlock().getHeader().getBaseFee());
 
+  private final org.hyperledger.besu.metrics.StubMetricsSystem metricsSystem =
+      new StubMetricsSystem();
+
+  private final BaseFeePendingTransactionsSorter transactions =
+      new BaseFeePendingTransactionsSorter(
+          ImmutableTransactionPoolConfiguration.builder().txPoolMaxSize(10).build(),
+          TestClock.system(ZoneId.systemDefault()),
+          metricsSystem,
+          MergeCoordinatorTest::mockBlockHeader);
+
+  final Transaction localTransaction0 = createTransaction(0);
+
   @Before
   public void setUp() {
     when(mergeContext.as(MergeContext.class)).thenReturn(mergeContext);
     when(mergeContext.getTerminalTotalDifficulty())
         .thenReturn(genesisState.getBlock().getHeader().getDifficulty().plus(1L));
     when(mergeContext.getTerminalPoWBlock()).thenReturn(Optional.of(terminalPowBlock()));
-
     protocolContext = new ProtocolContext(blockchain, worldStateArchive, mergeContext);
     var mutable = worldStateArchive.getMutable();
     genesisState.writeStateTo(mutable);
     mutable.persist(null);
 
+    when(ethContext.getScheduler()).thenReturn(ethScheduler);
+    when(ethScheduler.scheduleComputationTask(any()))
+        .thenAnswer(i -> CompletableFuture.completedFuture(i.getArgument(0, Supplier.class).get()));
+
     MergeConfigOptions.setMergeEnabled(true);
     this.coordinator =
         new MergeCoordinator(
             protocolContext,
             mockProtocolSchedule,
-            mockSorter,
-            new MiningParameters.Builder().coinbase(coinbase).build(),
+            ethContext,
+            transactions,
+            miningParameters,
             backwardSyncContext);
   }
 
@@ -126,6 +177,66 @@ public class MergeCoordinatorTest implements MergeGenesisConfigHelper {
     assertThat(block.getValue().getHeader().getCoinbase()).isEqualTo(suggestedFeeRecipient);
   }
 
+  @Test
+  public void shouldRetryBlockCreationIfStillHaveTime() {
+    when(mergeContext.getFinalized()).thenReturn(Optional.empty());
+    reset(ethContext, ethScheduler);
+    when(ethContext.getScheduler()).thenReturn(ethScheduler);
+    when(ethScheduler.scheduleComputationTask(any()))
+        .thenReturn(CompletableFuture.failedFuture(new StorageException("lock")))
+        .thenAnswer(i -> CompletableFuture.completedFuture(i.getArgument(0, Supplier.class).get()));
+
+    transactions.addLocalTransaction(localTransaction0);
+
+    var payloadId =
+        coordinator.preparePayload(
+            genesisState.getBlock().getHeader(),
+            System.currentTimeMillis() / 1000,
+            Bytes32.ZERO,
+            suggestedFeeRecipient);
+
+    ArgumentCaptor<Block> block = ArgumentCaptor.forClass(Block.class);
+
+    verify(mergeContext, times(2)).putPayloadById(eq(payloadId), block.capture());
+    verify(ethScheduler, times(2)).scheduleComputationTask(any());
+
+    assertThat(block.getAllValues().size()).isEqualTo(2);
+    assertThat(block.getAllValues().get(0).getBody().getTransactions()).hasSize(0);
+    assertThat(block.getAllValues().get(1).getBody().getTransactions()).hasSize(1);
+  }
+
+  @Test
+  public void shouldStopRetryBlockCreationIfTimeExpired() {
+    when(mergeContext.getFinalized()).thenReturn(Optional.empty());
+    doReturn(1L).when(miningParameters).getPosBlockCreationTimeout();
+    reset(ethContext, ethScheduler);
+    when(ethContext.getScheduler()).thenReturn(ethScheduler);
+    when(ethScheduler.scheduleComputationTask(any()))
+        .thenReturn(
+            CompletableFuture.supplyAsync(
+                () -> {
+                  try {
+                    Thread.sleep(1000L);
+                  } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                  }
+                  return CompletableFuture.failedFuture(new RuntimeException());
+                }))
+        .thenAnswer(i -> fail("Must not be called twice"));
+
+    var payloadId =
+        coordinator.preparePayload(
+            genesisState.getBlock().getHeader(),
+            System.currentTimeMillis() / 1000,
+            Bytes32.ZERO,
+            suggestedFeeRecipient);
+
+    ArgumentCaptor<Block> block = ArgumentCaptor.forClass(Block.class);
+
+    verify(mergeContext, times(1)).putPayloadById(eq(payloadId), block.capture());
+    verify(ethScheduler, times(1)).scheduleComputationTask(any());
+  }
+
   @Test
   public void childTimestampExceedsParentsFails() {
     BlockHeader terminalHeader = terminalPowBlock();
@@ -345,7 +456,8 @@ public class MergeCoordinatorTest implements MergeGenesisConfigHelper {
         new MergeCoordinator(
             mockProtocolContext,
             mockProtocolSchedule,
-            mockSorter,
+            ethContext,
+            transactions,
             new MiningParameters.Builder().coinbase(coinbase).build(),
             mock(BackwardSyncContext.class));
 
@@ -405,7 +517,8 @@ public class MergeCoordinatorTest implements MergeGenesisConfigHelper {
         new MergeCoordinator(
             mockProtocolContext,
             mockProtocolSchedule,
-            mockSorter,
+            ethContext,
+            transactions,
             new MiningParameters.Builder().coinbase(coinbase).build(),
             mock(BackwardSyncContext.class));
 
@@ -594,10 +707,33 @@ public class MergeCoordinatorTest implements MergeGenesisConfigHelper {
             new MergeCoordinator(
                 mockProtocolContext,
                 mockProtocolSchedule,
-                mockSorter,
+                ethContext,
+                transactions,
                 new MiningParameters.Builder().coinbase(coinbase).build(),
                 mock(BackwardSyncContext.class)));
 
     return mockCoordinator;
   }
+
+  private Transaction createTransaction(final long transactionNumber) {
+    return new TransactionTestFixture()
+        .value(Wei.of(transactionNumber + 1))
+        .to(Optional.of(Address.ZERO))
+        .gasLimit(53000L)
+        .gasPrice(
+            Wei.fromHexString("0x00000000000000000000000000000000000000000000000000000013b9aca00"))
+        .maxFeePerGas(
+            Optional.of(
+                Wei.fromHexString(
+                    "0x00000000000000000000000000000000000000000000000000000013b9aca00")))
+        .maxPriorityFeePerGas(Optional.of(Wei.of(100_000)))
+        .nonce(transactionNumber)
+        .createTransaction(KEYS1);
+  }
+
+  private static BlockHeader mockBlockHeader() {
+    final BlockHeader blockHeader = mock(BlockHeader.class);
+    when(blockHeader.getBaseFee()).thenReturn(Optional.of(Wei.ONE));
+    return blockHeader;
+  }
 }

consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java

@@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture;
 import org.hyperledger.besu.ethereum.core.Difficulty;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
+import org.hyperledger.besu.ethereum.eth.manager.EthContext;
 import org.hyperledger.besu.ethereum.eth.sync.backwardsync.BackwardSyncContext;
 import org.hyperledger.besu.ethereum.eth.transactions.sorter.AbstractPendingTransactionsSorter;
 import org.hyperledger.besu.ethereum.mainnet.BlockHeaderValidator;
@@ -86,6 +87,7 @@ public class MergeReorgTest implements MergeGenesisConfigHelper {
         new MergeCoordinator(
             protocolContext,
             mockProtocolSchedule,
+            mock(EthContext.class),
             mockSorter,
             new MiningParameters.Builder().coinbase(coinbase).build(),
             mock(BackwardSyncContext.class));

ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java

@@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket;
 import org.hyperledger.besu.evm.account.EvmAccount;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
+import org.hyperledger.besu.plugin.services.exception.StorageException;
 import org.hyperledger.besu.plugin.services.securitymodule.SecurityModuleException;
 
 import java.math.BigInteger;
@@ -206,6 +207,8 @@ public abstract class AbstractBlockCreator implements AsyncBlockCreator {
       throw new IllegalStateException("Failed to create block signature", ex);
     } catch (final CancellationException ex) {
       throw ex;
+    } catch (final StorageException ex) {
+      throw ex;
     } catch (final Exception ex) {
       // TODO(tmm): How are we going to know this has exploded, and thus restart it?
       throw new IllegalStateException(

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java

@@ -34,6 +34,8 @@ public class MiningParameters {
 
   public static final int DEFAULT_MAX_OMMERS_DEPTH = 8;
 
+  public static final long DEFAULT_POS_BLOCK_CREATION_TIMEOUT_MS = Duration.ofSeconds(7).toMillis();
+
   private final Optional<Address> coinbase;
   private final Optional<AtomicLong> targetGasLimit;
   private final Wei minTransactionGasPrice;
@@ -49,6 +51,7 @@ public class MiningParameters {
   private final long remoteSealersTimeToLive;
   private final long powJobTimeToLive;
   private final int maxOmmerDepth;
+  private final long posBlockCreationTimeout;
 
   private MiningParameters(
       final Address coinbase,
@@ -65,7 +68,8 @@ public class MiningParameters {
       final int remoteSealersLimit,
       final long remoteSealersTimeToLive,
       final long powJobTimeToLive,
-      final int maxOmmerDepth) {
+      final int maxOmmerDepth,
+      final long posBlockCreationTimeout) {
     this.coinbase = Optional.ofNullable(coinbase);
     this.targetGasLimit = Optional.ofNullable(targetGasLimit).map(AtomicLong::new);
     this.minTransactionGasPrice = minTransactionGasPrice;
@@ -81,6 +85,7 @@ public class MiningParameters {
     this.remoteSealersTimeToLive = remoteSealersTimeToLive;
     this.powJobTimeToLive = powJobTimeToLive;
     this.maxOmmerDepth = maxOmmerDepth;
+    this.posBlockCreationTimeout = posBlockCreationTimeout;
   }
 
   public Optional<Address> getCoinbase() {
@@ -143,6 +148,10 @@ public class MiningParameters {
     return maxOmmerDepth;
   }
 
+  public long getPosBlockCreationTimeout() {
+    return posBlockCreationTimeout;
+  }
+
   @Override
   public boolean equals(final Object o) {
     if (this == o) return true;
@@ -160,7 +169,8 @@ public class MiningParameters {
         && Objects.equals(minBlockOccupancyRatio, that.minBlockOccupancyRatio)
         && remoteSealersTimeToLive == that.remoteSealersTimeToLive
         && remoteSealersLimit == that.remoteSealersLimit
-        && powJobTimeToLive == that.powJobTimeToLive;
+        && powJobTimeToLive == that.powJobTimeToLive
+        && posBlockCreationTimeout == that.posBlockCreationTimeout;
   }
 
   @Override
@@ -178,7 +188,8 @@ public class MiningParameters {
         minBlockOccupancyRatio,
         remoteSealersLimit,
         remoteSealersTimeToLive,
-        powJobTimeToLive);
+        powJobTimeToLive,
+        posBlockCreationTimeout);
   }
 
   @Override
@@ -214,6 +225,8 @@ public class MiningParameters {
         + remoteSealersTimeToLive
         + ", powJobTimeToLive="
         + powJobTimeToLive
+        + ", posBlockCreationTimeout="
+        + posBlockCreationTimeout
         + '}';
   }
 
@@ -235,6 +248,8 @@ public class MiningParameters {
     private long powJobTimeToLive = DEFAULT_POW_JOB_TTL;
     private int maxOmmerDepth = DEFAULT_MAX_OMMERS_DEPTH;
 
+    private long posBlockCreationTimeout = DEFAULT_POS_BLOCK_CREATION_TIMEOUT_MS;
+
     public Builder() {
       // zero arg
     }
@@ -258,6 +273,7 @@ public class MiningParameters {
       this.remoteSealersTimeToLive = existing.getRemoteSealersTimeToLive();
       this.powJobTimeToLive = existing.getPowJobTimeToLive();
       this.maxOmmerDepth = existing.getMaxOmmerDepth();
+      this.posBlockCreationTimeout = existing.getPosBlockCreationTimeout();
     }
 
     public Builder coinbase(final Address address) {
@@ -335,6 +351,11 @@ public class MiningParameters {
       return this;
     }
 
+    public Builder posBlockCreationTimeout(final long posBlockCreationTimeoutMillis) {
+      this.posBlockCreationTimeout = posBlockCreationTimeoutMillis;
+      return this;
+    }
+
     public MiningParameters build() {
       return new MiningParameters(
           coinbase,
@@ -351,7 +372,8 @@ public class MiningParameters {
           remoteSealersLimit,
           remoteSealersTimeToLive,
           powJobTimeToLive,
-          maxOmmerDepth);
+          maxOmmerDepth,
+          posBlockCreationTimeout);
     }
   }
 }

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java

@@ -82,8 +82,9 @@ public class EthScheduler {
             metricsSystem),
         MonitoredExecutors.newCachedThreadPool(
             EthScheduler.class.getSimpleName() + "-Services", metricsSystem),
-        MonitoredExecutors.newFixedThreadPool(
+        MonitoredExecutors.newBoundedThreadPool(
             EthScheduler.class.getSimpleName() + "-Computation",
+            1,
             computationWorkerCount,
             metricsSystem));
   }

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/MonitoredExecutors.java

@@ -39,7 +39,7 @@ public class MonitoredExecutors {
 
   public static ExecutorService newFixedThreadPool(
       final String name, final int workerCount, final MetricsSystem metricsSystem) {
-    return newFixedThreadPool(name, workerCount, new LinkedBlockingQueue<>(), metricsSystem);
+    return newFixedThreadPool(name, 0, workerCount, new LinkedBlockingQueue<>(), metricsSystem);
   }
 
   public static ExecutorService newBoundedThreadPool(
@@ -47,16 +47,27 @@ public class MonitoredExecutors {
       final int workerCount,
       final int queueSize,
       final MetricsSystem metricsSystem) {
+    return newBoundedThreadPool(name, 0, workerCount, queueSize, metricsSystem);
+  }
+
+  public static ExecutorService newBoundedThreadPool(
+      final String name,
+      final int minWorkerCount,
+      final int maxWorkerCount,
+      final int queueSize,
+      final MetricsSystem metricsSystem) {
     return newFixedThreadPool(
         name,
-        workerCount,
+        minWorkerCount,
+        maxWorkerCount,
         new BoundedQueue(queueSize, toMetricName(name), metricsSystem),
         metricsSystem);
   }
 
-  public static ExecutorService newFixedThreadPool(
+  private static ExecutorService newFixedThreadPool(
       final String name,
-      final int workerCount,
+      final int minWorkerCount,
+      final int maxWorkerCount,
       final BlockingQueue<Runnable> workingQueue,
       final MetricsSystem metricsSystem) {
     return newMonitoredExecutor(
@@ -64,8 +75,8 @@ public class MonitoredExecutors {
         metricsSystem,
         (rejectedExecutionHandler, threadFactory) ->
             new ThreadPoolExecutor(
-                workerCount,
-                workerCount,
+                minWorkerCount,
+                maxWorkerCount,
                 0L,
                 TimeUnit.MILLISECONDS,
                 workingQueue,