Merge branch 'main' into eip-4844-interop

# Conflicts:
#	CHANGELOG.md
#	ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/BlockResultFactory.java
#	ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/ExecutionEngineJsonRpcMethods.java
pull/5091/head
Fabio Di Fabio 2 years ago
commit 8f2f94b6ea
  1. 4
      .github/pull_request_template.md
  2. 26
      CHANGELOG.md
  3. 5
      besu/src/main/java/org/hyperledger/besu/chainimport/JsonBlockImporter.java
  4. 2
      besu/src/main/java/org/hyperledger/besu/chainimport/RlpBlockImporter.java
  5. 3
      besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
  6. 16
      besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java
  7. 3
      besu/src/test/java/org/hyperledger/besu/RunnerTest.java
  8. 13
      besu/src/test/java/org/hyperledger/besu/controller/TransitionControllerBuilderTest.java
  9. 3
      besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java
  10. 13
      build.gradle
  11. 2
      consensus/clique/src/test/java/org/hyperledger/besu/consensus/clique/CliqueProtocolScheduleTest.java
  12. 7
      consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeContext.java
  13. 17
      consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PostMergeContext.java
  14. 5
      consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionContext.java
  15. 6
      consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionProtocolSchedule.java
  16. 10
      consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionUtils.java
  17. 13
      consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java
  18. 41
      consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/TransitionProtocolScheduleTest.java
  19. 2
      consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinatorTest.java
  20. 3
      ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java
  21. 3
      ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthGetBlockByNumberLatestDesyncIntegrationTest.java
  22. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java
  23. 10
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/DebugReplayBlock.java
  24. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetBadBlocks.java
  25. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugResyncWorldstate.java
  26. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBadBlockToFile.java
  27. 7
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthFeeHistory.java
  28. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetMinerDataByBlockHash.java
  29. 1
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdated.java
  30. 1
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineNewPayload.java
  31. 85
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadBodiesByHashV1.java
  32. 113
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadBodiesByRangeV1.java
  33. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/response/JsonRpcError.java
  34. 12
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/BlockResultFactory.java
  35. 79
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/EngineGetPayloadBodiesResultV1.java
  36. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/tracing/flat/FlatTraceGenerator.java
  37. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/tracing/flat/RewardTraceGenerator.java
  38. 6
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/ExecutionEngineJsonRpcMethods.java
  39. 2
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/PrivacyApiGroupJsonRpcMethods.java
  40. 9
      ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java
  41. 3
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/AbstractEthGraphQLHttpServiceTest.java
  42. 8
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetBadBlockTest.java
  43. 2
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBadBlockToFileTest.java
  44. 22
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthFeeHistoryTest.java
  45. 2
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetMinerDataByBlockHashTest.java
  46. 2
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetMinerDataByBlockNumberTest.java
  47. 267
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadBodiesByHashV1Test.java
  48. 352
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadBodiesByRangeV1Test.java
  49. 2
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/tracing/flat/RewardTraceGeneratorTest.java
  50. 11
      ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueriesTest.java
  51. 2
      ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_feeHistory_fieldNamesWithReward.json
  52. 2
      ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_feeHistory_fieldNamesWithoutReward.json
  53. 2
      ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockMiner.java
  54. 11
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java
  55. 12
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java
  56. 2
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java
  57. 24
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java
  58. 14
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldStateKeyValueStorage.java
  59. 27
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java
  60. 33
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java
  61. 37
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateKeyValueStorage.java
  62. 74
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java
  63. 87
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java
  64. 52
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java
  65. 2
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldView.java
  66. 5
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/SnapshotTrieLogManager.java
  67. 5
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/Blockchain.java
  68. 5
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Synchronizer.java
  69. 7
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java
  70. 4
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java
  71. 2
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionSimulator.java
  72. 2
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigration.java
  73. 5
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java
  74. 2
      ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java
  75. 3
      ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockchainSetupUtil.java
  76. 72
      ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java
  77. 3
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/BlockPropagationManager.java
  78. 89
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java
  79. 1
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java
  80. 6
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java
  81. 9
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java
  82. 6
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardsSyncAlgorithm.java
  83. 4
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/FinalBlockConfirmation.java
  84. 4
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStep.java
  85. 32
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ProcessKnownAncestorsStep.java
  86. 17
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java
  87. 2
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStep.java
  88. 1
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java
  89. 12
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java
  90. 8
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java
  91. 2
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullImportBlockStep.java
  92. 3
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/range/RangeHeadersValidationStep.java
  93. 15
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java
  94. 16
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapPersistedContext.java
  95. 1
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java
  96. 11
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java
  97. 26
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/state/SyncState.java
  98. 4
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/DownloadHeaderSequenceTask.java
  99. 3
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/PersistBlockTask.java
  100. 172
      ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStatePeerTrieNodeFinder.java
  101. Some files were not shown because too many files have changed in this diff Show More

@ -12,6 +12,10 @@
- [ ] I thought about documentation and added the `doc-change-required` label to this PR if
[updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).
## Acceptance Tests (Non Mainnet)
- [ ] I have considered running `./gradlew acceptanceTestNonMainnet` locally if my PR affects non-mainnet modules.
## Changelog
- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).

@ -1,40 +1,36 @@
# Changelog
## 23.1.1
## 23.1.0-RC2
### Breaking Changes
- Add a new CLI option to limit the number of requests in a single RPC batch request. Default=1 [#4965](https://github.com/hyperledger/besu/pull/4965)
- Changed JsonRpc http service to return the error -32602 (Invalid params) with a 200 http status code
- Change JsonRpc http service to return the error -32602 (Invalid params) with a 200 http status code
- Besu requires minimum Java 17 and up to build and run [#3320](https://github.com/hyperledger/besu/issues/3320)
- PKCS11 with nss module (PKCS11 based HSM can be used in DevP2P TLS and QBFT PKI) does not work with RSA keys
in Java 17. SoftHSM is tested manually and working. (Other PKCS11 HSM are not tested). The relevant unit and acceptance
tests are updated to use EC private keys instead of RSA keys.
- Change eth_feeHistory parameter `blockCount` to accept hexadecimal string (was accepting plain integer) [#5047](https://github.com/hyperledger/besu/pull/5047)
### Additions and Improvements
- Support for new DATAHASH opcode as part of EIP-4844 [#4823](https://github.com/hyperledger/besu/issues/4823)
- Send only hash announcement for blob transaction type [#4940](https://github.com/hyperledger/besu/pull/4940)
- Add `excess_data_gas` field to block header [#4958](https://github.com/hyperledger/besu/pull/4958)
- Add `max_fee_per_data_gas` field to transaction [#4970](https://github.com/hyperledger/besu/pull/4970)
- Added option to evm CLI tool to allow code execution at specific forks [#4913](https://github.com/hyperledger/besu/pull/4913)
- Improve get account performance by using the world state updater cache [#4897](https://github.com/hyperledger/besu/pull/4897)
- Add new KZG precompile and option to override the trusted setup being used [#4822](https://github.com/hyperledger/besu/issues/4822)
- Add implementation for eth_createAccessList RPC method [#4942](https://github.com/hyperledger/besu/pull/4942)
- Add implementation for engine_exchangeCapabilities [#4997](https://github.com/hyperledger/besu/pull/4997)
- Add implementation for engine_getPayloadBodiesByRangeV1 and engine_getPayloadBodiesByHashV1 [#4980](https://github.com/hyperledger/besu/pull/4980)
- Updated reference tests to v11.3 [#4996](https://github.com/hyperledger/besu/pull/4996)
- Add DebugGetRawBlock and DebugGetRawHeader RPC methods [#5011](https://github.com/hyperledger/besu/pull/5011)
- Besu requires minimum Java 17 and up to build and run [#3320](https://github.com/hyperledger/besu/issues/3320)
- Add `max_fee_per_data_gas` field to transaction [#4970](https://github.com/hyperledger/besu/pull/4970)
- Add worldstate auto-heal mechanism [#5059](https://github.com/hyperledger/besu/pull/5059)
- Support for EIP-4895 - Withdrawals for Shanghai fork
- Gas accounting for EIP-4844 [#4992](https://github.com/hyperledger/besu/pull/4992)
### Bug Fixes
## 23.1.0
### Breaking Changes
### Additions and Improvements
- Support for new DATAHASH opcode as part of EIP-4844 [#4823](https://github.com/hyperledger/besu/issues/4823)
- Send only hash announcement for blob transaction type [#4940](https://github.com/hyperledger/besu/pull/4940)
- Add `excess_data_gas` field to block header [#4958](https://github.com/hyperledger/besu/pull/4958)
### Bug Fixes
- Mitigation fix for stale bonsai code storage leading to log rolling issues on contract recreates [#4906](https://github.com/hyperledger/besu/pull/4906)
- Ensure latest cached layered worldstate is subscribed to storage, fix problem with RPC calls using 'latest' [#5039](https://github.com/hyperledger/besu/pull/5039)
## 23.1.0-RC1

@ -153,7 +153,10 @@ public class JsonBlockImporter {
private void importBlock(final Block block) {
final BlockImporter importer =
controller.getProtocolSchedule().getByBlockHeader(block.getHeader()).getBlockImporter();
controller
.getProtocolSchedule()
.getByBlockNumber(block.getHeader().getNumber())
.getBlockImporter();
final BlockImportResult importResult =
importer.importBlock(controller.getProtocolContext(), block, HeaderValidationMode.NONE);

@ -127,7 +127,7 @@ public class RlpBlockImporter implements Closeable {
if (previousHeader == null) {
previousHeader = lookupPreviousHeader(blockchain, header);
}
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(header);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(blockNumber);
final BlockHeader lastHeader = previousHeader;
final CompletableFuture<Void> validationFuture =

@ -632,6 +632,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
ethProtocolManager,
pivotBlockSelector);
protocolContext.setSynchronizer(Optional.of(synchronizer));
final MiningCoordinator miningCoordinator =
createMiningCoordinator(
protocolSchedule,
@ -713,7 +715,6 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
clock,
metricsSystem,
getFullSyncTerminationCondition(protocolContext.getBlockchain()),
ethProtocolManager,
pivotBlockSelector);
return toUse;

@ -14,6 +14,7 @@
*/
package org.hyperledger.besu.controller;
import org.hyperledger.besu.config.GenesisConfigOptions;
import org.hyperledger.besu.consensus.merge.MergeContext;
import org.hyperledger.besu.consensus.merge.MergeProtocolSchedule;
import org.hyperledger.besu.consensus.merge.PostMergeContext;
@ -186,19 +187,24 @@ public class MergeBesuControllerBuilder extends BesuControllerBuilder {
final WorldStateArchive worldStateArchive,
final ProtocolSchedule protocolSchedule) {
OptionalLong terminalBlockNumber = configOptionsSupplier.get().getTerminalBlockNumber();
Optional<Hash> terminalBlockHash = configOptionsSupplier.get().getTerminalBlockHash();
final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get();
final OptionalLong terminalBlockNumber = genesisConfigOptions.getTerminalBlockNumber();
final Optional<Hash> terminalBlockHash = genesisConfigOptions.getTerminalBlockHash();
final boolean isPostMergeAtGenesis =
genesisConfigOptions.getTerminalTotalDifficulty().isPresent()
&& genesisConfigOptions.getTerminalTotalDifficulty().get().isZero()
&& blockchain.getGenesisBlockHeader().getDifficulty().isZero();
final MergeContext mergeContext =
PostMergeContext.get()
.setSyncState(syncState.get())
.setTerminalTotalDifficulty(
configOptionsSupplier
.get()
genesisConfigOptions
.getTerminalTotalDifficulty()
.map(Difficulty::of)
.orElse(Difficulty.ZERO))
.setCheckpointPostMergeSync(syncConfig.isCheckpointPostMergeEnabled());
.setCheckpointPostMergeSync(syncConfig.isCheckpointPostMergeEnabled())
.setPostMergeAtGenesis(isPostMergeAtGenesis);
blockchain
.getFinalized()

@ -427,7 +427,8 @@ public final class RunnerTest {
for (int i = 1; i < count + 1; ++i) {
final Block block = blocks.get(i);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
final ProtocolSpec protocolSpec =
protocolSchedule.getByBlockNumber(block.getHeader().getNumber());
final BlockImporter blockImporter = protocolSpec.getBlockImporter();
final BlockImportResult result =
blockImporter.importBlock(protocolContext, block, HeaderValidationMode.FULL);

@ -16,6 +16,7 @@ package org.hyperledger.besu.controller;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@ -130,7 +131,7 @@ public class TransitionControllerBuilderTest {
var preMergeProtocolSpec = mock(ProtocolSpec.class);
when(mergeContext.isPostMerge()).thenReturn(Boolean.FALSE);
when(mergeContext.getFinalized()).thenReturn(Optional.empty());
when(preMergeProtocolSchedule.getByBlockHeader(any())).thenReturn(preMergeProtocolSpec);
when(preMergeProtocolSchedule.getByBlockNumber(anyLong())).thenReturn(preMergeProtocolSpec);
assertThat(transitionProtocolSchedule.getByBlockHeader(mockBlock))
.isEqualTo(preMergeProtocolSpec);
}
@ -140,7 +141,7 @@ public class TransitionControllerBuilderTest {
var mockBlock = new BlockHeaderTestFixture().buildHeader();
var postMergeProtocolSpec = mock(ProtocolSpec.class);
when(mergeContext.getFinalized()).thenReturn(Optional.of(mockBlock));
when(postMergeProtocolSchedule.getByBlockHeader(any())).thenReturn(postMergeProtocolSpec);
when(postMergeProtocolSchedule.getByBlockNumber(anyLong())).thenReturn(postMergeProtocolSpec);
assertThat(transitionProtocolSchedule.getByBlockHeader(mockBlock))
.isEqualTo(postMergeProtocolSpec);
}
@ -163,7 +164,7 @@ public class TransitionControllerBuilderTest {
when(mockBlockchain.getTotalDifficultyByHash(any()))
.thenReturn(Optional.of(Difficulty.of(1335L)));
when(preMergeProtocolSchedule.getByBlockHeader(any())).thenReturn(preMergeProtocolSpec);
when(preMergeProtocolSchedule.getByBlockNumber(anyLong())).thenReturn(preMergeProtocolSpec);
assertThat(transitionProtocolSchedule.getByBlockHeader(mockBlock))
.isEqualTo(preMergeProtocolSpec);
}
@ -186,7 +187,7 @@ public class TransitionControllerBuilderTest {
when(mockBlockchain.getTotalDifficultyByHash(any()))
.thenReturn(Optional.of(Difficulty.of(1337L)));
when(postMergeProtocolSchedule.getByBlockHeader(any())).thenReturn(postMergeProtocolSpec);
when(postMergeProtocolSchedule.getByBlockNumber(anyLong())).thenReturn(postMergeProtocolSpec);
assertThat(transitionProtocolSchedule.getByBlockHeader(mockBlock))
.isEqualTo(postMergeProtocolSpec);
}
@ -212,7 +213,7 @@ public class TransitionControllerBuilderTest {
var preMergeProtocolSpec = mock(ProtocolSpec.class);
when(preMergeProtocolSpec.getBlockHeaderValidator()).thenReturn(validator);
when(preMergeProtocolSchedule.getByBlockHeader(any())).thenReturn(preMergeProtocolSpec);
when(preMergeProtocolSchedule.getByBlockNumber(anyLong())).thenReturn(preMergeProtocolSpec);
var mockParentBlock =
new BlockHeaderTestFixture()
@ -234,7 +235,7 @@ public class TransitionControllerBuilderTest {
var mergeFriendlyValidation =
transitionProtocolSchedule
.getPreMergeSchedule()
.getByBlockHeader(mockBlock)
.getByBlockNumber(1L)
.getBlockHeaderValidator()
.validateHeader(
mockBlock, mockParentBlock, protocolContext, HeaderValidationMode.DETACHED_ONLY);

@ -17,6 +17,7 @@ package org.hyperledger.besu.services;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -124,7 +125,7 @@ public class BesuEventsImplTest {
when(mockEthPeers.streamAvailablePeers()).thenAnswer(z -> Stream.empty());
when(mockProtocolContext.getBlockchain()).thenReturn(blockchain);
when(mockProtocolContext.getWorldStateArchive()).thenReturn(mockWorldStateArchive);
when(mockProtocolSchedule.getByBlockHeader(any())).thenReturn(mockProtocolSpec);
when(mockProtocolSchedule.getByBlockNumber(anyLong())).thenReturn(mockProtocolSpec);
when(mockProtocolSpec.getTransactionValidator()).thenReturn(mockTransactionValidator);
when(mockProtocolSpec.getFeeMarket()).thenReturn(FeeMarket.london(0L));
when(mockTransactionValidator.validate(any(), any(Optional.class), any()))

@ -715,10 +715,7 @@ task dockerUpload {
additionalTags.add('develop')
}
def isInterimBuild = (dockerBuildVersion ==~ /.*-SNAPSHOT/) || (dockerBuildVersion ==~ /.*-alpha/)
|| (dockerBuildVersion ==~ /.*-beta/) || (dockerBuildVersion ==~ /.*-RC.*/)
if (!isInterimBuild) {
if (!isInterimBuild(dockerBuildVersion)) {
additionalTags.add(dockerBuildVersion.split(/\./)[0..1].join('.'))
}
@ -778,7 +775,7 @@ task manifestDocker {
tags.add("${dockerImageName}:develop")
}
if (!(dockerBuildVersion ==~ /.*-SNAPSHOT/)) {
if (!isInterimBuild(dockerBuildVersion)) {
tags.add("${dockerImageName}:" + dockerBuildVersion.split(/\./)[0..1].join('.'))
}
@ -929,6 +926,12 @@ def getCheckedOutGitCommitHash(length = 8) {
}
}
// Takes the version and if it contains SNAPSHOT, alpha, beta or RC in version then return true indicating an interim build
def isInterimBuild(dockerBuildVersion) {
return (dockerBuildVersion ==~ /.*-SNAPSHOT/) || (dockerBuildVersion ==~ /.*-alpha/)
|| (dockerBuildVersion ==~ /.*-beta/) || (dockerBuildVersion ==~ /.*-RC.*/)
}
tasks.register("verifyDistributions") {
dependsOn distTar
dependsOn distZip

@ -176,7 +176,7 @@ public class CliqueProtocolScheduleTest {
final BlockHeader parentBlockHeader) {
return schedule
.getByBlockHeader(blockHeader)
.getByBlockNumber(blockHeader.getNumber())
.getBlockHeaderValidator()
.validateHeader(blockHeader, parentBlockHeader, null, HeaderValidationMode.LIGHT);
}

@ -192,4 +192,11 @@ public interface MergeContext extends ConsensusContext {
* @return the boolean
*/
boolean isCheckpointPostMergeSync();
/**
* Is configured for a post-merge from genesis.
*
* @return the boolean
*/
boolean isPostMergeAtGenesis();
}

@ -70,6 +70,7 @@ public class PostMergeContext implements MergeContext {
private final AtomicReference<Optional<BlockHeader>> terminalPoWBlock =
new AtomicReference<>(Optional.empty());
private boolean isCheckpointPostMergeSync;
private boolean isPostMergeAtGenesis;
// TODO: cleanup - isChainPruningEnabled will not be required after
// https://github.com/hyperledger/besu/pull/4703 is merged.
@ -329,4 +330,20 @@ public class PostMergeContext implements MergeContext {
public boolean isCheckpointPostMergeSync() {
return this.isCheckpointPostMergeSync;
}
@Override
public boolean isPostMergeAtGenesis() {
return this.isPostMergeAtGenesis;
}
/**
* Sets whether it is post merge at genesis
*
* @param isPostMergeAtGenesis the is post merge at genesis state
* @return the post merge context
*/
public PostMergeContext setPostMergeAtGenesis(final boolean isPostMergeAtGenesis) {
this.isPostMergeAtGenesis = isPostMergeAtGenesis;
return this;
}
}

@ -154,4 +154,9 @@ public class TransitionContext implements MergeContext {
public boolean isCheckpointPostMergeSync() {
return false;
}
@Override
public boolean isPostMergeAtGenesis() {
return postMergeContext.isPostMergeAtGenesis();
}
}

@ -102,7 +102,7 @@ public class TransitionProtocolSchedule implements ProtocolSchedule {
LOG,
"for {} returning a pre-merge schedule because we are not post-merge",
blockHeader::toLogString);
return getPreMergeSchedule().getByBlockHeader(blockHeader);
return getPreMergeSchedule().getByBlockNumber(blockHeader.getNumber());
}
// otherwise check to see if this block represents a re-org TTD block:
@ -128,12 +128,12 @@ public class TransitionProtocolSchedule implements ProtocolSchedule {
LOG,
"returning a pre-merge schedule because block {} is pre-merge or TTD",
blockHeader::toLogString);
return getPreMergeSchedule().getByBlockHeader(blockHeader);
return getPreMergeSchedule().getByBlockNumber(blockHeader.getNumber());
}
}
// else return post-merge schedule
debugLambda(LOG, " for {} returning a post-merge schedule", blockHeader::toLogString);
return getPostMergeSchedule().getByBlockHeader(blockHeader);
return getPostMergeSchedule().getByBlockNumber(blockHeader.getNumber());
}
/**

@ -116,6 +116,13 @@ public class TransitionUtils<SwitchingObject> {
// if we cannot find difficulty or are merge-at-genesis
.orElse(Difficulty.ZERO);
final MergeContext consensusContext = context.getConsensusContext(MergeContext.class);
// Genesis is configured for post-merge we will never have a terminal pow block
if (consensusContext.isPostMergeAtGenesis()) {
return false;
}
if (currentChainTotalDifficulty.isZero()) {
warnLambda(
LOG,
@ -123,8 +130,7 @@ public class TransitionUtils<SwitchingObject> {
header::toLogString,
header::getParentHash);
}
Difficulty configuredTotalTerminalDifficulty =
context.getConsensusContext(MergeContext.class).getTerminalTotalDifficulty();
Difficulty configuredTotalTerminalDifficulty = consensusContext.getTerminalTotalDifficulty();
if (currentChainTotalDifficulty
.add(headerDifficulty)

@ -681,9 +681,9 @@ public class MergeCoordinator implements MergeMiningCoordinator, BadChainListene
@Override
public Optional<Hash> getLatestValidAncestor(final Hash blockHash) {
final var chain = protocolContext.getBlockchain();
final var chainHeadHeader = chain.getChainHeadHeader();
final var chainHeadNum = chain.getChainHeadBlockNumber();
return findValidAncestor(
chain, blockHash, protocolSchedule.getByBlockHeader(chainHeadHeader).getBadBlocksManager());
chain, blockHash, protocolSchedule.getByBlockNumber(chainHeadNum).getBadBlocksManager());
}
@Override
@ -692,7 +692,8 @@ public class MergeCoordinator implements MergeMiningCoordinator, BadChainListene
final var self = chain.getBlockHeader(blockHeader.getHash());
if (self.isEmpty()) {
final var badBlocks = protocolSchedule.getByBlockHeader(blockHeader).getBadBlocksManager();
final var badBlocks =
protocolSchedule.getByBlockNumber(blockHeader.getNumber()).getBadBlocksManager();
return findValidAncestor(chain, blockHeader.getParentHash(), badBlocks);
}
return self.map(BlockHeader::getHash);
@ -823,7 +824,7 @@ public class MergeCoordinator implements MergeMiningCoordinator, BadChainListene
@Override
public void addBadBlock(final Block block, final Optional<Throwable> maybeCause) {
protocolSchedule
.getByBlockHeader(protocolContext.getBlockchain().getChainHeadHeader())
.getByBlockNumber(protocolContext.getBlockchain().getChainHeadBlockNumber())
.getBadBlocksManager()
.addBadBlock(block, maybeCause);
}
@ -838,7 +839,7 @@ public class MergeCoordinator implements MergeMiningCoordinator, BadChainListene
private BadBlockManager getBadBlockManager() {
final BadBlockManager badBlocksManager =
protocolSchedule
.getByBlockHeader(protocolContext.getBlockchain().getChainHeadHeader())
.getByBlockNumber(protocolContext.getBlockchain().getChainHeadBlockNumber())
.getBadBlocksManager();
return badBlocksManager;
}
@ -846,7 +847,7 @@ public class MergeCoordinator implements MergeMiningCoordinator, BadChainListene
@Override
public Optional<Hash> getLatestValidHashOfBadBlock(Hash blockHash) {
return protocolSchedule
.getByBlockHeader(protocolContext.getBlockchain().getChainHeadHeader())
.getByBlockNumber(protocolContext.getBlockchain().getChainHeadBlockNumber())
.getBadBlocksManager()
.getLatestValidHash(blockHash);
}

@ -15,7 +15,6 @@
package org.hyperledger.besu.consensus.merge;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
@ -72,6 +71,7 @@ public class TransitionProtocolScheduleTest {
@Test
public void returnPostMergeIfFinalizedExists() {
when(mergeContext.getFinalized()).thenReturn(Optional.of(mock(BlockHeader.class)));
when(blockHeader.getNumber()).thenReturn(BLOCK_NUMBER);
transitionProtocolSchedule.getByBlockHeader(blockHeader);
@ -83,6 +83,8 @@ public class TransitionProtocolScheduleTest {
when(mergeContext.getFinalized()).thenReturn(Optional.empty());
when(mergeContext.isPostMerge()).thenReturn(false);
when(blockHeader.getNumber()).thenReturn(BLOCK_NUMBER);
transitionProtocolSchedule.getByBlockHeader(blockHeader);
verifyPreMergeProtocolScheduleReturned();
@ -96,6 +98,7 @@ public class TransitionProtocolScheduleTest {
final Hash parentHash = Hash.fromHexStringLenient("0xabc123");
when(blockHeader.getNumber()).thenReturn(BLOCK_NUMBER);
when(blockHeader.getParentHash()).thenReturn(parentHash);
when(blockHeader.getDifficulty()).thenReturn(Difficulty.of(10L));
when(blockchain.getTotalDifficultyByHash(parentHash))
@ -114,6 +117,7 @@ public class TransitionProtocolScheduleTest {
final Hash parentHash = Hash.fromHexStringLenient("0xabc123");
when(blockHeader.getNumber()).thenReturn(BLOCK_NUMBER);
when(blockHeader.getParentHash()).thenReturn(parentHash);
when(blockHeader.getDifficulty()).thenReturn(Difficulty.of(2L));
when(blockchain.getTotalDifficultyByHash(parentHash))
@ -155,45 +159,52 @@ public class TransitionProtocolScheduleTest {
}
@Test
public void getByBlockHeader_delegatesToPreMergeScheduleWhenBlockNotFound() {
public void getByBlockNumber_returnsTimestampScheduleIfPresent() {
when(blockchain.getBlockHeader(BLOCK_NUMBER)).thenReturn(Optional.of(blockHeader));
when(timestampSchedule.getByBlockHeader(blockHeader)).thenReturn(mock(ProtocolSpec.class));
assertThat(transitionProtocolSchedule.getByBlockNumber(BLOCK_NUMBER)).isNotNull();
verify(timestampSchedule).getByBlockHeader(blockHeader);
verifyNoMergeScheduleInteractions();
}
@Test
public void getByBlockNumber_delegatesToPreMergeScheduleWhenBlockNotFound() {
when(blockchain.getBlockHeader(BLOCK_NUMBER)).thenReturn(Optional.empty());
when(mergeContext.isPostMerge()).thenReturn(false);
transitionProtocolSchedule.getByBlockHeader(blockHeader);
transitionProtocolSchedule.getByBlockNumber(BLOCK_NUMBER);
verifyPreMergeProtocolScheduleReturned();
}
@Test
public void getByBlockHeader_delegatesToPostMergeScheduleWhenBlockNotFound() {
when(blockHeader.getNumber()).thenReturn(BLOCK_NUMBER);
when(blockHeader.getDifficulty()).thenReturn(Difficulty.ZERO);
when(blockchain.getTotalDifficultyByHash(any())).thenReturn(Optional.of(TTD));
public void getByBlockNumber_delegatesToPostMergeScheduleWhenBlockNotFound() {
when(blockchain.getBlockHeader(BLOCK_NUMBER)).thenReturn(Optional.empty());
when(mergeContext.isPostMerge()).thenReturn(true);
transitionProtocolSchedule.getByBlockHeader(blockHeader);
transitionProtocolSchedule.getByBlockNumber(BLOCK_NUMBER);
verifyPostMergeProtocolScheduleReturned();
}
@Test
public void getByBlockHeader_delegatesToPostMergeScheduleWhenTimestampScheduleDoesNotExist() {
public void getByBlockNumber_delegatesToPostMergeScheduleWhenTimestampScheduleDoesNotExist() {
when(mergeContext.isPostMerge()).thenReturn(true);
when(blockHeader.getDifficulty()).thenReturn(Difficulty.ZERO);
when(blockHeader.getNumber()).thenReturn(BLOCK_NUMBER);
when(blockchain.getTotalDifficultyByHash(any())).thenReturn(Optional.of(TTD));
transitionProtocolSchedule.getByBlockHeader(blockHeader);
transitionProtocolSchedule.getByBlockNumber(BLOCK_NUMBER);
verifyPostMergeProtocolScheduleReturned();
}
private void verifyPreMergeProtocolScheduleReturned() {
verify(preMergeProtocolSchedule).getByBlockHeader(any());
verify(preMergeProtocolSchedule).getByBlockNumber(BLOCK_NUMBER);
verifyNoInteractions(postMergeProtocolSchedule);
}
private void verifyPostMergeProtocolScheduleReturned() {
verify(postMergeProtocolSchedule).getByBlockHeader(any());
verify(postMergeProtocolSchedule).getByBlockNumber(BLOCK_NUMBER);
verifyNoInteractions(preMergeProtocolSchedule);
}

@ -174,7 +174,7 @@ public class MergeCoordinatorTest implements MergeGenesisConfigHelper {
return spec;
})
.when(protocolSchedule)
.getByBlockHeader(any(BlockHeader.class));
.getByBlockNumber(anyLong());
protocolContext = new ProtocolContext(blockchain, worldStateArchive, mergeContext);
var mutable = worldStateArchive.getMutable();

@ -82,7 +82,8 @@ public class JsonRpcTestMethodsFactory {
this.synchronizer = mock(Synchronizer.class);
for (final Block block : importer.getBlocks()) {
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
final ProtocolSpec protocolSpec =
protocolSchedule.getByBlockNumber(block.getHeader().getNumber());
final BlockImporter blockImporter = protocolSpec.getBlockImporter();
blockImporter.importBlock(context, block, HeaderValidationMode.FULL);
}

@ -71,7 +71,8 @@ public class EthGetBlockByNumberLatestDesyncIntegrationTest {
for (final Block block : importer.getBlocks()) {
final ProtocolSchedule protocolSchedule = importer.getProtocolSchedule();
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
final ProtocolSpec protocolSpec =
protocolSchedule.getByBlockNumber(block.getHeader().getNumber());
final BlockImporter blockImporter = protocolSpec.getBlockImporter();
blockImporter.importBlock(context, block, HeaderValidationMode.FULL);
}

@ -58,6 +58,8 @@ public enum RpcMethod {
ENGINE_FORKCHOICE_UPDATED_V1("engine_forkchoiceUpdatedV1"),
ENGINE_FORKCHOICE_UPDATED_V2("engine_forkchoiceUpdatedV2"),
ENGINE_EXCHANGE_TRANSITION_CONFIGURATION("engine_exchangeTransitionConfigurationV1"),
ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1("engine_getPayloadBodiesByHashV1"),
ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1("engine_getPayloadBodiesByRangeV1"),
ENGINE_EXCHANGE_CAPABILITIES("engine_exchangeCapabilities"),
ENGINE_GET_BLOBS_BUNDLE_V1("engine_getBlobsBundleV1"),
GOQUORUM_ETH_GET_QUORUM_PAYLOAD("eth_getQuorumPayload"),

@ -69,18 +69,20 @@ public class DebugReplayBlock extends AbstractBlockParameterMethod {
return new JsonRpcErrorResponse(request.getRequest().getId(), UNKNOWN_BLOCK);
}
final Block block = maybeBlock.get();
// rewind to the block before the one we want to replay
protocolContext.getBlockchain().rewindToBlock(blockNumber - 1);
try {
// replay block and persist it
protocolSchedule
.getByBlockHeader(block.getHeader())
.getByBlockNumber(blockNumber)
.getBlockValidator()
.validateAndProcessBlock(
protocolContext, block, HeaderValidationMode.FULL, HeaderValidationMode.NONE, true);
protocolContext,
maybeBlock.get(),
HeaderValidationMode.FULL,
HeaderValidationMode.NONE,
true);
} catch (Exception e) {
LOG.error(e.getMessage());
return new JsonRpcErrorResponse(request.getRequest().getId(), INTERNAL_ERROR);

@ -50,7 +50,7 @@ public class DebugGetBadBlocks implements JsonRpcMethod {
public JsonRpcResponse response(final JsonRpcRequestContext requestContext) {
final List<BadBlockResult> response =
protocolSchedule
.getByBlockHeader(blockchain.headBlockHeader())
.getByBlockNumber(blockchain.headBlockNumber())
.getBadBlocksManager()
.getBadBlocks()
.stream()

@ -44,7 +44,7 @@ public class DebugResyncWorldstate implements JsonRpcMethod {
@Override
public JsonRpcResponse response(final JsonRpcRequestContext request) {
protocolSchedule
.getByBlockHeader(blockchain.getChainHeadHeader())
.getByBlockNumber(blockchain.getChainHeadBlockNumber())
.getBadBlocksManager()
.reset();
return new JsonRpcSuccessResponse(

@ -60,7 +60,7 @@ public class DebugStandardTraceBadBlockToFile extends DebugStandardTraceBlockToF
final Blockchain blockchain = blockchainQueries.get().getBlockchain();
final ProtocolSpec protocolSpec =
protocolSchedule.getByBlockHeader(blockchain.getChainHeadHeader());
protocolSchedule.getByBlockNumber(blockchain.getChainHeadHeader().getNumber());
final BadBlockManager badBlockManager = protocolSpec.getBadBlocksManager();
return badBlockManager

@ -20,7 +20,7 @@ import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.UnsignedLongParameter;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.UnsignedIntParameter;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
@ -66,10 +66,7 @@ public class EthFeeHistory implements JsonRpcMethod {
public JsonRpcResponse response(final JsonRpcRequestContext request) {
final Object requestId = request.getRequest().getId();
final long blockCount =
Optional.of(request.getRequiredParameter(0, UnsignedLongParameter.class))
.map(UnsignedLongParameter::getValue)
.orElse(0L);
final int blockCount = request.getRequiredParameter(0, UnsignedIntParameter.class).getValue();
if (blockCount < 1 || blockCount > 1024) {
return new JsonRpcErrorResponse(requestId, JsonRpcError.INVALID_PARAMS);

@ -79,7 +79,7 @@ public class EthGetMinerDataByBlockHash implements JsonRpcMethod {
final ProtocolSchedule protocolSchedule,
final BlockchainQueries blockchainQueries) {
final BlockHeader blockHeader = block.getHeader();
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(blockHeader);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(blockHeader.getNumber());
final Wei staticBlockReward = protocolSpec.getBlockReward();
final Wei transactionFee =
block.getTransactions().stream()

@ -121,6 +121,7 @@ public abstract class AbstractEngineForkchoiceUpdated extends ExecutionEngineJso
// TODO: post-merge cleanup, this should be unnecessary after merge
if (requireTerminalPoWBlockValidation()
&& !mergeContext.get().isCheckpointPostMergeSync()
&& !mergeContext.get().isPostMergeAtGenesis()
&& !mergeCoordinator.latestValidAncestorDescendsFromTerminal(newHead)
&& !mergeContext.get().isChainPruningEnabled()) {
logForkchoiceUpdatedCall(INVALID, forkChoice);

@ -216,6 +216,7 @@ public abstract class AbstractEngineNewPayload extends ExecutionEngineJsonRpcMet
// TODO: post-merge cleanup
if (requireTerminalPoWBlockValidation()
&& !mergeContext.get().isCheckpointPostMergeSync()
&& !mergeContext.get().isPostMergeAtGenesis()
&& !mergeCoordinator.latestValidAncestorDescendsFromTerminal(newBlockHeader)
&& !mergeContext.get().isChainPruningEnabled()) {
mergeCoordinator.addBadBlock(block, Optional.empty());

@ -0,0 +1,85 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine;
import static org.hyperledger.besu.util.Slf4jLambdaHelper.traceLambda;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.ExecutionEngineJsonRpcMethod;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.BlockResultFactory;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EngineGetPayloadBodiesResultV1;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import java.util.Arrays;
import java.util.stream.Collectors;
import io.vertx.core.Vertx;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class EngineGetPayloadBodiesByHashV1 extends ExecutionEngineJsonRpcMethod {
private static final int MAX_REQUEST_BLOCKS = 1024;
private static final Logger LOG = LoggerFactory.getLogger(EngineGetPayloadBodiesByHashV1.class);
private final BlockResultFactory blockResultFactory;
public EngineGetPayloadBodiesByHashV1(
final Vertx vertx,
final ProtocolContext protocolContext,
final BlockResultFactory blockResultFactory,
final EngineCallListener engineCallListener) {
super(vertx, protocolContext, engineCallListener);
this.blockResultFactory = blockResultFactory;
}
@Override
public String getName() {
return RpcMethod.ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1.getMethodName();
}
@Override
public JsonRpcResponse syncResponse(final JsonRpcRequestContext request) {
engineCallListener.executionEngineCalled();
final Object reqId = request.getRequest().getId();
final Hash[] blockHashes = request.getRequiredParameter(0, Hash[].class);
traceLambda(LOG, "{} parameters: blockHashes {}", () -> getName(), () -> blockHashes);
if (blockHashes.length > getMaxRequestBlocks()) {
return new JsonRpcErrorResponse(reqId, JsonRpcError.INVALID_RANGE_REQUEST_TOO_LARGE);
}
final Blockchain blockchain = protocolContext.getBlockchain();
final EngineGetPayloadBodiesResultV1 engineGetPayloadBodiesResultV1 =
blockResultFactory.payloadBodiesCompleteV1(
Arrays.stream(blockHashes).map(blockchain::getBlockBody).collect(Collectors.toList()));
return new JsonRpcSuccessResponse(reqId, engineGetPayloadBodiesResultV1);
}
protected int getMaxRequestBlocks() {
return MAX_REQUEST_BLOCKS;
}
}

@ -0,0 +1,113 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine;
import static org.hyperledger.besu.util.Slf4jLambdaHelper.traceLambda;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.ExecutionEngineJsonRpcMethod;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.UnsignedLongParameter;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.BlockResultFactory;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EngineGetPayloadBodiesResultV1;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
import io.vertx.core.Vertx;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class EngineGetPayloadBodiesByRangeV1 extends ExecutionEngineJsonRpcMethod {
private static final Logger LOG = LoggerFactory.getLogger(EngineGetPayloadBodiesByRangeV1.class);
private static final int MAX_REQUEST_BLOCKS = 1024;
private final BlockResultFactory blockResultFactory;
public EngineGetPayloadBodiesByRangeV1(
final Vertx vertx,
final ProtocolContext protocolContext,
final BlockResultFactory blockResultFactory,
final EngineCallListener engineCallListener) {
super(vertx, protocolContext, engineCallListener);
this.blockResultFactory = blockResultFactory;
}
@Override
public String getName() {
return RpcMethod.ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1.getMethodName();
}
@Override
public JsonRpcResponse syncResponse(final JsonRpcRequestContext request) {
engineCallListener.executionEngineCalled();
final long startBlockNumber =
request.getRequiredParameter(0, UnsignedLongParameter.class).getValue();
final long count = request.getRequiredParameter(1, UnsignedLongParameter.class).getValue();
final Object reqId = request.getRequest().getId();
traceLambda(
LOG,
"{} parameters: start block number {} count {}",
() -> getName(),
() -> startBlockNumber,
() -> count);
if (startBlockNumber < 1 || count < 1) {
return new JsonRpcErrorResponse(reqId, JsonRpcError.INVALID_PARAMS);
}
if (count > getMaxRequestBlocks()) {
return new JsonRpcErrorResponse(reqId, JsonRpcError.INVALID_RANGE_REQUEST_TOO_LARGE);
}
final Blockchain blockchain = protocolContext.getBlockchain();
final long chainHeadBlockNumber = blockchain.getChainHeadBlockNumber();
// request startBlockNumber is beyond head of chain
if (chainHeadBlockNumber < startBlockNumber) {
// Empty List of payloadBodies
return new JsonRpcSuccessResponse(reqId, new EngineGetPayloadBodiesResultV1());
}
final long upperBound = startBlockNumber + count;
// if we've received request from blocks beyond the head we exclude those from the query
final long endExclusiveBlockNumber =
chainHeadBlockNumber < upperBound ? chainHeadBlockNumber + 1 : upperBound;
EngineGetPayloadBodiesResultV1 engineGetPayloadBodiesResultV1 =
blockResultFactory.payloadBodiesCompleteV1(
LongStream.range(startBlockNumber, endExclusiveBlockNumber)
.mapToObj(
blockNumber ->
blockchain
.getBlockHashByNumber(blockNumber)
.flatMap(blockchain::getBlockBody))
.collect(Collectors.toList()));
return new JsonRpcSuccessResponse(reqId, engineGetPayloadBodiesResultV1);
}
protected int getMaxRequestBlocks() {
return MAX_REQUEST_BLOCKS;
}
}

@ -83,7 +83,7 @@ public enum JsonRpcError {
INVALID_TERMINAL_BLOCK(-32002, "Terminal block doesn't satisfy terminal block conditions"),
INVALID_FORKCHOICE_STATE(-38002, "Invalid forkchoice state"),
INVALID_PAYLOAD_ATTRIBUTES(-38003, "Invalid payload attributes"),
INVALID_RANGE_REQUEST_TOO_LARGE(-38004, "Too large request"),
// Miner failures
COINBASE_NOT_SET(-32010, "Coinbase not set. Unable to start mining without a coinbase"),
NO_HASHES_PER_SECOND(-32011, "No hashes being generated by the current node"),

@ -16,9 +16,11 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.results;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EngineGetPayloadBodiesResultV1.PayloadBody;
import org.hyperledger.besu.ethereum.api.query.BlockWithMetadata;
import org.hyperledger.besu.ethereum.api.query.TransactionWithMetadata;
import org.hyperledger.besu.ethereum.core.Block;
import org.hyperledger.besu.ethereum.core.BlockBody;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.BlockValueCalculator;
import org.hyperledger.besu.ethereum.core.BlockWithReceipts;
@ -26,6 +28,7 @@ import org.hyperledger.besu.ethereum.core.encoding.TransactionEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.JsonNode;
@ -115,6 +118,15 @@ public class BlockResultFactory {
Quantity.create(blockValue));
}
public EngineGetPayloadBodiesResultV1 payloadBodiesCompleteV1(
final List<Optional<BlockBody>> blockBodies) {
final List<PayloadBody> payloadBodies =
blockBodies.stream()
.map(maybeBody -> maybeBody.map(PayloadBody::new).orElse(null))
.collect(Collectors.toList());
return new EngineGetPayloadBodiesResultV1(payloadBodies);
}
public EngineGetPayloadResultV3 payloadTransactionCompleteV3(
final BlockWithReceipts blockWithReceipts) {
final List<String> txs =

@ -0,0 +1,79 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.api.jsonrpc.internal.results;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.WithdrawalParameter;
import org.hyperledger.besu.ethereum.core.BlockBody;
import org.hyperledger.besu.ethereum.core.encoding.TransactionEncoder;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.tuweni.bytes.Bytes;
@JsonPropertyOrder({"payloadBodies"})
public class EngineGetPayloadBodiesResultV1 {
private final List<PayloadBody> payloadBodies;
public EngineGetPayloadBodiesResultV1() {
this.payloadBodies = Collections.<PayloadBody>emptyList();
}
public EngineGetPayloadBodiesResultV1(final List<PayloadBody> payloadBody) {
this.payloadBodies = payloadBody;
}
@JsonValue
public List<PayloadBody> getPayloadBodies() {
return payloadBodies;
}
public static class PayloadBody {
private final List<String> transactions;
private final List<WithdrawalParameter> withdrawals;
public PayloadBody(final BlockBody blockBody) {
this.transactions =
blockBody.getTransactions().stream()
.map(TransactionEncoder::encodeOpaqueBytes)
.map(Bytes::toHexString)
.collect(Collectors.toList());
this.withdrawals =
blockBody
.getWithdrawals()
.map(
ws ->
ws.stream()
.map(WithdrawalParameter::fromWithdrawal)
.collect(Collectors.toList()))
.orElse(null);
}
@JsonGetter(value = "transactions")
public List<String> getTransactions() {
return transactions;
}
@JsonGetter(value = "withdrawals")
public List<WithdrawalParameter> getWithdrawals() {
return withdrawals;
}
}
}

@ -336,7 +336,7 @@ public class FlatTraceGenerator {
if ("STOP".equals(traceFrame.getOpcode()) && resultBuilder.isGasUsedEmpty()) {
final long callStipend =
protocolSchedule
.getByBlockHeader(block.getHeader())
.getByBlockNumber(block.getHeader().getNumber())
.getGasCalculator()
.getAdditionalCallStipend();
tracesContexts.stream()

@ -46,7 +46,7 @@ public class RewardTraceGenerator {
final BlockHeader blockHeader = block.getHeader();
final List<BlockHeader> ommers = block.getBody().getOmmers();
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(blockHeader);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(blockHeader.getNumber());
final Wei blockReward = protocolSpec.getBlockReward();
final MiningBeneficiaryCalculator miningBeneficiaryCalculator =
protocolSpec.getMiningBeneficiaryCalculator();

@ -23,6 +23,8 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineE
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineForkchoiceUpdatedV1;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineForkchoiceUpdatedV2;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetBlobsBundleV1;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadBodiesByHashV1;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadBodiesByRangeV1;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadV1;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadV2;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadV3;
@ -140,6 +142,10 @@ public class ExecutionEngineJsonRpcMethods extends ApiGroupJsonRpcMethods {
timestampSchedule),
new EngineExchangeTransitionConfiguration(
consensusEngineServer, protocolContext, engineQosTimer),
new EngineGetPayloadBodiesByHashV1(
consensusEngineServer, protocolContext, blockResultFactory, engineQosTimer),
new EngineGetPayloadBodiesByRangeV1(
consensusEngineServer, protocolContext, blockResultFactory, engineQosTimer),
new EngineExchangeCapabilities(consensusEngineServer, protocolContext, engineQosTimer));
} else {
return mapOf(

@ -89,7 +89,7 @@ public abstract class PrivacyApiGroupJsonRpcMethods extends ApiGroupJsonRpcMetho
public GasCalculator getGasCalculator() {
return protocolSchedule
.getByBlockHeader(blockchainQueries.headBlockHeader())
.getByBlockNumber(blockchainQueries.headBlockNumber())
.getGasCalculator();
}

@ -141,15 +141,6 @@ public class BlockchainQueries {
return blockchain.getChainHeadBlockNumber();
}
/**
* Return the header of the head of the chain.
*
* @return The header of the head of the chain.
*/
public BlockHeader headBlockHeader() {
return blockchain.getChainHeadHeader();
}
/**
* Return the header of the last finalized block.
*

@ -214,7 +214,8 @@ public abstract class AbstractEthGraphQLHttpServiceTest {
void importBlock(final int n) {
final Block block = BLOCKS.get(n);
final ProtocolSpec protocolSpec = PROTOCOL_SCHEDULE.getByBlockHeader(block.getHeader());
final ProtocolSpec protocolSpec =
PROTOCOL_SCHEDULE.getByBlockNumber(block.getHeader().getNumber());
final BlockImporter blockImporter = protocolSpec.getBlockImporter();
blockImporter.importBlock(context, block, HeaderValidationMode.FULL);
}

@ -16,7 +16,7 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -95,8 +95,9 @@ public class DebugGetBadBlockTest {
badBlockManager.addBadBlock(badBlockWithTransaction, Optional.empty());
badBlockManager.addBadBlock(badBlockWoTransaction, Optional.empty());
final ProtocolSpec protocolSpec = mock(ProtocolSpec.class);
when(protocolSchedule.getByBlockHeader(any())).thenReturn(protocolSpec);
when(protocolSchedule.getByBlockNumber(anyLong())).thenReturn(protocolSpec);
when(protocolSpec.getBadBlocksManager()).thenReturn(badBlockManager);
final JsonRpcRequestContext request =
@ -124,8 +125,7 @@ public class DebugGetBadBlockTest {
@Test
public void shouldReturnCorrectResponseWhenNoInvalidBlockFound() {
final ProtocolSpec protocolSpec = mock(ProtocolSpec.class);
when(protocolSchedule.getByBlockHeader(any())).thenReturn(protocolSpec);
when(protocolSchedule.getByBlockNumber(anyLong())).thenReturn(protocolSpec);
when(protocolSpec.getBadBlocksManager()).thenReturn(badBlockManager);
final JsonRpcRequestContext request =

@ -88,7 +88,7 @@ public class DebugStandardTraceBadBlockToFileTest {
when(protocolSpec.getBadBlocksManager()).thenReturn(badBlockManager);
when(blockchainQueries.getBlockchain()).thenReturn(blockchain);
when(blockchain.getChainHeadHeader()).thenReturn(new BlockHeaderTestFixture().buildHeader());
when(protocolSchedule.getByBlockHeader(blockHeader)).thenReturn(protocolSpec);
when(protocolSchedule.getByBlockNumber(blockHeader.getNumber())).thenReturn(protocolSpec);
when(transactionTracer.traceTransactionToFile(eq(block.getHash()), any(), any()))
.thenReturn(paths);
final JsonRpcSuccessResponse response =

@ -69,14 +69,14 @@ public class EthFeeHistoryTest {
// should fail because no required params given
assertThatThrownBy(this::feeHistoryRequest).isInstanceOf(InvalidJsonRpcParameters.class);
// should fail because newestBlock not given
assertThatThrownBy(() -> feeHistoryRequest(1)).isInstanceOf(InvalidJsonRpcParameters.class);
assertThatThrownBy(() -> feeHistoryRequest("0x1")).isInstanceOf(InvalidJsonRpcParameters.class);
// should fail because blockCount not given
assertThatThrownBy(() -> feeHistoryRequest("latest"))
.isInstanceOf(InvalidJsonRpcParameters.class);
// should pass because both required params given
feeHistoryRequest(1, "latest");
feeHistoryRequest("0x1", "latest");
// should pass because both required params and optional param given
feeHistoryRequest(1, "latest", new double[] {1, 20.4});
feeHistoryRequest("0x1", "latest", new double[] {1, 20.4});
// should pass because both required params and optional param given
feeHistoryRequest("0x1", "latest", new double[] {1, 20.4});
}
@ -87,7 +87,7 @@ public class EthFeeHistoryTest {
when(londonSpec.getFeeMarket()).thenReturn(FeeMarket.london(5));
when(protocolSchedule.getByBlockNumber(eq(11L))).thenReturn(londonSpec);
assertThat(
((JsonRpcSuccessResponse) feeHistoryRequest(1, "latest", new double[] {100.0}))
((JsonRpcSuccessResponse) feeHistoryRequest("0x1", "latest", new double[] {100.0}))
.getResult())
.isEqualTo(
FeeHistory.FeeHistoryResult.from(
@ -104,7 +104,9 @@ public class EthFeeHistoryTest {
final ProtocolSpec londonSpec = mock(ProtocolSpec.class);
when(londonSpec.getFeeMarket()).thenReturn(FeeMarket.london(5));
when(protocolSchedule.getByBlockNumber(anyLong())).thenReturn(londonSpec);
assertThat(((JsonRpcErrorResponse) feeHistoryRequest(2, "11", new double[] {100.0})).getError())
assertThat(
((JsonRpcErrorResponse) feeHistoryRequest("0x2", "11", new double[] {100.0}))
.getError())
.isEqualTo(JsonRpcError.INVALID_PARAMS);
}
@ -114,11 +116,11 @@ public class EthFeeHistoryTest {
when(londonSpec.getFeeMarket()).thenReturn(FeeMarket.london(5));
when(protocolSchedule.getByBlockNumber(anyLong())).thenReturn(londonSpec);
assertThat(
((JsonRpcErrorResponse) feeHistoryRequest(0, "latest", new double[] {100.0}))
((JsonRpcErrorResponse) feeHistoryRequest("0x0", "latest", new double[] {100.0}))
.getError())
.isEqualTo(JsonRpcError.INVALID_PARAMS);
assertThat(
((JsonRpcErrorResponse) feeHistoryRequest(1025, "latest", new double[] {100.0}))
((JsonRpcErrorResponse) feeHistoryRequest("0x401", "latest", new double[] {100.0}))
.getError())
.isEqualTo(JsonRpcError.INVALID_PARAMS);
}
@ -130,7 +132,7 @@ public class EthFeeHistoryTest {
when(protocolSchedule.getByBlockNumber(anyLong())).thenReturn(londonSpec);
final FeeHistory.FeeHistoryResult result =
(ImmutableFeeHistoryResult)
((JsonRpcSuccessResponse) feeHistoryRequest(20, "latest")).getResult();
((JsonRpcSuccessResponse) feeHistoryRequest("0x14", "latest")).getResult();
assertThat(Long.decode(result.getOldestBlock())).isEqualTo(0);
assertThat(result.getBaseFeePerGas()).hasSize(12);
assertThat(result.getGasUsedRatio()).hasSize(11);
@ -144,7 +146,7 @@ public class EthFeeHistoryTest {
when(protocolSchedule.getByBlockNumber(anyLong())).thenReturn(londonSpec);
final FeeHistory.FeeHistoryResult result =
(FeeHistory.FeeHistoryResult)
((JsonRpcSuccessResponse) feeHistoryRequest(1, "latest")).getResult();
((JsonRpcSuccessResponse) feeHistoryRequest("0x1", "latest")).getResult();
assertThat(Wei.fromHexString(result.getBaseFeePerGas().get(1)))
.isEqualTo(FeeMarket.london(11).getInitialBasefee());
}
@ -162,7 +164,7 @@ public class EthFeeHistoryTest {
blockchain.appendBlock(emptyBlock, gen.receipts(emptyBlock));
final FeeHistory.FeeHistoryResult result =
(FeeHistory.FeeHistoryResult)
((JsonRpcSuccessResponse) feeHistoryRequest(1, "latest", new double[] {100.0}))
((JsonRpcSuccessResponse) feeHistoryRequest("0x1", "latest", new double[] {100.0}))
.getResult();
assertThat(result.getReward()).isEqualTo(List.of(List.of("0x0")));
}

@ -78,7 +78,7 @@ public class EthGetMinerDataByBlockHashTest {
header, Collections.emptyList(), Collections.emptyList(), Difficulty.of(100L), 5);
when(blockchainQueries.blockByHash(any())).thenReturn(Optional.of(blockWithMetadata));
when(protocolSchedule.getByBlockHeader(header)).thenReturn(protocolSpec);
when(protocolSchedule.getByBlockNumber(header.getNumber())).thenReturn(protocolSpec);
when(protocolSpec.getBlockReward()).thenReturn(Wei.fromEth(2));
when(blockchainQueries.getBlockchain()).thenReturn(blockChain);

@ -76,7 +76,7 @@ public class EthGetMinerDataByBlockNumberTest {
header, Collections.emptyList(), Collections.emptyList(), Difficulty.of(100L), 5);
when(blockchainQueries.blockByNumber(anyLong())).thenReturn(Optional.of(blockWithMetadata));
when(protocolSchedule.getByBlockHeader(header)).thenReturn(protocolSpec);
when(protocolSchedule.getByBlockNumber(header.getNumber())).thenReturn(protocolSpec);
when(protocolSpec.getBlockReward()).thenReturn(Wei.fromEth(2));
when(blockchainQueries.getBlockchain()).thenReturn(blockChain);

@ -0,0 +1,267 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError.INVALID_RANGE_REQUEST_TOO_LARGE;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import org.hyperledger.besu.crypto.SignatureAlgorithm;
import org.hyperledger.besu.crypto.SignatureAlgorithmFactory;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.GWei;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponseType;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.BlockResultFactory;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EngineGetPayloadBodiesResultV1;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.core.BlockBody;
import org.hyperledger.besu.ethereum.core.TransactionTestFixture;
import org.hyperledger.besu.ethereum.core.Withdrawal;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import io.vertx.core.Vertx;
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt64;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class EngineGetPayloadBodiesByHashV1Test {
private EngineGetPayloadBodiesByHashV1 method;
private static final Vertx vertx = Vertx.vertx();
private static final BlockResultFactory blockResultFactory = new BlockResultFactory();
@Mock private ProtocolContext protocolContext;
@Mock private EngineCallListener engineCallListener;
@Mock private MutableBlockchain blockchain;
@Before
public void before() {
when(protocolContext.getBlockchain()).thenReturn(blockchain);
this.method =
spy(
new EngineGetPayloadBodiesByHashV1(
vertx, protocolContext, blockResultFactory, engineCallListener));
}
@Test
public void shouldReturnExpectedMethodName() {
assertThat(method.getName()).isEqualTo("engine_getPayloadBodiesByHashV1");
}
@Test
public void shouldReturnEmptyPayloadBodiesWithEmptyHash() {
final var resp = resp(new Hash[] {});
final EngineGetPayloadBodiesResultV1 result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().isEmpty()).isTrue();
}
@Test
public void shouldReturnPayloadForKnownHashes() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final Hash blockHash3 = Hash.wrap(Bytes32.random());
final BlockBody blockBody1 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
final BlockBody blockBody2 =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
final BlockBody blockBody3 =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(blockBody1));
when(blockchain.getBlockBody(blockHash2)).thenReturn(Optional.of(blockBody2));
when(blockchain.getBlockBody(blockHash3)).thenReturn(Optional.of(blockBody3));
final var resp = resp(new Hash[] {blockHash1, blockHash2, blockHash3});
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0).getTransactions().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1).getTransactions().size()).isEqualTo(2);
assertThat(result.getPayloadBodies().get(2).getTransactions().size()).isEqualTo(3);
}
@Test
public void shouldReturnNullForUnknownHashes() {
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final Hash blockHash3 = Hash.wrap(Bytes32.random());
final var resp = resp(new Hash[] {blockHash1, blockHash2, blockHash3});
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0)).isNull();
assertThat(result.getPayloadBodies().get(1)).isNull();
assertThat(result.getPayloadBodies().get(2)).isNull();
}
@Test
public void shouldReturnNullForUnknownHashAndPayloadForKnownHash() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final Hash blockHash3 = Hash.wrap(Bytes32.random());
final BlockBody blockBody1 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
final BlockBody blockBody3 =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(blockBody1));
when(blockchain.getBlockBody(blockHash3)).thenReturn(Optional.of(blockBody3));
final var resp = resp(new Hash[] {blockHash1, blockHash2, blockHash3});
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0).getTransactions().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1)).isNull();
assertThat(result.getPayloadBodies().get(2).getTransactions().size()).isEqualTo(3);
}
@Test
public void shouldReturnWithdrawalNullWhenBlockIsPreShanghai() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final BlockBody preShanghaiBlockBody =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
final BlockBody preShanghaiBlockBody2 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.empty());
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(preShanghaiBlockBody));
when(blockchain.getBlockBody(blockHash2)).thenReturn(Optional.of(preShanghaiBlockBody2));
final var resp = resp(new Hash[] {blockHash1, blockHash2});
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(2);
assertThat(result.getPayloadBodies().get(0).getTransactions().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0).getWithdrawals()).isNull();
assertThat(result.getPayloadBodies().get(1).getTransactions().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1).getWithdrawals()).isNull();
}
@Test
public void shouldReturnWithdrawalsWhenBlockIsPostShanghai() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final Withdrawal withdrawal =
new Withdrawal(UInt64.ONE, UInt64.ONE, Address.fromHexString("0x1"), GWei.ONE);
final Withdrawal withdrawal2 =
new Withdrawal(UInt64.ONE, UInt64.ONE, Address.fromHexString("0x2"), GWei.ONE);
final BlockBody shanghaiBlockBody =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.of(List.of(withdrawal)));
final BlockBody shanghaiBlockBody2 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.of(List.of(withdrawal2)));
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(shanghaiBlockBody));
when(blockchain.getBlockBody(blockHash2)).thenReturn(Optional.of(shanghaiBlockBody2));
final var resp = resp(new Hash[] {blockHash1, blockHash2});
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(2);
assertThat(result.getPayloadBodies().get(0).getTransactions().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0).getWithdrawals().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1).getTransactions().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1).getWithdrawals().size()).isEqualTo(1);
}
@Test
public void shouldReturnErrorWhenRequestExceedsPermittedNumberOfBlocks() {
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final Hash[] hashes = new Hash[] {blockHash1, blockHash2};
doReturn(1).when(method).getMaxRequestBlocks();
final JsonRpcResponse resp = resp(hashes);
final var result = fromErrorResp(resp);
assertThat(result.getCode()).isEqualTo(INVALID_RANGE_REQUEST_TOO_LARGE.getCode());
}
private JsonRpcResponse resp(final Hash[] hashes) {
return method.response(
new JsonRpcRequestContext(
new JsonRpcRequest(
"2.0",
RpcMethod.ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1.getMethodName(),
new Object[] {hashes})));
}
private EngineGetPayloadBodiesResultV1 fromSuccessResp(final JsonRpcResponse resp) {
assertThat(resp.getType()).isEqualTo(JsonRpcResponseType.SUCCESS);
return Optional.of(resp)
.map(JsonRpcSuccessResponse.class::cast)
.map(JsonRpcSuccessResponse::getResult)
.map(EngineGetPayloadBodiesResultV1.class::cast)
.get();
}
private JsonRpcError fromErrorResp(final JsonRpcResponse resp) {
assertThat(resp.getType()).isEqualTo(JsonRpcResponseType.ERROR);
return Optional.of(resp)
.map(JsonRpcErrorResponse.class::cast)
.map(JsonRpcErrorResponse::getError)
.get();
}
}

@ -0,0 +1,352 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError.INVALID_PARAMS;
import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError.INVALID_RANGE_REQUEST_TOO_LARGE;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import org.hyperledger.besu.crypto.SignatureAlgorithm;
import org.hyperledger.besu.crypto.SignatureAlgorithmFactory;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.GWei;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponseType;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.BlockResultFactory;
import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EngineGetPayloadBodiesResultV1;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.core.BlockBody;
import org.hyperledger.besu.ethereum.core.TransactionTestFixture;
import org.hyperledger.besu.ethereum.core.Withdrawal;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import io.vertx.core.Vertx;
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt64;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class EngineGetPayloadBodiesByRangeV1Test {
private EngineGetPayloadBodiesByRangeV1 method;
private static final Vertx vertx = Vertx.vertx();
private static final BlockResultFactory blockResultFactory = new BlockResultFactory();
@Mock private ProtocolContext protocolContext;
@Mock private EngineCallListener engineCallListener;
@Mock private MutableBlockchain blockchain;
@Before
public void before() {
when(protocolContext.getBlockchain()).thenReturn(blockchain);
this.method =
spy(
new EngineGetPayloadBodiesByRangeV1(
vertx, protocolContext, blockResultFactory, engineCallListener));
}
@Test
public void shouldReturnExpectedMethodName() {
assertThat(method.getName()).isEqualTo("engine_getPayloadBodiesByRangeV1");
}
@Test
public void shouldReturnPayloadForKnownNumber() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final Hash blockHash3 = Hash.wrap(Bytes32.random());
final BlockBody blockBody1 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
final BlockBody blockBody2 =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
final BlockBody blockBody3 =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
when(blockchain.getChainHeadBlockNumber()).thenReturn(Long.valueOf(130));
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(blockBody1));
when(blockchain.getBlockBody(blockHash2)).thenReturn(Optional.of(blockBody2));
when(blockchain.getBlockBody(blockHash3)).thenReturn(Optional.of(blockBody3));
when(blockchain.getBlockHashByNumber(123)).thenReturn(Optional.of(blockHash1));
when(blockchain.getBlockHashByNumber(124)).thenReturn(Optional.of(blockHash2));
when(blockchain.getBlockHashByNumber(125)).thenReturn(Optional.of(blockHash3));
final var resp = resp("0x7b", "0x3");
final EngineGetPayloadBodiesResultV1 result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0).getTransactions().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1).getTransactions().size()).isEqualTo(2);
assertThat(result.getPayloadBodies().get(2).getTransactions().size()).isEqualTo(3);
}
@Test
public void shouldReturnNullForUnknownNumber() {
when(blockchain.getChainHeadBlockNumber()).thenReturn(Long.valueOf(130));
final var resp = resp("0x7b", "0x3");
final EngineGetPayloadBodiesResultV1 result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0)).isNull();
assertThat(result.getPayloadBodies().get(1)).isNull();
assertThat(result.getPayloadBodies().get(2)).isNull();
}
@Test
public void shouldReturnNullForUnknownNumberAndPayloadForKnownNumber() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash3 = Hash.wrap(Bytes32.random());
final BlockBody blockBody1 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
final BlockBody blockBody3 =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
when(blockchain.getChainHeadBlockNumber()).thenReturn(Long.valueOf(130));
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(blockBody1));
when(blockchain.getBlockBody(blockHash3)).thenReturn(Optional.of(blockBody3));
when(blockchain.getBlockHashByNumber(123)).thenReturn(Optional.of(blockHash1));
when(blockchain.getBlockHashByNumber(125)).thenReturn(Optional.of(blockHash3));
final var resp = resp("0x7b", "0x3");
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0).getTransactions().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1)).isNull();
assertThat(result.getPayloadBodies().get(2).getTransactions().size()).isEqualTo(3);
}
@Test
public void shouldReturnNullForWithdrawalsWhenBlockIsPreShanghai() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final BlockBody preShanghaiBlockBody =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList());
final BlockBody preShanghaiBlockBody2 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.empty());
when(blockchain.getChainHeadBlockNumber()).thenReturn(Long.valueOf(130));
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(preShanghaiBlockBody));
when(blockchain.getBlockBody(blockHash2)).thenReturn(Optional.of(preShanghaiBlockBody2));
when(blockchain.getBlockHashByNumber(123)).thenReturn(Optional.of(blockHash1));
when(blockchain.getBlockHashByNumber(124)).thenReturn(Optional.of(blockHash2));
final var resp = resp("0x7b", "0x2");
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(2);
assertThat(result.getPayloadBodies().get(0).getTransactions().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0).getWithdrawals()).isNull();
assertThat(result.getPayloadBodies().get(1).getTransactions().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1).getWithdrawals()).isNull();
;
}
@Test
public void shouldReturnWithdrawalsWhenBlockIsPostShanghai() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final Withdrawal withdrawal =
new Withdrawal(UInt64.ONE, UInt64.ONE, Address.fromHexString("0x1"), GWei.ONE);
final Withdrawal withdrawal2 =
new Withdrawal(UInt64.ONE, UInt64.ONE, Address.fromHexString("0x2"), GWei.ONE);
final BlockBody shanghaiBlockBody =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.of(List.of(withdrawal)));
final BlockBody shanghaiBlockBody2 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.of(List.of(withdrawal2)));
when(blockchain.getChainHeadBlockNumber()).thenReturn(Long.valueOf(130));
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(shanghaiBlockBody));
when(blockchain.getBlockBody(blockHash2)).thenReturn(Optional.of(shanghaiBlockBody2));
when(blockchain.getBlockHashByNumber(123)).thenReturn(Optional.of(blockHash1));
when(blockchain.getBlockHashByNumber(124)).thenReturn(Optional.of(blockHash2));
final var resp = resp("0x7b", "0x2");
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(2);
assertThat(result.getPayloadBodies().get(0).getTransactions().size()).isEqualTo(3);
assertThat(result.getPayloadBodies().get(0).getWithdrawals().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1).getTransactions().size()).isEqualTo(1);
assertThat(result.getPayloadBodies().get(1).getWithdrawals().size()).isEqualTo(1);
}
@Test
public void shouldNotContainTrailingNullForBlocksPastTheCurrentHead() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Withdrawal withdrawal =
new Withdrawal(UInt64.ONE, UInt64.ONE, Address.fromHexString("0x1"), GWei.ONE);
final BlockBody shanghaiBlockBody =
new BlockBody(
List.of(
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair()),
new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.of(List.of(withdrawal)));
when(blockchain.getChainHeadBlockNumber()).thenReturn(Long.valueOf(123));
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(shanghaiBlockBody));
when(blockchain.getBlockHashByNumber(123)).thenReturn(Optional.of(blockHash1));
final var resp = resp("0x7b", "0x3");
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(1);
}
@Test
public void shouldReturnUpUntilHeadWhenStartBlockPlusCountEqualsHeadNumber() {
final SignatureAlgorithm sig = SignatureAlgorithmFactory.getInstance();
final Hash blockHash1 = Hash.wrap(Bytes32.random());
final Hash blockHash2 = Hash.wrap(Bytes32.random());
final Hash blockHash3 = Hash.wrap(Bytes32.random());
final Withdrawal withdrawal =
new Withdrawal(UInt64.ONE, UInt64.ONE, Address.fromHexString("0x1"), GWei.ONE);
final BlockBody shanghaiBlockBody =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.of(List.of(withdrawal)));
final BlockBody shanghaiBlockBody2 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.of(List.of(withdrawal)));
final BlockBody shanghaiBlockBody3 =
new BlockBody(
List.of(new TransactionTestFixture().createTransaction(sig.generateKeyPair())),
Collections.emptyList(),
Optional.of(List.of(withdrawal)));
when(blockchain.getChainHeadBlockNumber()).thenReturn(Long.valueOf(125));
when(blockchain.getBlockBody(blockHash1)).thenReturn(Optional.of(shanghaiBlockBody));
when(blockchain.getBlockBody(blockHash2)).thenReturn(Optional.of(shanghaiBlockBody2));
when(blockchain.getBlockBody(blockHash3)).thenReturn(Optional.of(shanghaiBlockBody3));
when(blockchain.getBlockHashByNumber(123)).thenReturn(Optional.of(blockHash1));
when(blockchain.getBlockHashByNumber(124)).thenReturn(Optional.of(blockHash2));
when(blockchain.getBlockHashByNumber(125)).thenReturn(Optional.of(blockHash3));
final var resp = resp("0x7b", "0x3");
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies().size()).isEqualTo(3);
}
@Test
public void ShouldReturnEmptyPayloadForRequestsPastCurrentHead() {
when(blockchain.getChainHeadBlockNumber()).thenReturn(Long.valueOf(123));
final JsonRpcResponse resp = resp("0x7d", "0x3");
final var result = fromSuccessResp(resp);
assertThat(result.getPayloadBodies()).isEqualTo(Collections.EMPTY_LIST);
}
@Test
public void shouldReturnErrorWhenRequestExceedsPermittedNumberOfBlocks() {
doReturn(3).when(method).getMaxRequestBlocks();
final JsonRpcResponse resp = resp("0x539", "0x4");
final var result = fromErrorResp(resp);
assertThat(result.getCode()).isEqualTo(INVALID_RANGE_REQUEST_TOO_LARGE.getCode());
}
@Test
public void shouldReturnInvalidParamsIfStartIsZero() {
final JsonRpcResponse resp = resp("0x0", "0x539");
final var result = fromErrorResp(resp);
assertThat(result.getCode()).isEqualTo(INVALID_PARAMS.getCode());
}
@Test
public void shouldReturnInvalidParamsIfCountIsZero() {
final JsonRpcResponse resp = resp("0x539", "0x0");
final var result = fromErrorResp(resp);
assertThat(result.getCode()).isEqualTo(INVALID_PARAMS.getCode());
}
private JsonRpcResponse resp(final String startBlockNumber, final String range) {
return method.response(
new JsonRpcRequestContext(
new JsonRpcRequest(
"2.0",
RpcMethod.ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1.getMethodName(),
new Object[] {startBlockNumber, range})));
}
private EngineGetPayloadBodiesResultV1 fromSuccessResp(final JsonRpcResponse resp) {
assertThat(resp.getType()).isEqualTo(JsonRpcResponseType.SUCCESS);
return Optional.of(resp)
.map(JsonRpcSuccessResponse.class::cast)
.map(JsonRpcSuccessResponse::getResult)
.map(EngineGetPayloadBodiesResultV1.class::cast)
.get();
}
private JsonRpcError fromErrorResp(final JsonRpcResponse resp) {
assertThat(resp.getType()).isEqualTo(JsonRpcResponseType.ERROR);
return Optional.of(resp)
.map(JsonRpcErrorResponse.class::cast)
.map(JsonRpcErrorResponse::getError)
.get();
}
}

@ -72,7 +72,7 @@ public class RewardTraceGeneratorTest {
final BlockHeader blockHeader =
gen.header(0x0A, blockBody, new BlockDataGenerator.BlockOptions());
block = new Block(blockHeader, blockBody);
when(protocolSchedule.getByBlockHeader(block.getHeader())).thenReturn(protocolSpec);
when(protocolSchedule.getByBlockNumber(block.getHeader().getNumber())).thenReturn(protocolSpec);
when(protocolSpec.getBlockReward()).thenReturn(blockReward);
when(protocolSpec.getMiningBeneficiaryCalculator()).thenReturn(miningBeneficiaryCalculator);
when(miningBeneficiaryCalculator.calculateBeneficiary(block.getHeader()))

@ -196,17 +196,6 @@ public class BlockchainQueriesTest {
assertThat(result).isEqualTo(3L);
}
@Test
public void getHeadBlockHeader() {
final BlockchainWithData data = setupBlockchain(3);
final BlockchainQueries queries = data.blockchainQueries;
final BlockHeader targetBlockHeader = data.blockData.get(2).block.getHeader();
BlockHeader result = queries.headBlockHeader();
assertThat(targetBlockHeader).isEqualTo(result);
}
@Test
public void getAccountStorageBlockNumber() {
final List<Address> addresses = Arrays.asList(gen.address(), gen.address(), gen.address());

@ -4,7 +4,7 @@
"jsonrpc": "2.0",
"method": "eth_feeHistory",
"params": [
2,
"0x2",
"latest",
[
0.0,

@ -4,7 +4,7 @@
"jsonrpc": "2.0",
"method": "eth_feeHistory",
"params": [
2,
"0x2",
"latest"
]
},

@ -141,7 +141,7 @@ public class BlockMiner<M extends AbstractBlockCreator> implements Runnable {
block.getBody().getTransactions().size());
final BlockImporter importer =
protocolSchedule.getByBlockHeader(block.getHeader()).getBlockImporter();
protocolSchedule.getByBlockNumber(block.getHeader().getNumber()).getBlockImporter();
final BlockImportResult blockImportResult =
importer.importBlock(protocolContext, block, HeaderValidationMode.FULL);
if (blockImportResult.isImported()) {

@ -167,7 +167,16 @@ public class MainnetBlockValidator implements BlockValidator {
return new BlockProcessingResult(
Optional.of(new BlockProcessingOutputs(worldState, receipts)));
}
} catch (StorageException | MerkleTrieException ex) {
} catch (MerkleTrieException ex) {
context
.getSynchronizer()
.ifPresentOrElse(
synchronizer -> synchronizer.healWorldState(ex.getMaybeAddress(), ex.getLocation()),
() ->
handleAndLogImportFailure(
block, new BlockProcessingResult(Optional.empty(), ex)));
return new BlockProcessingResult(Optional.empty(), ex);
} catch (StorageException ex) {
var retval = new BlockProcessingResult(Optional.empty(), ex);
handleAndLogImportFailure(block, retval);
return retval;

@ -15,6 +15,7 @@
package org.hyperledger.besu.ethereum;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.core.Synchronizer;
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
@ -30,6 +31,8 @@ public class ProtocolContext {
private final WorldStateArchive worldStateArchive;
private final ConsensusContext consensusContext;
private Optional<Synchronizer> synchronizer;
public ProtocolContext(
final MutableBlockchain blockchain,
final WorldStateArchive worldStateArchive,
@ -37,6 +40,7 @@ public class ProtocolContext {
this.blockchain = blockchain;
this.worldStateArchive = worldStateArchive;
this.consensusContext = consensusContext;
this.synchronizer = Optional.empty();
}
public static ProtocolContext init(
@ -50,6 +54,14 @@ public class ProtocolContext {
consensusContextFactory.create(blockchain, worldStateArchive, protocolSchedule));
}
public Optional<Synchronizer> getSynchronizer() {
return synchronizer;
}
public void setSynchronizer(final Optional<Synchronizer> synchronizer) {
this.synchronizer = synchronizer;
}
public MutableBlockchain getBlockchain() {
return blockchain;
}

@ -180,7 +180,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
@Override
public Bytes getCode() {
if (code == null) {
code = context.getCode(address).orElse(Bytes.EMPTY);
code = context.getCode(address, codeHash).orElse(Bytes.EMPTY);
}
return code;
}

@ -20,9 +20,11 @@ import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import org.apache.tuweni.bytes.Bytes;
@ -84,6 +86,7 @@ public class BonsaiInMemoryWorldState extends BonsaiPersistedWorldState
final Bytes accountKey = accountUpdate.getKey();
final BonsaiValue<BonsaiAccount> bonsaiValue = accountUpdate.getValue();
final BonsaiAccount updatedAccount = bonsaiValue.getUpdated();
try {
if (updatedAccount == null) {
final Hash addressHash = Hash.hash(accountKey);
accountTrie.remove(addressHash);
@ -92,6 +95,11 @@ public class BonsaiInMemoryWorldState extends BonsaiPersistedWorldState
final Bytes accountValue = updatedAccount.serializeAccount();
accountTrie.put(addressHash, accountValue);
}
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(), Optional.of(Address.wrap(accountKey)), e.getHash(), e.getLocation());
}
}
// TODO write to a cache and then generate a layer update from that and the
@ -129,11 +137,20 @@ public class BonsaiInMemoryWorldState extends BonsaiPersistedWorldState
storageAccountUpdate.getValue().entrySet()) {
final Hash keyHash = storageUpdate.getKey();
final UInt256 updatedStorage = storageUpdate.getValue().getUpdated();
try {
if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
storageTrie.remove(keyHash);
} else {
storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage));
}
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(),
Optional.of(Address.wrap(updatedAddress)),
e.getHash(),
e.getLocation());
}
}
final BonsaiAccount accountUpdated = accountValue.getUpdated();
@ -150,12 +167,7 @@ public class BonsaiInMemoryWorldState extends BonsaiPersistedWorldState
final Hash newWorldStateRootHash = rootHash(localUpdater);
archive
.getTrieLogManager()
.saveTrieLog(
archive,
localUpdater,
newWorldStateRootHash,
blockHeader,
(BonsaiPersistedWorldState) this.copy());
.saveTrieLog(archive, localUpdater, newWorldStateRootHash, blockHeader, this);
worldStateRootHash = newWorldStateRootHash;
worldStateBlockHash = blockHeader.getBlockHash();
isPersisted = true;

@ -14,13 +14,10 @@
*/
package org.hyperledger.besu.ethereum.bonsai;
import org.hyperledger.besu.ethereum.worldstate.PeerTrieNodeFinder;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -35,15 +32,8 @@ public class BonsaiInMemoryWorldStateKeyValueStorage extends BonsaiWorldStateKey
final KeyValueStorage codeStorage,
final KeyValueStorage storageStorage,
final KeyValueStorage trieBranchStorage,
final KeyValueStorage trieLogStorage,
final Optional<PeerTrieNodeFinder> fallbackNodeFinder) {
super(
accountStorage,
codeStorage,
storageStorage,
trieBranchStorage,
trieLogStorage,
fallbackNodeFinder);
final KeyValueStorage trieLogStorage) {
super(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage);
}
@Override

@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.MutableWorldState;
import org.hyperledger.besu.ethereum.core.SnapshotMutableWorldState;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.evm.account.Account;
import org.hyperledger.besu.evm.worldstate.WorldState;
@ -75,29 +76,9 @@ public class BonsaiLayeredWorldState implements MutableWorldState, BonsaiWorldVi
}
public void setNextWorldView(final Optional<BonsaiWorldView> nextWorldView) {
maybeUnSubscribe();
this.nextWorldView = nextWorldView;
}
private void maybeUnSubscribe() {
nextWorldView
.filter(WorldState.class::isInstance)
.map(WorldState.class::cast)
.ifPresent(
ws -> {
try {
ws.close();
} catch (final Exception e) {
// no-op
}
});
}
@Override
public void close() throws Exception {
maybeUnSubscribe();
}
public TrieLogLayer getTrieLog() {
return trieLog;
}
@ -107,7 +88,7 @@ public class BonsaiLayeredWorldState implements MutableWorldState, BonsaiWorldVi
}
@Override
public Optional<Bytes> getCode(final Address address) {
public Optional<Bytes> getCode(final Address address, final Hash codeHash) {
BonsaiLayeredWorldState currentLayer = this;
while (currentLayer != null) {
final Optional<Bytes> maybeCode = currentLayer.trieLog.getCode(address);
@ -124,7 +105,7 @@ public class BonsaiLayeredWorldState implements MutableWorldState, BonsaiWorldVi
} else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) {
currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get();
} else {
return currentLayer.getNextWorldView().get().getCode(address);
return currentLayer.getNextWorldView().get().getCode(address, codeHash);
}
}
return Optional.empty();
@ -291,6 +272,8 @@ public class BonsaiLayeredWorldState implements MutableWorldState, BonsaiWorldVi
new StorageException(
"Unable to copy Layered Worldstate for " + blockHash().toHexString()))) {
return new BonsaiInMemoryWorldState(archive, snapshot.getWorldStateStorage());
} catch (MerkleTrieException ex) {
throw ex; // need to throw to trigger the heal
} catch (Exception ex) {
throw new RuntimeException(ex);
}

@ -25,6 +25,7 @@ import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.MutableWorldState;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.evm.account.Account;
@ -92,15 +93,14 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
worldStateStorage.codeStorage,
worldStateStorage.storageStorage,
worldStateStorage.trieBranchStorage,
worldStateStorage.trieLogStorage,
getWorldStateStorage().getMaybeFallbackNodeFinder());
worldStateStorage.trieLogStorage);
return new BonsaiInMemoryWorldState(archive, bonsaiInMemoryWorldStateKeyValueStorage);
}
@Override
public Optional<Bytes> getCode(@Nonnull final Address address) {
return worldStateStorage.getCode(null, Hash.hash(address));
public Optional<Bytes> getCode(@Nonnull final Address address, final Hash codeHash) {
return worldStateStorage.getCode(codeHash, Hash.hash(address));
}
public void setArchiveStateUnSafe(final BlockHeader blockHeader) {
@ -148,7 +148,7 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
return Hash.wrap(rootHash);
}
private static void addTheAccounts(
private void addTheAccounts(
final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater,
final BonsaiWorldStateUpdater worldStateUpdater,
final StoredMerklePatriciaTrie<Bytes, Bytes> accountTrie) {
@ -157,6 +157,7 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
final Bytes accountKey = accountUpdate.getKey();
final BonsaiValue<BonsaiAccount> bonsaiValue = accountUpdate.getValue();
final BonsaiAccount updatedAccount = bonsaiValue.getUpdated();
try {
if (updatedAccount == null) {
final Hash addressHash = Hash.hash(accountKey);
accountTrie.remove(addressHash);
@ -167,10 +168,15 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
stateUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue);
accountTrie.put(addressHash, accountValue);
}
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(), Optional.of(Address.wrap(accountKey)), e.getHash(), e.getLocation());
}
}
}
private static void updateCode(
private void updateCode(
final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater,
final BonsaiWorldStateUpdater worldStateUpdater) {
for (final Map.Entry<Address, BonsaiValue<Bytes>> codeUpdate :
@ -215,6 +221,7 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
storageAccountUpdate.getValue().entrySet()) {
final Hash keyHash = storageUpdate.getKey();
final UInt256 updatedStorage = storageUpdate.getValue().getUpdated();
try {
if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) {
stateUpdater.removeStorageValueBySlotHash(updatedAddressHash, keyHash);
storageTrie.remove(keyHash);
@ -222,6 +229,14 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
stateUpdater.putStorageValueBySlotHash(updatedAddressHash, keyHash, updatedStorage);
storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage));
}
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(),
Optional.of(Address.wrap(updatedAddress)),
e.getHash(),
e.getLocation());
}
}
final BonsaiAccount accountUpdated = accountValue.getUpdated();
@ -259,6 +274,7 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
oldAccount.getStorageRoot(),
Function.identity(),
Function.identity());
try {
Map<Bytes32, Bytes> entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256);
while (!entriesToDelete.isEmpty()) {
entriesToDelete
@ -272,6 +288,11 @@ public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorld
break;
}
}
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(), Optional.of(Address.wrap(address)), e.getHash(), e.getLocation());
}
}
}

@ -19,8 +19,6 @@ import static org.hyperledger.besu.util.Slf4jLambdaHelper.warnLambda;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.plugin.services.exception.StorageException;
@ -50,25 +48,6 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey
private final AtomicBoolean shouldClose = new AtomicBoolean(false);
private final AtomicBoolean isClosed = new AtomicBoolean(false);
public BonsaiSnapshotWorldStateKeyValueStorage(final StorageProvider snappableStorageProvider) {
this(
snappableStorageProvider
.getSnappableStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE)
.takeSnapshot(),
snappableStorageProvider
.getSnappableStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE)
.takeSnapshot(),
snappableStorageProvider
.getSnappableStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE)
.takeSnapshot(),
snappableStorageProvider
.getSnappableStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE)
.takeSnapshot(),
snappableStorageProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.TRIE_LOG_STORAGE));
}
public BonsaiSnapshotWorldStateKeyValueStorage(
final SnappedKeyValueStorage accountStorage,
final SnappedKeyValueStorage codeStorage,
@ -100,6 +79,12 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey
throw new StorageException("Snapshot storage does not implement clear");
}
@Override
public void clearTrieLog() {
// snapshot storage does not implement clear
throw new StorageException("Snapshot storage does not implement clear");
}
@Override
public synchronized long subscribe(final BonsaiStorageSubscriber sub) {
if (isClosed.get()) {
@ -143,6 +128,16 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey
}
}
@Override
public void onClearTrieLog() {
// when the parent storage clears, close regardless of subscribers
try {
doClose();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public synchronized void close() throws Exception {
// when the parent storage clears, close

@ -16,7 +16,6 @@
package org.hyperledger.besu.ethereum.bonsai;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.hyperledger.besu.datatypes.Hash.fromPlugin;
import static org.hyperledger.besu.ethereum.bonsai.LayeredTrieLogManager.RETAINED_LAYERS;
@ -28,16 +27,22 @@ import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.MutableWorldState;
import org.hyperledger.besu.ethereum.core.SnapshotMutableWorldState;
import org.hyperledger.besu.ethereum.proof.WorldStateProof;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
import org.hyperledger.besu.ethereum.worldstate.PeerTrieNodeFinder;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.evm.worldstate.WorldState;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import com.google.common.annotations.VisibleForTesting;
import org.apache.tuweni.bytes.Bytes;
@ -275,6 +280,9 @@ public class BonsaiWorldStateArchive implements WorldStateArchive {
LOG.debug("Archive rolling finished, now at {}", blockHash);
return Optional.of(mutableState);
} catch (final MerkleTrieException re) {
// need to throw to trigger the heal
throw re;
} catch (final Exception e) {
// if we fail we must clean up the updater
bonsaiUpdater.reset();
@ -282,7 +290,11 @@ public class BonsaiWorldStateArchive implements WorldStateArchive {
return Optional.empty();
}
} catch (final RuntimeException re) {
LOG.error("Archive rolling failed for block hash " + blockHash, re);
LOG.trace("Archive rolling failed for block hash " + blockHash, re);
if (re instanceof MerkleTrieException) {
// need to throw to trigger the heal
throw re;
}
return Optional.empty();
}
}
@ -302,6 +314,57 @@ public class BonsaiWorldStateArchive implements WorldStateArchive {
return persistedState;
}
public void prepareStateHealing(final Address address, final Bytes location) {
final Set<Bytes> keysToDelete = new HashSet<>();
final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = worldStateStorage.updater();
final Hash accountHash = Hash.hash(address);
final StoredMerklePatriciaTrie<Bytes, Bytes> accountTrie =
new StoredMerklePatriciaTrie<>(
(l, h) -> {
final Optional<Bytes> node = worldStateStorage.getAccountStateTrieNode(l, h);
if (node.isPresent()) {
keysToDelete.add(l);
}
return node;
},
persistedState.worldStateRootHash,
Function.identity(),
Function.identity());
try {
accountTrie
.get(accountHash)
.map(RLP::input)
.map(StateTrieAccountValue::readFrom)
.ifPresent(
account -> {
final StoredMerklePatriciaTrie<Bytes, Bytes> storageTrie =
new StoredMerklePatriciaTrie<>(
(l, h) -> {
Optional<Bytes> node =
worldStateStorage.getAccountStorageTrieNode(accountHash, l, h);
if (node.isPresent()) {
keysToDelete.add(Bytes.concatenate(accountHash, l));
}
return node;
},
account.getStorageRoot(),
Function.identity(),
Function.identity());
try {
storageTrie.getPath(location);
} catch (Exception eA) {
LOG.warn("Invalid slot found for account {} at location {}", address, location);
// ignore
}
});
} catch (Exception eA) {
LOG.warn("Invalid node for account {} at location {}", address, location);
// ignore
}
keysToDelete.forEach(bytes -> updater.removeAccountStateTrieNode(bytes, null));
updater.commit();
}
public TrieLogManager getTrieLogManager() {
return trieLogManager;
}
@ -324,9 +387,4 @@ public class BonsaiWorldStateArchive implements WorldStateArchive {
// FIXME we can do proofs for layered tries and the persisted trie
return Optional.empty();
}
public void useFallbackNodeFinder(final Optional<PeerTrieNodeFinder> fallbackNodeFinder) {
checkNotNull(fallbackNodeFinder);
worldStateStorage.useFallbackNodeFinder(fallbackNodeFinder);
}
}

@ -14,15 +14,12 @@
*/
package org.hyperledger.besu.ethereum.bonsai;
import static com.google.common.base.Preconditions.checkNotNull;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie;
import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie;
import org.hyperledger.besu.ethereum.trie.StoredNodeFactory;
import org.hyperledger.besu.ethereum.worldstate.PeerTrieNodeFinder;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
@ -53,16 +50,13 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
protected final KeyValueStorage trieLogStorage;
protected final Subscribers<BonsaiStorageSubscriber> subscribers = Subscribers.create();
private Optional<PeerTrieNodeFinder> maybeFallbackNodeFinder;
public BonsaiWorldStateKeyValueStorage(final StorageProvider provider) {
this(
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE),
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE),
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE),
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE),
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE),
Optional.empty());
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE));
}
public BonsaiWorldStateKeyValueStorage(
@ -71,33 +65,23 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
final KeyValueStorage storageStorage,
final KeyValueStorage trieBranchStorage,
final KeyValueStorage trieLogStorage) {
this(
accountStorage,
codeStorage,
storageStorage,
trieBranchStorage,
trieLogStorage,
Optional.empty());
}
public BonsaiWorldStateKeyValueStorage(
final KeyValueStorage accountStorage,
final KeyValueStorage codeStorage,
final KeyValueStorage storageStorage,
final KeyValueStorage trieBranchStorage,
final KeyValueStorage trieLogStorage,
final Optional<PeerTrieNodeFinder> fallbackNodeFinder) {
this.accountStorage = accountStorage;
this.codeStorage = codeStorage;
this.storageStorage = storageStorage;
this.trieBranchStorage = trieBranchStorage;
this.trieLogStorage = trieLogStorage;
this.maybeFallbackNodeFinder = fallbackNodeFinder;
}
@Override
public Optional<Bytes> getCode(final Bytes32 codeHash, final Hash accountHash) {
return codeStorage.get(accountHash.toArrayUnsafe()).map(Bytes::wrap);
if (codeHash.equals(Hash.EMPTY)) {
return Optional.of(Bytes.EMPTY);
} else {
return codeStorage
.get(accountHash.toArrayUnsafe())
.map(Bytes::wrap)
.filter(b -> Hash.hash(b).equals(codeHash));
}
}
public Optional<Bytes> getAccount(final Hash accountHash) {
@ -128,17 +112,10 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) {
return Optional.of(MerklePatriciaTrie.EMPTY_TRIE_NODE);
} else {
final Optional<Bytes> value =
trieBranchStorage.get(location.toArrayUnsafe()).map(Bytes::wrap);
if (value.isPresent()) {
return value
.filter(b -> Hash.hash(b).equals(nodeHash))
.or(
() ->
maybeFallbackNodeFinder.flatMap(
finder -> finder.getAccountStateTrieNode(location, nodeHash)));
}
return Optional.empty();
return trieBranchStorage
.get(location.toArrayUnsafe())
.map(Bytes::wrap)
.filter(b -> Hash.hash(b).equals(nodeHash));
}
}
@ -148,20 +125,10 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) {
return Optional.of(MerklePatriciaTrie.EMPTY_TRIE_NODE);
} else {
final Optional<Bytes> value =
trieBranchStorage
return trieBranchStorage
.get(Bytes.concatenate(accountHash, location).toArrayUnsafe())
.map(Bytes::wrap);
if (value.isPresent()) {
return value
.filter(b -> Hash.hash(b).equals(nodeHash))
.or(
() ->
maybeFallbackNodeFinder.flatMap(
finder ->
finder.getAccountStorageTrieNode(accountHash, location, nodeHash)));
}
return Optional.empty();
.map(Bytes::wrap)
.filter(b -> Hash.hash(b).equals(nodeHash));
}
}
@ -230,9 +197,8 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
return trieBranchStorage
.get(WORLD_ROOT_HASH_KEY)
.map(Bytes32::wrap)
.filter(hash -> hash.equals(rootHash))
.isPresent()
|| trieLogStorage.containsKey(blockHash.toArrayUnsafe());
.map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe()))
.orElse(false);
}
@Override
@ -245,6 +211,12 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
trieLogStorage.clear();
}
@Override
public void clearTrieLog() {
subscribers.forEach(BonsaiStorageSubscriber::onClearTrieLog);
trieLogStorage.clear();
}
@Override
public void clearFlatDatabase() {
subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage);
@ -277,15 +249,6 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
throw new RuntimeException("removeNodeAddedListener not available");
}
public Optional<PeerTrieNodeFinder> getMaybeFallbackNodeFinder() {
return maybeFallbackNodeFinder;
}
public void useFallbackNodeFinder(final Optional<PeerTrieNodeFinder> maybeFallbackNodeFinder) {
checkNotNull(maybeFallbackNodeFinder);
this.maybeFallbackNodeFinder = maybeFallbackNodeFinder;
}
public synchronized long subscribe(final BonsaiStorageSubscriber sub) {
return subscribers.subscribe(sub);
}
@ -456,6 +419,8 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
default void onClearFlatDatabaseStorage() {}
default void onClearTrieLog() {}
default void onCloseStorage() {}
}
}

@ -20,6 +20,7 @@ import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.evm.account.Account;
import org.hyperledger.besu.evm.account.EvmAccount;
@ -158,11 +159,13 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
protected BonsaiAccount loadAccount(
final Address address,
final Function<BonsaiValue<BonsaiAccount>, BonsaiAccount> bonsaiAccountFunction) {
try {
final BonsaiValue<BonsaiAccount> bonsaiValue = accountsToUpdate.get(address);
if (bonsaiValue == null) {
final Account account = wrappedWorldView().get(address);
if (account instanceof BonsaiAccount) {
final BonsaiAccount mutableAccount = new BonsaiAccount((BonsaiAccount) account, this, true);
final BonsaiAccount mutableAccount =
new BonsaiAccount((BonsaiAccount) account, this, true);
accountsToUpdate.put(address, new BonsaiValue<>((BonsaiAccount) account, mutableAccount));
return mutableAccount;
} else {
@ -171,6 +174,11 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
} else {
return bonsaiAccountFunction.apply(bonsaiValue);
}
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(), Optional.of(address), e.getHash(), e.getLocation());
}
}
@Override
@ -201,7 +209,12 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
codeValue.setUpdated(null);
} else {
wrappedWorldView()
.getCode(deletedAddress)
.getCode(
deletedAddress,
Optional.ofNullable(accountValue)
.map(BonsaiValue::getPrior)
.map(BonsaiAccount::getCodeHash)
.orElse(Hash.EMPTY))
.ifPresent(
deletedCode ->
codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null)));
@ -251,9 +264,9 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
tracked -> {
final Address updatedAddress = tracked.getAddress();
final BonsaiAccount updatedAccount;
if (tracked.getWrappedAccount() == null) {
final BonsaiValue<BonsaiAccount> updatedAccountValue =
accountsToUpdate.get(updatedAddress);
if (tracked.getWrappedAccount() == null) {
updatedAccount = new BonsaiAccount(this, tracked);
tracked.setWrappedAccount(updatedAccount);
if (updatedAccountValue == null) {
@ -281,7 +294,16 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
codeToUpdate.computeIfAbsent(
updatedAddress,
addr ->
new BonsaiValue<>(wrappedWorldView().getCode(addr).orElse(null), null));
new BonsaiValue<>(
wrappedWorldView()
.getCode(
addr,
Optional.ofNullable(updatedAccountValue)
.map(BonsaiValue::getPrior)
.map(BonsaiAccount::getCodeHash)
.orElse(Hash.EMPTY))
.orElse(null),
null));
pendingCode.setUpdated(updatedAccount.getCode());
}
// This is especially to avoid unnecessary computation for withdrawals
@ -331,10 +353,10 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
}
@Override
public Optional<Bytes> getCode(final Address address) {
public Optional<Bytes> getCode(final Address address, final Hash codeHash) {
final BonsaiValue<Bytes> localCode = codeToUpdate.get(address);
if (localCode == null) {
return wrappedWorldView().getCode(address);
return wrappedWorldView().getCode(address, codeHash);
} else {
return Optional.ofNullable(localCode.getUpdated());
}
@ -366,6 +388,7 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
if (emptySlot.contains(slot)) {
return Optional.empty();
} else {
try {
final Optional<UInt256> valueUInt =
(wrappedWorldView() instanceof BonsaiPersistedWorldState)
? ((BonsaiPersistedWorldState) wrappedWorldView())
@ -389,6 +412,11 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
emptySlot.add(Bytes.concatenate(Hash.hash(address), slotHash));
});
return valueUInt;
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(), Optional.of(address), e.getHash(), e.getLocation());
}
}
}
@ -580,6 +608,7 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
private BonsaiValue<BonsaiAccount> loadAccountFromParent(
final Address address, final BonsaiValue<BonsaiAccount> defaultValue) {
try {
final Account parentAccount = wrappedWorldView().get(address);
if (parentAccount instanceof BonsaiAccount) {
final BonsaiAccount account = (BonsaiAccount) parentAccount;
@ -590,6 +619,11 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
} else {
return defaultValue;
}
} catch (MerkleTrieException e) {
// need to throw to trigger the heal
throw new MerkleTrieException(
e.getMessage(), Optional.of(address), e.getHash(), e.getLocation());
}
}
private void rollCodeChange(
@ -600,7 +634,11 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater<BonsaiWorldVie
}
BonsaiValue<Bytes> codeValue = codeToUpdate.get(address);
if (codeValue == null) {
final Bytes storedCode = wrappedWorldView().getCode(address).orElse(Bytes.EMPTY);
final Bytes storedCode =
wrappedWorldView()
.getCode(
address, Optional.ofNullable(expectedCode).map(Hash::hash).orElse(Hash.EMPTY))
.orElse(Bytes.EMPTY);
if (!storedCode.isEmpty()) {
codeValue = new BonsaiValue<>(storedCode, storedCode);
codeToUpdate.put(address, codeValue);

@ -30,7 +30,7 @@ import org.apache.tuweni.units.bigints.UInt256;
public interface BonsaiWorldView extends WorldView {
Optional<Bytes> getCode(Address address);
Optional<Bytes> getCode(Address address, final Hash codeHash);
Optional<Bytes> getStateTrieNode(Bytes location);

@ -105,6 +105,11 @@ public class SnapshotTrieLogManager extends AbstractTrieLogManager<BonsaiSnapsho
dropArchive();
}
@Override
public void onClearTrieLog() {
dropArchive();
}
private void dropArchive() {
// drop all cached snapshot worldstates, they are unsafe when the db has been truncated
LOG.info("Key-value storage truncated, dropping cached worldstates");

@ -94,6 +94,11 @@ public interface Blockchain {
.orElseThrow(() -> new IllegalStateException("Missing genesis block."));
}
default BlockHeader getGenesisBlockHeader() {
return getBlockHeader(BlockHeader.GENESIS_BLOCK_NUMBER)
.orElseThrow(() -> new IllegalStateException("Missing genesis block header."));
}
default Optional<Block> getBlockByHash(final Hash blockHash) {
return getBlockHeader(blockHash)
.flatMap(header -> getBlockBody(blockHash).map(body -> new Block(header, body)));

@ -14,12 +14,15 @@
*/
package org.hyperledger.besu.ethereum.core;
import org.hyperledger.besu.plugin.data.Address;
import org.hyperledger.besu.plugin.data.SyncStatus;
import org.hyperledger.besu.plugin.services.BesuEvents;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import org.apache.tuweni.bytes.Bytes;
/** Provides an interface to block synchronization processes. */
public interface Synchronizer {
@ -42,6 +45,8 @@ public interface Synchronizer {
boolean resyncWorldState();
boolean healWorldState(final Optional<Address> maybeAccountToRepair, final Bytes location);
long subscribeSyncStatus(final BesuEvents.SyncStatusListener listener);
boolean unsubscribeSyncStatus(long observerId);

@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.core.TransactionReceipt;
import org.hyperledger.besu.ethereum.core.Withdrawal;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateMetadataUpdater;
import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
import org.hyperledger.besu.evm.tracing.OperationTracer;
import org.hyperledger.besu.evm.worldstate.WorldState;
@ -165,6 +166,12 @@ public abstract class AbstractBlockProcessor implements BlockProcessor {
try {
worldState.persist(blockHeader);
} catch (MerkleTrieException e) {
LOG.trace("Merkle trie exception during Transaction processing ", e);
if (worldState instanceof BonsaiPersistedWorldState) {
((BonsaiWorldStateUpdater) worldState.updater()).reset();
}
throw e;
} catch (Exception e) {
LOG.error("failed persisting block", e);
return new BlockProcessingResult(Optional.empty(), e);

@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket;
import org.hyperledger.besu.ethereum.privacy.storage.PrivateMetadataUpdater;
import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult;
import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
import org.hyperledger.besu.ethereum.worldstate.GoQuorumMutablePrivateWorldStateUpdater;
import org.hyperledger.besu.evm.AccessListEntry;
@ -490,6 +491,9 @@ public class MainnetTransactionProcessor {
return TransactionProcessingResult.failed(
gasUsedByTransaction, refundedGas, validationResult, initialFrame.getRevertReason());
}
} catch (final MerkleTrieException re) {
// need to throw to trigger the heal
throw re;
} catch (final RuntimeException re) {
LOG.error("Critical Exception Processing Transaction", re);
return TransactionProcessingResult.invalid(

@ -125,7 +125,7 @@ public class PrivateTransactionSimulator {
final PrivateTransaction transaction =
getPrivateTransaction(callParams, header, privacyGroupId, disposablePrivateState);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(header);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(header.getNumber());
final PrivateTransactionProcessor privateTransactionProcessor =
protocolSpec.getPrivateTransactionProcessor();

@ -93,7 +93,7 @@ public class PrivateStorageMigration {
final int lastPmtIndex = findLastPMTIndexInBlock(block);
if (lastPmtIndex >= 0) {
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(blockHeader);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(blockNumber);
final PrivateMigrationBlockProcessor privateMigrationBlockProcessor =
privateMigrationBlockProcessorBuilder.apply(protocolSpec);

@ -97,6 +97,11 @@ public class WorldStateKeyValueStorage implements WorldStateStorage {
keyValueStorage.clear();
}
@Override
public void clearTrieLog() {
// nothing to do for forest
}
@Override
public void clearFlatDatabase() {
// nothing to do for forest

@ -44,6 +44,8 @@ public interface WorldStateStorage {
void clear();
void clearTrieLog();
void clearFlatDatabase();
Updater updater();

@ -243,7 +243,8 @@ public class BlockchainSetupUtil {
if (block.getHeader().getNumber() == BlockHeader.GENESIS_BLOCK_NUMBER) {
continue;
}
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
final ProtocolSpec protocolSpec =
protocolSchedule.getByBlockNumber(block.getHeader().getNumber());
final BlockImporter blockImporter = protocolSpec.getBlockImporter();
final BlockImportResult result =
blockImporter.importBlock(protocolContext, block, HeaderValidationMode.FULL);

@ -18,11 +18,9 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
@ -32,10 +30,8 @@ import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie;
import org.hyperledger.besu.ethereum.trie.StorageEntriesCollector;
import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie;
import org.hyperledger.besu.ethereum.worldstate.PeerTrieNodeFinder;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import java.util.Optional;
import java.util.TreeMap;
import org.apache.tuweni.bytes.Bytes;
@ -48,7 +44,7 @@ public class BonsaiWorldStateKeyValueStorageTest {
@Test
public void getCode_returnsEmpty() {
final BonsaiWorldStateKeyValueStorage storage = emptyStorage();
assertThat(storage.getCode(null, Hash.EMPTY)).isEmpty();
assertThat(storage.getCode(Hash.EMPTY, Hash.EMPTY)).contains(Bytes.EMPTY);
}
@Test
@ -89,7 +85,8 @@ public class BonsaiWorldStateKeyValueStorageTest {
.putCode(Hash.EMPTY, Bytes.EMPTY)
.commit();
assertThat(storage.getCode(null, Hash.EMPTY)).contains(MerklePatriciaTrie.EMPTY_TRIE_NODE);
assertThat(storage.getCode(Hash.hash(MerklePatriciaTrie.EMPTY_TRIE_NODE), Hash.EMPTY))
.contains(MerklePatriciaTrie.EMPTY_TRIE_NODE);
}
@Test
@ -98,7 +95,7 @@ public class BonsaiWorldStateKeyValueStorageTest {
final BonsaiWorldStateKeyValueStorage storage = emptyStorage();
storage.updater().putCode(Hash.EMPTY, bytes).commit();
assertThat(storage.getCode(null, Hash.EMPTY)).contains(bytes);
assertThat(storage.getCode(Hash.hash(bytes), Hash.EMPTY)).contains(bytes);
}
@Test
@ -255,9 +252,8 @@ public class BonsaiWorldStateKeyValueStorageTest {
updaterA.commit();
updaterB.commit();
assertThat(storage.getCode(null, accountHashA)).contains(bytesA);
assertThat(storage.getCode(null, accountHashB)).contains(bytesB);
assertThat(storage.getCode(null, accountHashD)).contains(bytesC);
assertThat(storage.getCode(Hash.hash(bytesB), accountHashB)).contains(bytesB);
assertThat(storage.getCode(Hash.hash(bytesC), accountHashD)).contains(bytesC);
}
@Test
@ -295,62 +291,6 @@ public class BonsaiWorldStateKeyValueStorageTest {
assertThat(storage.isWorldStateAvailable(Bytes32.wrap(nodeHashKey), Hash.EMPTY)).isTrue();
}
@Test
public void getAccountStateTrieNode_callFallbackMechanismForInvalidNode() {
PeerTrieNodeFinder peerTrieNodeFinder = mock(PeerTrieNodeFinder.class);
final Bytes location = Bytes.fromHexString("0x01");
final Bytes bytesInDB = Bytes.fromHexString("0x123456");
final Hash hashToFind = Hash.hash(Bytes.of(1));
final Bytes bytesToFind = Bytes.fromHexString("0x123457");
final BonsaiWorldStateKeyValueStorage storage = emptyStorage();
when(peerTrieNodeFinder.getAccountStateTrieNode(location, hashToFind))
.thenReturn(Optional.of(bytesToFind));
storage.useFallbackNodeFinder(Optional.of(peerTrieNodeFinder));
storage.updater().putAccountStateTrieNode(location, Hash.hash(bytesInDB), bytesInDB).commit();
Optional<Bytes> accountStateTrieNodeResult =
storage.getAccountStateTrieNode(location, hashToFind);
verify(peerTrieNodeFinder).getAccountStateTrieNode(location, hashToFind);
assertThat(accountStateTrieNodeResult).contains(bytesToFind);
}
@Test
public void getAccountStorageTrieNode_callFallbackMechanismForInvalidNode() {
PeerTrieNodeFinder peerTrieNodeFinder = mock(PeerTrieNodeFinder.class);
final Hash account = Hash.hash(Bytes32.ZERO);
final Bytes location = Bytes.fromHexString("0x01");
final Bytes bytesInDB = Bytes.fromHexString("0x123456");
final Hash hashToFind = Hash.hash(Bytes.of(1));
final Bytes bytesToFind = Bytes.fromHexString("0x123457");
final BonsaiWorldStateKeyValueStorage storage = emptyStorage();
when(peerTrieNodeFinder.getAccountStorageTrieNode(account, location, hashToFind))
.thenReturn(Optional.of(bytesToFind));
storage.useFallbackNodeFinder(Optional.of(peerTrieNodeFinder));
storage
.updater()
.putAccountStorageTrieNode(account, location, Hash.hash(bytesInDB), bytesInDB)
.commit();
Optional<Bytes> accountStateTrieNodeResult =
storage.getAccountStorageTrieNode(account, location, hashToFind);
verify(peerTrieNodeFinder).getAccountStorageTrieNode(account, location, hashToFind);
assertThat(accountStateTrieNodeResult).contains(bytesToFind);
}
private BonsaiWorldStateKeyValueStorage emptyStorage() {
return new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider());
}

@ -584,7 +584,8 @@ public class BlockPropagationManager implements UnverifiedForkchoiceListener {
new IllegalArgumentException(
"Incapable of retrieving header from non-existent parent of "
+ block.toLogString()));
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
final ProtocolSpec protocolSpec =
protocolSchedule.getByBlockNumber(block.getHeader().getNumber());
final BlockHeaderValidator blockHeaderValidator = protocolSpec.getBlockHeaderValidator();
final BadBlockManager badBlockManager = protocolSpec.getBadBlocksManager();
return ethContext

@ -15,12 +15,12 @@
package org.hyperledger.besu.ethereum.eth.sync;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.hyperledger.besu.util.Slf4jLambdaHelper.infoLambda;
import org.hyperledger.besu.consensus.merge.ForkchoiceEvent;
import org.hyperledger.besu.consensus.merge.UnverifiedForkchoiceListener;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive;
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.core.Synchronizer;
import org.hyperledger.besu.ethereum.eth.manager.EthContext;
import org.hyperledger.besu.ethereum.eth.sync.checkpointsync.CheckpointDownloaderFactory;
@ -33,25 +33,27 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapDownloaderFactory;
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapPersistedContext;
import org.hyperledger.besu.ethereum.eth.sync.state.PendingBlocksManager;
import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStatePeerTrieNodeFinder;
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
import org.hyperledger.besu.ethereum.p2p.network.ProtocolManager;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.worldstate.PeerTrieNodeFinder;
import org.hyperledger.besu.ethereum.worldstate.Pruner;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.metrics.BesuMetricCategory;
import org.hyperledger.besu.plugin.data.Address;
import org.hyperledger.besu.plugin.data.SyncStatus;
import org.hyperledger.besu.plugin.services.BesuEvents.SyncStatusListener;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.util.log.FramedLogMessage;
import java.nio.file.Path;
import java.time.Clock;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.function.Supplier;
import org.apache.tuweni.bytes.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -63,14 +65,10 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
private final SyncState syncState;
private final AtomicBoolean running = new AtomicBoolean(false);
private final Optional<BlockPropagationManager> blockPropagationManager;
private final Function<Boolean, Optional<FastSyncDownloader<?>>> fastSyncFactory;
private final Supplier<Optional<FastSyncDownloader<?>>> fastSyncFactory;
private Optional<FastSyncDownloader<?>> fastSyncDownloader;
private final Optional<FullSyncDownloader> fullSyncDownloader;
private final EthContext ethContext;
private final ProtocolContext protocolContext;
private final ProtocolManager protocolManager;
private final WorldStateStorage worldStateStorage;
private final MetricsSystem metricsSystem;
private final PivotBlockSelector pivotBlockSelector;
private final SyncTerminationCondition terminationCondition;
@ -88,16 +86,11 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
final Clock clock,
final MetricsSystem metricsSystem,
final SyncTerminationCondition terminationCondition,
final ProtocolManager protocolManager,
final PivotBlockSelector pivotBlockSelector) {
this.maybePruner = maybePruner;
this.syncState = syncState;
this.protocolManager = protocolManager;
this.pivotBlockSelector = pivotBlockSelector;
this.ethContext = ethContext;
this.protocolContext = protocolContext;
this.worldStateStorage = worldStateStorage;
this.metricsSystem = metricsSystem;
this.terminationCondition = terminationCondition;
ChainHeadTracker.trackChainHeadForPeers(
@ -136,7 +129,7 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
if (SyncMode.FAST.equals(syncConfig.getSyncMode())) {
this.fastSyncFactory =
(isResync) ->
() ->
FastDownloaderFactory.create(
pivotBlockSelector,
syncConfig,
@ -147,11 +140,10 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
ethContext,
worldStateStorage,
syncState,
clock,
isResync);
clock);
} else if (SyncMode.X_CHECKPOINT.equals(syncConfig.getSyncMode())) {
this.fastSyncFactory =
(isResync) ->
() ->
CheckpointDownloaderFactory.createCheckpointDownloader(
new SnapPersistedContext(storageProvider),
pivotBlockSelector,
@ -163,11 +155,10 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
ethContext,
worldStateStorage,
syncState,
clock,
isResync);
clock);
} else {
this.fastSyncFactory =
(isResync) ->
() ->
SnapDownloaderFactory.createSnapDownloader(
new SnapPersistedContext(storageProvider),
pivotBlockSelector,
@ -179,12 +170,11 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
ethContext,
worldStateStorage,
syncState,
clock,
isResync);
clock);
}
// create a non-resync fast sync downloader:
this.fastSyncDownloader = this.fastSyncFactory.apply(false);
this.fastSyncDownloader = this.fastSyncFactory.get();
metricsSystem.createLongGauge(
BesuMetricCategory.ETHEREUM,
@ -222,7 +212,6 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
future = fastSyncDownloader.get().start().thenCompose(this::handleSyncResult);
} else {
syncState.markInitialSyncPhaseAsDone();
enableFallbackNodeFinder();
future = startFullSync();
}
return future.thenApply(this::finalizeSync);
@ -271,8 +260,6 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
pivotBlockSelector.close();
syncState.markInitialSyncPhaseAsDone();
enableFallbackNodeFinder();
if (terminationCondition.shouldContinueDownload()) {
return startFullSync();
} else {
@ -281,19 +268,6 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
}
}
private void enableFallbackNodeFinder() {
if (worldStateStorage instanceof BonsaiWorldStateKeyValueStorage) {
final Optional<PeerTrieNodeFinder> fallbackNodeFinder =
Optional.of(
new WorldStatePeerTrieNodeFinder(
ethContext, protocolManager, protocolContext.getBlockchain(), metricsSystem));
((BonsaiWorldStateArchive) protocolContext.getWorldStateArchive())
.useFallbackNodeFinder(fallbackNodeFinder);
((BonsaiWorldStateKeyValueStorage) worldStateStorage)
.useFallbackNodeFinder(fallbackNodeFinder);
}
}
private CompletableFuture<Void> startFullSync() {
maybePruner.ifPresent(Pruner::start);
return fullSyncDownloader
@ -322,10 +296,41 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi
stop();
fastSyncDownloader.get().deleteFastSyncState();
}
// recreate fast sync with resync and start
this.syncState.markInitialSyncRestart();
this.syncState.markResyncNeeded();
this.fastSyncDownloader = this.fastSyncFactory.get();
start();
return true;
}
@Override
public boolean healWorldState(
final Optional<Address> maybeAccountToRepair, final Bytes location) {
// recreate fast sync with resync and start
if (fastSyncDownloader.isPresent() && running.get()) {
stop();
fastSyncDownloader.get().deleteFastSyncState();
}
final List<String> lines = new ArrayList<>();
lines.add("Besu has identified a problem with its worldstate database.");
lines.add("Your node will fetch the correct data from peers to repair the problem.");
lines.add("Starting the sync pipeline...");
infoLambda(LOG, FramedLogMessage.generate(lines));
this.syncState.markInitialSyncRestart();
this.fastSyncDownloader = this.fastSyncFactory.apply(true);
this.syncState.markResyncNeeded();
maybeAccountToRepair.ifPresent(
address -> {
if (this.protocolContext.getWorldStateArchive() instanceof BonsaiWorldStateArchive) {
((BonsaiWorldStateArchive) this.protocolContext.getWorldStateArchive())
.prepareStateHealing(
org.hyperledger.besu.datatypes.Address.wrap(address), location);
}
this.syncState.markAccountToRepair(maybeAccountToRepair);
});
this.fastSyncDownloader = this.fastSyncFactory.get();
start();
return true;
}

@ -157,6 +157,7 @@ public class PipelineChainDownloader implements ChainDownloader {
if (!syncTargetManager.shouldContinueDownloading()) {
return CompletableFuture.completedFuture(null);
}
syncState.setSyncTarget(target.peer(), target.commonAncestor());
debugLambda(
LOG,

@ -260,12 +260,12 @@ public class BackwardSyncContext {
return protocolContext;
}
public BlockValidator getBlockValidator(final BlockHeader blockHeader) {
return protocolSchedule.getByBlockHeader(blockHeader).getBlockValidator();
public BlockValidator getBlockValidator(final long blockNumber) {
return protocolSchedule.getByBlockNumber(blockNumber).getBlockValidator();
}
public BlockValidator getBlockValidatorForBlock(final Block block) {
return getBlockValidator(block.getHeader());
return getBlockValidator(block.getHeader().getNumber());
}
public boolean isReady() {

@ -59,10 +59,12 @@ public class BackwardSyncStep {
@VisibleForTesting
protected CompletableFuture<List<BlockHeader>> requestHeaders(final Hash hash) {
if (context.getProtocolContext().getBlockchain().contains(hash)) {
final Optional<BlockHeader> blockHeader =
context.getProtocolContext().getBlockchain().getBlockHeader(hash);
if (blockHeader.isPresent()) {
LOG.debug(
"Hash {} already present in local blockchain no need to request headers to peers", hash);
return CompletableFuture.completedFuture(List.of());
return CompletableFuture.completedFuture(List.of(blockHeader.get()));
}
final int batchSize = context.getBatchSize();
@ -104,8 +106,9 @@ public class BackwardSyncStep {
saveHeader(blockHeader);
}
if (!blockHeaders.isEmpty()) {
logProgress(blockHeaders.get(blockHeaders.size() - 1).getNumber());
}
return null;
}

@ -84,11 +84,13 @@ public class BackwardsSyncAlgorithm implements BesuEvents.InitialSyncCompletionL
final MutableBlockchain blockchain = context.getProtocolContext().getBlockchain();
final BlockHeader firstAncestorHeader = maybeFirstAncestorHeader.get();
if (blockchain.contains(firstAncestorHeader.getHash())) {
final BlockHeader chainHeader = blockchain.getChainHeadHeader();
if (blockchain.contains(firstAncestorHeader.getHash())
&& firstAncestorHeader.getNumber() <= chainHeader.getNumber()) {
return executeProcessKnownAncestors();
}
if (blockchain.getChainHead().getHeight() > firstAncestorHeader.getNumber()) {
if (chainHeader.getNumber() > firstAncestorHeader.getNumber()) {
debugLambda(
LOG,
"Backward reached below current chain head {} : {}",

@ -44,7 +44,9 @@ public interface FinalBlockConfirmation {
}
static FinalBlockConfirmation ancestorConfirmation(final MutableBlockchain blockchain) {
return firstHeader -> blockchain.contains(firstHeader.getParentHash());
return firstHeader ->
blockchain.contains(firstHeader.getParentHash())
&& blockchain.getChainHeadBlockNumber() + 1 >= firstHeader.getNumber();
}
static FinalBlockConfirmation confirmationChain(final FinalBlockConfirmation... confirmations) {

@ -105,11 +105,11 @@ public class ForwardSyncStep {
}
for (Block block : blocks) {
final Optional<Block> parent =
final Optional<BlockHeader> parent =
context
.getProtocolContext()
.getBlockchain()
.getBlockByHash(block.getHeader().getParentHash());
.getBlockHeader(block.getHeader().getParentHash());
if (parent.isEmpty()) {
context.halveBatchSize();

@ -20,8 +20,10 @@ package org.hyperledger.besu.ethereum.eth.sync.backwardsync;
import static org.hyperledger.besu.util.Slf4jLambdaHelper.debugLambda;
import static org.slf4j.LoggerFactory.getLogger;
import org.hyperledger.besu.ethereum.core.Block;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import com.google.common.annotations.VisibleForTesting;
@ -47,17 +49,33 @@ public class ProcessKnownAncestorsStep {
protected void processKnownAncestors() {
while (backwardChain.getFirstAncestorHeader().isPresent()) {
BlockHeader header = backwardChain.getFirstAncestorHeader().orElseThrow();
if (context.getProtocolContext().getBlockchain().contains(header.getHash())) {
final long chainHeadBlockNumber =
context.getProtocolContext().getBlockchain().getChainHeadBlockNumber();
boolean isFirstUnProcessedHeader = true;
if (context.getProtocolContext().getBlockchain().contains(header.getHash())
&& header.getNumber() <= chainHeadBlockNumber) {
debugLambda(
LOG,
"Block {} is already imported, we can ignore it for the sync process",
() -> header.toLogString());
header::toLogString);
backwardChain.dropFirstHeader();
} else if (context.getProtocolContext().getBlockchain().contains(header.getParentHash())
&& backwardChain.isTrusted(header.getHash())) {
debugLambda(LOG, "Importing trusted block {}", header::toLogString);
context.saveBlock(backwardChain.getTrustedBlock(header.getHash()));
} else {
isFirstUnProcessedHeader = false;
} else if (context.getProtocolContext().getBlockchain().contains(header.getParentHash())) {
final boolean isTrustedBlock = backwardChain.isTrusted(header.getHash());
final Optional<Block> block =
isTrustedBlock
? Optional.of(backwardChain.getTrustedBlock(header.getHash()))
: context.getProtocolContext().getBlockchain().getBlockByHash(header.getHash());
if (block.isPresent()) {
debugLambda(LOG, "Importing block {}", header::toLogString);
context.saveBlock(block.get());
if (isTrustedBlock) {
backwardChain.dropFirstHeader();
isFirstUnProcessedHeader = false;
}
}
}
if (isFirstUnProcessedHeader) {
debugLambda(LOG, "First unprocessed header is {}", header::toLogString);
return;
}

@ -14,6 +14,7 @@
*/
package org.hyperledger.besu.ethereum.eth.sync.checkpointsync;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.eth.manager.EthContext;
@ -60,8 +61,7 @@ public class CheckpointDownloaderFactory extends SnapDownloaderFactory {
final EthContext ethContext,
final WorldStateStorage worldStateStorage,
final SyncState syncState,
final Clock clock,
final boolean isResync) {
final Clock clock) {
final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
final FastSyncStateStorage fastSyncStateStorage =
@ -81,13 +81,12 @@ public class CheckpointDownloaderFactory extends SnapDownloaderFactory {
final FastSyncState fastSyncState =
fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule));
if (isResync) {
if (syncState.isResyncNeeded()) {
snapContext.clear();
worldStateStorage.clear();
}
if (!isResync
&& fastSyncState.getPivotBlockHeader().isEmpty()
syncState
.getAccountToRepair()
.ifPresent(address -> snapContext.addInconsistentAccount(Hash.hash(address)));
} else if (fastSyncState.getPivotBlockHeader().isEmpty()
&& protocolContext.getBlockchain().getChainHeadBlockNumber()
!= BlockHeader.GENESIS_BLOCK_NUMBER) {
LOG.info(
@ -109,10 +108,12 @@ public class CheckpointDownloaderFactory extends SnapDownloaderFactory {
pivotBlockSelector,
metricsSystem);
} else {
if (!syncState.isResyncNeeded()) {
LOG.info(
"Checkpoint sync start with block {} and hash {}",
syncState.getCheckpoint().get().blockNumber(),
syncState.getCheckpoint().get().blockHash());
}
fastSyncActions =
new CheckpointSyncActions(
syncConfig,

@ -114,7 +114,7 @@ public class FastImportBlocksStep implements Consumer<List<BlockWithReceipts>> {
protected boolean importBlock(final BlockWithReceipts blockWithReceipts) {
final BlockImporter importer =
protocolSchedule.getByBlockHeader(blockWithReceipts.getHeader()).getBlockImporter();
protocolSchedule.getByBlockNumber(blockWithReceipts.getNumber()).getBlockImporter();
final BlockImportResult blockImportResult =
importer.fastImportBlock(
protocolContext,

@ -85,6 +85,7 @@ public class FastSyncDownloader<REQUEST> {
if (worldStateStorage instanceof BonsaiWorldStateKeyValueStorage) {
LOG.info("Clearing bonsai flat account db");
worldStateStorage.clearFlatDatabase();
worldStateStorage.clearTrieLog();
}
LOG.debug("Start sync with initial sync state {}", fastSyncState);
return findPivotBlock(fastSyncState, fss -> downloadChainAndWorldState(fastSyncActions, fss));

@ -167,8 +167,16 @@ public class FastSyncTargetManager extends SyncTargetManager {
@Override
public boolean shouldContinueDownloading() {
final BlockHeader pivotBlockHeader = fastSyncState.getPivotBlockHeader().get();
return !protocolContext.getBlockchain().getChainHeadHash().equals(pivotBlockHeader.getHash())
|| !worldStateStorage.isWorldStateAvailable(
boolean isValidChainHead =
protocolContext.getBlockchain().getChainHeadHash().equals(pivotBlockHeader.getHash());
if (!isValidChainHead) {
if (protocolContext.getBlockchain().contains(pivotBlockHeader.getHash())) {
protocolContext.getBlockchain().rewindToBlock(pivotBlockHeader.getHash());
} else {
return true;
}
}
return !worldStateStorage.isWorldStateAvailable(
pivotBlockHeader.getStateRoot(), pivotBlockHeader.getBlockHash());
}
}

@ -61,8 +61,7 @@ public class FastDownloaderFactory {
final EthContext ethContext,
final WorldStateStorage worldStateStorage,
final SyncState syncState,
final Clock clock,
final boolean isResync) {
final Clock clock) {
final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
final FastSyncStateStorage fastSyncStateStorage =
@ -82,9 +81,8 @@ public class FastDownloaderFactory {
final FastSyncState fastSyncState =
fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule));
if (isResync) {
worldStateStorage.clear();
} else if (fastSyncState.getPivotBlockHeader().isEmpty()
if (!syncState.isResyncNeeded()
&& fastSyncState.getPivotBlockHeader().isEmpty()
&& protocolContext.getBlockchain().getChainHeadBlockNumber()
!= BlockHeader.GENESIS_BLOCK_NUMBER) {
LOG.info(

@ -58,7 +58,7 @@ public class FullImportBlockStep implements Consumer<Block> {
final long blockNumber = block.getHeader().getNumber();
final String blockHash = block.getHash().toHexString();
final BlockImporter importer =
protocolSchedule.getByBlockHeader(block.getHeader()).getBlockImporter();
protocolSchedule.getByBlockNumber(blockNumber).getBlockImporter();
final BlockImportResult blockImportResult =
importer.importBlock(protocolContext, block, HeaderValidationMode.SKIP_DETACHED);
if (!blockImportResult.isImported()) {

@ -75,7 +75,8 @@ public class RangeHeadersValidationStep implements Function<RangeHeaders, Stream
}
private boolean isValid(final BlockHeader expectedParent, final BlockHeader firstHeaderToImport) {
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(firstHeaderToImport);
final ProtocolSpec protocolSpec =
protocolSchedule.getByBlockNumber(firstHeaderToImport.getNumber());
final BlockHeaderValidator validator = protocolSpec.getBlockHeaderValidator();
return validator.validateHeader(
firstHeaderToImport,

@ -14,6 +14,7 @@
*/
package org.hyperledger.besu.ethereum.eth.sync.snapsync;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.eth.manager.EthContext;
@ -56,8 +57,7 @@ public class SnapDownloaderFactory extends FastDownloaderFactory {
final EthContext ethContext,
final WorldStateStorage worldStateStorage,
final SyncState syncState,
final Clock clock,
final boolean isResync) {
final Clock clock) {
final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
final FastSyncStateStorage fastSyncStateStorage =
@ -77,13 +77,12 @@ public class SnapDownloaderFactory extends FastDownloaderFactory {
final FastSyncState fastSyncState =
fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule));
if (isResync) {
if (syncState.isResyncNeeded()) {
snapContext.clear();
worldStateStorage.clear();
}
if (!isResync
&& fastSyncState.getPivotBlockHeader().isEmpty()
syncState
.getAccountToRepair()
.ifPresent(address -> snapContext.addInconsistentAccount(Hash.hash(address)));
} else if (fastSyncState.getPivotBlockHeader().isEmpty()
&& protocolContext.getBlockchain().getChainHeadBlockNumber()
!= BlockHeader.GENESIS_BLOCK_NUMBER) {
LOG.info(

@ -41,7 +41,7 @@ public class SnapPersistedContext {
private final GenericKeyValueStorageFacade<BigInteger, AccountRangeDataRequest>
accountRangeToDownload;
private final GenericKeyValueStorageFacade<BigInteger, Bytes> inconsistentAccounts;
private final GenericKeyValueStorageFacade<BigInteger, Bytes> healContext;
public SnapPersistedContext(final StorageProvider storageProvider) {
this.accountRangeToDownload =
@ -61,7 +61,7 @@ public class SnapPersistedContext {
},
storageProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.SNAPSYNC_MISSING_ACCOUNT_RANGE));
this.inconsistentAccounts =
this.healContext =
new GenericKeyValueStorageFacade<>(
BigInteger::toByteArray,
new ValueConvertor<>() {
@ -95,25 +95,25 @@ public class SnapPersistedContext {
public void addInconsistentAccount(final Bytes inconsistentAccount) {
final BigInteger index =
inconsistentAccounts
healContext
.get(SNAP_INCONSISTENT_ACCOUNT_INDEX)
.map(bytes -> new BigInteger(bytes.toArrayUnsafe()).add(BigInteger.ONE))
.orElse(BigInteger.ZERO);
inconsistentAccounts.putAll(
healContext.putAll(
keyValueStorageTransaction -> {
keyValueStorageTransaction.put(SNAP_INCONSISTENT_ACCOUNT_INDEX, index.toByteArray());
keyValueStorageTransaction.put(index.toByteArray(), inconsistentAccount.toArrayUnsafe());
});
}
public List<AccountRangeDataRequest> getPersistedTasks() {
public List<AccountRangeDataRequest> getCurrentAccountRange() {
return accountRangeToDownload
.streamValuesFromKeysThat(bytes -> true)
.collect(Collectors.toList());
}
public HashSet<Bytes> getInconsistentAccounts() {
return inconsistentAccounts
return healContext
.streamValuesFromKeysThat(notEqualsTo(SNAP_INCONSISTENT_ACCOUNT_INDEX))
.collect(Collectors.toCollection(HashSet::new));
}
@ -124,12 +124,12 @@ public class SnapPersistedContext {
public void clear() {
accountRangeToDownload.clear();
inconsistentAccounts.clear();
healContext.clear();
}
public void close() throws IOException {
accountRangeToDownload.close();
inconsistentAccounts.close();
healContext.close();
}
private Predicate<byte[]> notEqualsTo(final byte[] name) {

@ -198,6 +198,7 @@ public class SnapWorldDownloadState extends WorldDownloadState<SnapDataRequest>
public synchronized void reloadHeal() {
worldStateStorage.clearFlatDatabase();
worldStateStorage.clearTrieLog();
pendingTrieNodeRequests.clear();
pendingCodeRequests.clear();
snapSyncState.setHealStatus(false);

@ -147,21 +147,24 @@ public class SnapWorldStateDownloader implements WorldStateDownloader {
final Map<Bytes32, Bytes32> ranges = RangeManager.generateAllRanges(16);
snapsyncMetricsManager.initRange(ranges);
final List<AccountRangeDataRequest> persistedTasks = snapContext.getPersistedTasks();
final List<AccountRangeDataRequest> currentAccountRange =
snapContext.getCurrentAccountRange();
final HashSet<Bytes> inconsistentAccounts = snapContext.getInconsistentAccounts();
if (!persistedTasks.isEmpty()) { // continue to download worldstate ranges
if (!currentAccountRange.isEmpty()) { // continue to download worldstate ranges
newDownloadState.setInconsistentAccounts(inconsistentAccounts);
snapContext
.getPersistedTasks()
.getCurrentAccountRange()
.forEach(
snapDataRequest -> {
snapsyncMetricsManager.notifyStateDownloaded(
snapDataRequest.getStartKeyHash(), snapDataRequest.getEndKeyHash());
newDownloadState.enqueueRequest(snapDataRequest);
});
} else if (!inconsistentAccounts.isEmpty()) { // restart only the heal step
} else if (!snapContext.getInconsistentAccounts().isEmpty()) { // restart only the heal step
snapSyncState.setHealStatus(true);
worldStateStorage.clearFlatDatabase();
worldStateStorage.clearTrieLog();
newDownloadState.setInconsistentAccounts(inconsistentAccounts);
newDownloadState.enqueueRequest(
SnapDataRequest.createAccountTrieNodeDataRequest(

@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthPeers;
import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.Checkpoint;
import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadStatus;
import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason;
import org.hyperledger.besu.plugin.data.Address;
import org.hyperledger.besu.plugin.data.SyncStatus;
import org.hyperledger.besu.plugin.services.BesuEvents.InitialSyncCompletionListener;
import org.hyperledger.besu.plugin.services.BesuEvents.SyncStatusListener;
@ -58,6 +59,10 @@ public class SyncState {
private final Optional<Checkpoint> checkpoint;
private volatile boolean isInitialSyncPhaseDone;
private volatile boolean isResyncNeeded;
private Optional<Address> maybeAccountToRepair;
public SyncState(final Blockchain blockchain, final EthPeers ethPeers) {
this(blockchain, ethPeers, false, Optional.empty());
}
@ -311,13 +316,30 @@ public class SyncState {
return checkpoint;
}
public boolean isInitialSyncPhaseDone() {
return isInitialSyncPhaseDone;
}
public void markInitialSyncPhaseAsDone() {
isInitialSyncPhaseDone = true;
isResyncNeeded = false;
completionListenerSubscribers.forEach(InitialSyncCompletionListener::onInitialSyncCompleted);
}
public boolean isInitialSyncPhaseDone() {
return isInitialSyncPhaseDone;
public boolean isResyncNeeded() {
return isResyncNeeded;
}
public void markResyncNeeded() {
isResyncNeeded = true;
}
public Optional<Address> getAccountToRepair() {
return maybeAccountToRepair;
}
public void markAccountToRepair(final Optional<Address> address) {
maybeAccountToRepair = address;
}
public void markInitialSyncRestart() {

@ -194,7 +194,7 @@ public class DownloadHeaderSequenceTask extends AbstractRetryingPeerTask<List<Bl
child =
(headerIndex == segmentLength - 1) ? referenceHeader : headers[headerIndex + 1];
}
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(child);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(child.getNumber());
final BadBlockManager badBlockManager = protocolSpec.getBadBlocksManager();
if (!validateHeader(child, header)) {
@ -253,7 +253,7 @@ public class DownloadHeaderSequenceTask extends AbstractRetryingPeerTask<List<Bl
return false;
}
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(child);
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(child.getNumber());
final BlockHeaderValidator blockHeaderValidator = protocolSpec.getBlockHeaderValidator();
return blockHeaderValidator.validateHeader(
child, header, protocolContext, validationPolicy.getValidationModeForNextBlock());

@ -195,7 +195,8 @@ public class PersistBlockTask extends AbstractEthTask<Block> {
@Override
protected void executeTask() {
try {
final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
final ProtocolSpec protocolSpec =
protocolSchedule.getByBlockNumber(block.getHeader().getNumber());
final BlockImporter blockImporter = protocolSpec.getBlockImporter();
debugLambda(LOG, "Running import task for block {}", block::toLogString);
blockImportResult = blockImporter.importBlock(protocolContext, block, validateHeaders);

@ -1,172 +0,0 @@
/*
* Copyright contributors to Hyperledger Besu
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.eth.sync.worldstate;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.eth.EthProtocolVersion;
import org.hyperledger.besu.ethereum.eth.manager.EthContext;
import org.hyperledger.besu.ethereum.eth.manager.snap.RetryingGetTrieNodeFromPeerTask;
import org.hyperledger.besu.ethereum.eth.manager.task.EthTask;
import org.hyperledger.besu.ethereum.eth.manager.task.RetryingGetNodeDataFromPeerTask;
import org.hyperledger.besu.ethereum.p2p.network.ProtocolManager;
import org.hyperledger.besu.ethereum.trie.CompactEncoding;
import org.hyperledger.besu.ethereum.worldstate.PeerTrieNodeFinder;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** This class is used to retrieve missing nodes in the trie by querying the peers */
public class WorldStatePeerTrieNodeFinder implements PeerTrieNodeFinder {
private static final Logger LOG = LoggerFactory.getLogger(WorldStatePeerTrieNodeFinder.class);
private final Cache<Bytes32, Bytes> foundNodes =
CacheBuilder.newBuilder().maximumSize(10_000).expireAfterWrite(5, TimeUnit.MINUTES).build();
private static final long TIMEOUT_SECONDS = 1;
final ProtocolManager protocolManager;
final EthContext ethContext;
final Blockchain blockchain;
final MetricsSystem metricsSystem;
public WorldStatePeerTrieNodeFinder(
final EthContext ethContext,
final ProtocolManager protocolManager,
final Blockchain blockchain,
final MetricsSystem metricsSystem) {
this.ethContext = ethContext;
this.protocolManager = protocolManager;
this.blockchain = blockchain;
this.metricsSystem = metricsSystem;
}
@Override
public Optional<Bytes> getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) {
Optional<Bytes> cachedValue = Optional.ofNullable(foundNodes.getIfPresent(nodeHash));
if (cachedValue.isPresent()) {
return cachedValue;
}
final Optional<Bytes> response =
findByGetNodeData(Hash.wrap(nodeHash))
.or(() -> findByGetTrieNodeData(Hash.wrap(nodeHash), Optional.empty(), location));
response.ifPresent(
bytes -> {
LOG.debug(
"Fixed missing account state trie node for location {} and hash {}",
location,
nodeHash);
foundNodes.put(nodeHash, bytes);
});
return response;
}
@Override
public Optional<Bytes> getAccountStorageTrieNode(
final Hash accountHash, final Bytes location, final Bytes32 nodeHash) {
Optional<Bytes> cachedValue = Optional.ofNullable(foundNodes.getIfPresent(nodeHash));
if (cachedValue.isPresent()) {
return cachedValue;
}
final Optional<Bytes> response =
// Call findByGetNodeData only if protocol version < eth 67
(protocolManager.getHighestProtocolVersion() < EthProtocolVersion.V67
? findByGetNodeData(Hash.wrap(nodeHash))
: Optional.<Bytes>empty())
.or(
() ->
findByGetTrieNodeData(Hash.wrap(nodeHash), Optional.of(accountHash), location));
response.ifPresent(
bytes -> {
LOG.debug(
"Fixed missing storage state trie node for location {} and hash {}",
location,
nodeHash);
foundNodes.put(nodeHash, bytes);
});
return response;
}
@VisibleForTesting
public Optional<Bytes> findByGetNodeData(final Hash nodeHash) {
if (protocolManager.getHighestProtocolVersion() >= EthProtocolVersion.V67) {
return Optional.empty();
}
final BlockHeader chainHead = blockchain.getChainHeadHeader();
final RetryingGetNodeDataFromPeerTask retryingGetNodeDataFromPeerTask =
RetryingGetNodeDataFromPeerTask.forHashes(
ethContext, List.of(nodeHash), chainHead.getNumber(), metricsSystem);
try {
final Map<Hash, Bytes> response =
retryingGetNodeDataFromPeerTask.run().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
if (response.containsKey(nodeHash)) {
LOG.debug("Found node {} with getNodeData request", nodeHash);
return Optional.of(response.get(nodeHash));
} else {
LOG.debug("Found invalid node {} with getNodeData request", nodeHash);
}
} catch (InterruptedException | ExecutionException | TimeoutException e) {
LOG.debug("Error when trying to find node {} with getNodeData request", nodeHash);
}
return Optional.empty();
}
@VisibleForTesting
public Optional<Bytes> findByGetTrieNodeData(
final Hash nodeHash, final Optional<Bytes32> accountHash, final Bytes location) {
final BlockHeader chainHead = blockchain.getChainHeadHeader();
final Map<Bytes, List<Bytes>> request = new HashMap<>();
if (accountHash.isPresent()) {
request.put(accountHash.get(), List.of(CompactEncoding.encode(location)));
} else {
request.put(CompactEncoding.encode(location), new ArrayList<>());
}
final Bytes path = CompactEncoding.encode(location);
final EthTask<Map<Bytes, Bytes>> getTrieNodeFromPeerTask =
RetryingGetTrieNodeFromPeerTask.forTrieNodes(ethContext, request, chainHead, metricsSystem);
try {
final Map<Bytes, Bytes> response =
getTrieNodeFromPeerTask.run().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
final Bytes nodeValue =
response.get(Bytes.concatenate(accountHash.map(Bytes::wrap).orElse(Bytes.EMPTY), path));
if (nodeValue != null && Hash.hash(nodeValue).equals(nodeHash)) {
LOG.debug("Found node {} with getTrieNode request", nodeHash);
return Optional.of(nodeValue);
} else {
LOG.debug("Found invalid node {} with getTrieNode request", nodeHash);
}
} catch (InterruptedException | ExecutionException | TimeoutException e) {
LOG.debug("Error when trying to find node {} with getTrieNode request", nodeHash);
}
return Optional.empty();
}
}

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save