From e24bfdebeacf48cade8c735a8ba9c3919a2da517 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Mon, 10 Apr 2023 16:53:47 -0700 Subject: [PATCH] merge of 23.1.2 into release-23.1.x (#5323) Signed-off-by: garyschulte --- .circleci/config.yml | 2 +- .github/issue_template.md | 1 + .github/pull_request_template.md | 15 +- .github/workflows/checks.yml | 5 +- .github/workflows/codeql.yml | 3 +- .github/workflows/dco-merge-group.yml | 10 + .github/workflows/dco.yml | 3 +- .../workflows/gradle-wrapper-validation.yml | 4 +- .github/workflows/pr-checklist-on-open.yml | 20 + .github/workflows/release.yml | 2 +- .github/workflows/repolinter.yml | 3 +- .github/workflows/sonarcloud.yml | 42 ++ CHANGELOG.md | 41 +- ISSUE_TEMPLATE.md | 0 PULL_REQUEST_TEMPLATE.md | 39 -- .../plugins/TestRpcEndpointServicePlugin.java | 13 +- .../CreateAccountAcceptanceTest.java | 6 +- .../GossipTransactionAcceptanceTest.java | 6 +- .../besu/tests/acceptance/LoggingTest.java | 38 ++ .../acceptance/MetricsAcceptanceTest.java | 6 +- .../OpenTelemetryAcceptanceTest.java | 10 +- .../RpcApisTogglesAcceptanceTest.java | 6 +- .../besu/tests/acceptance/RunHelpTest.java | 2 +- .../jsonrpc/AbstractJsonRpcTest.java | 5 +- .../DeploySmartContractAcceptanceTest.java | 6 +- .../DeployTransactionAcceptanceTest.java | 6 +- .../jsonrpc/EthGetWorkAcceptanceTest.java | 10 +- .../jsonrpc/EthSendRawTransactionTest.java | 6 +- .../jsonrpc/EventEmitterAcceptanceTest.java | 6 +- ...onRpcHttpAuthenticationAcceptanceTest.java | 6 +- ...WebsocketAuthenticationAcceptanceTest.java | 6 +- .../jsonrpc/NetServicesAcceptanceTest.java | 10 +- .../jsonrpc/RevertReasonAcceptanceTest.java | 6 +- .../jsonrpc/Web3Sha3AcceptanceTest.java | 6 +- .../admin/AdminAddPeerAcceptanceTest.java | 10 +- .../plugins/BadCLIOptionsPluginTest.java | 14 +- .../plugins/BesuEventsPluginTest.java | 8 +- .../plugins/PermissioningPluginTest.java | 6 +- .../plugins/PicoCLIOptionsPluginTest.java | 10 +- .../plugins/RpcEndpointServicePluginTest.java | 6 +- .../privacy/PrivacyGroupAcceptanceTest.java | 5 +- .../privacy/contracts/PrivacyGroupTest.java | 6 +- .../privacy/contracts/PrivacyProxyTest.java | 6 +- .../NewPendingTransactionAcceptanceTest.java | 10 +- besu/build.gradle | 1 + .../main/java/org/hyperledger/besu/Besu.java | 16 +- .../org/hyperledger/besu/RunnerBuilder.java | 19 +- .../org/hyperledger/besu/cli/BesuCommand.java | 20 +- .../error/BesuParameterExceptionHandler.java | 10 +- .../options/stable/LoggingLevelOption.java | 9 +- .../cli/subcommands/RetestethSubCommand.java | 11 +- .../controller/BesuControllerBuilder.java | 12 +- .../besu/ForkIdsNetworkConfigTest.java | 5 +- .../hyperledger/besu/RunnerBuilderTest.java | 8 +- .../hyperledger/besu/cli/BesuCommandTest.java | 5 +- .../stable/LoggingLevelOptionTest.java | 4 +- .../controller/BesuControllerBuilderTest.java | 27 +- .../MergeBesuControllerBuilderTest.java | 2 + .../config/DefaultDiscoveryConfiguration.java | 4 - .../besu/services/BesuEventsImplTest.java | 3 +- ...rmissioningConfigurationValidatorTest.java | 10 +- .../PrivateStorageMigrationServiceTest.java | 8 +- .../besu/util/StringUtilsTest.java | 8 +- ...eckpoint_total_difficulty_same_as_TTD.json | 19 - .../invalid_post_merge_merge_at_genesis.json | 19 - ...valid_post_merge_near_head_checkpoint.json | 19 - ...alid_pos_checkpoint_pos_TD_equals_TTD.json | 19 - ...valid_post_merge_near_head_checkpoint.json | 19 - build.gradle | 15 +- config/src/main/resources/mainnet.json | 5 +- .../CombinedProtocolScheduleFactory.java | 6 +- .../merge/TransitionProtocolSchedule.java | 17 + .../merge/TransitionProtocolScheduleTest.java | 62 ++ .../merge/blockcreation/MergeReorgTest.java | 4 +- .../api/graphql/GraphQLDataFetchers.java | 8 +- .../pojoadapter/BlockAdapterBase.java | 4 +- .../internal/pojoadapter/LogAdapter.java | 2 +- .../pojoadapter/PendingStateAdapter.java | 2 +- .../pojoadapter/TransactionAdapter.java | 12 +- .../internal/methods/AbstractTraceByHash.java | 36 +- .../internal/methods/AdminChangeLogLevel.java | 14 +- .../internal/methods/DebugAccountAt.java | 87 +-- .../internal/methods/DebugAccountRange.java | 20 +- .../DebugStandardTraceBlockToFile.java | 25 +- .../internal/methods/DebugStorageRangeAt.java | 45 +- .../internal/methods/DebugTraceBlock.java | 17 +- .../methods/DebugTraceBlockByHash.java | 22 +- .../methods/DebugTraceBlockByNumber.java | 16 +- .../methods/DebugTraceTransaction.java | 11 +- .../jsonrpc/internal/methods/EthGetProof.java | 9 +- .../jsonrpc/internal/methods/TraceBlock.java | 31 +- .../internal/methods/TraceCallMany.java | 14 +- .../methods/TraceReplayBlockTransactions.java | 16 +- .../internal/processor/BlockReplay.java | 73 +-- .../internal/processor/BlockTracer.java | 24 +- .../jsonrpc/internal/processor/Tracer.java | 101 ++++ .../internal/processor/TransactionTracer.java | 45 +- .../jsonrpc/methods/DebugJsonRpcMethods.java | 7 +- .../jsonrpc/methods/TraceJsonRpcMethods.java | 5 +- .../ethereum/api/query/BlockchainQueries.java | 28 +- .../besu/ethereum/api/util/TraceUtil.java | 100 --- .../methods/AdminChangeLogLevelTest.java | 4 +- .../internal/methods/DebugAccountAtTest.java | 42 +- .../DebugStandardTraceBadBlockToFileTest.java | 22 +- .../DebugStandardTraceBlockToFileTest.java | 13 +- .../methods/DebugStorageRangeAtTest.java | 27 +- .../methods/DebugTraceBlockByHashTest.java | 33 +- .../methods/DebugTraceBlockByNumberTest.java | 33 +- .../internal/methods/DebugTraceBlockTest.java | 39 +- .../methods/DebugTraceTransactionTest.java | 49 +- .../internal/methods/EthGetProofTest.java | 55 +- .../processor/TransactionTracerTest.java | 28 +- .../blockcreation/AbstractBlockCreator.java | 11 +- .../worldstate/PrunerIntegrationTest.java | 7 +- .../besu/ethereum/MainnetBlockValidator.java | 13 +- .../besu/ethereum/bonsai/BonsaiAccount.java | 19 +- .../bonsai/BonsaiInMemoryWorldState.java | 182 ------ ...nsaiInMemoryWorldStateKeyValueStorage.java | 71 --- .../bonsai/BonsaiLayeredWorldState.java | 328 ---------- .../bonsai/BonsaiPersistedWorldState.java | 478 --------------- .../bonsai/BonsaiSnapshotWorldState.java | 105 ---- ...nsaiSnapshotWorldStateKeyValueStorage.java | 307 ---------- .../besu/ethereum/bonsai/BonsaiValue.java | 10 +- ...ive.java => BonsaiWorldStateProvider.java} | 217 +++---- .../bonsai/LayeredTrieLogManager.java | 115 ---- .../bonsai/SnapshotTrieLogManager.java | 157 ----- .../bonsai/cache/CachedBonsaiWorldView.java | 76 +++ .../{ => cache}/CachedMerkleTrieLoader.java | 5 +- .../cache/CachedWorldStorageManager.java | 183 ++++++ ...nsaiSnapshotWorldStateKeyValueStorage.java | 217 +++++++ .../BonsaiWorldStateKeyValueStorage.java | 83 ++- .../storage/BonsaiWorldStateLayerStorage.java | 56 ++ .../{ => trielog}/AbstractTrieLogManager.java | 86 +-- .../bonsai/{ => trielog}/TrieLogLayer.java | 11 +- .../bonsai/{ => trielog}/TrieLogManager.java | 31 +- .../bonsai/worldview/BonsaiWorldState.java | 570 ++++++++++++++++++ .../BonsaiWorldStateUpdateAccumulator.java} | 106 ++-- .../{ => worldview}/BonsaiWorldView.java | 16 +- .../besu/ethereum/core/MutableWorldState.java | 18 +- .../core/SnapshotMutableWorldState.java | 22 - .../mainnet/AbstractBlockProcessor.java | 16 +- .../AbstractProtocolScheduleBuilder.java | 33 +- .../mainnet/DefaultTimestampSchedule.java | 53 +- .../mainnet/HeaderBasedProtocolSchedule.java | 6 + .../mainnet/MutableProtocolSchedule.java | 36 +- .../mainnet/ProtocolScheduleBuilder.java | 23 +- .../ScheduleBasedBlockHeaderFunctions.java | 8 +- .../mainnet/ScheduledProtocolSpec.java | 23 +- .../mainnet/TimestampScheduleBuilder.java | 12 +- .../keyvalue/KeyValueStorageProvider.java | 2 +- .../transaction/TransactionSimulator.java | 12 +- .../worldstate/DefaultMutableWorldState.java | 5 - .../worldstate/DefaultWorldStateArchive.java | 9 +- .../worldstate/WorldStateArchive.java | 5 +- .../core/InMemoryKeyValueStorageProvider.java | 8 +- .../besu/ethereum/core/TrieGenerator.java | 2 +- .../BlockImportExceptionHandlingTest.java | 18 +- .../ethereum/MainnetBlockValidatorTest.java | 30 +- .../bonsai/AbstractIsolationTests.java | 13 +- .../bonsai/BonsaiInMemoryIsolationTests.java | 104 ---- .../bonsai/BonsaiSnapshotIsolationTests.java | 182 +----- .../BonsaiSnapshotWorldStateArchiveTest.java | 172 ------ .../bonsai/BonsaiWorldStateArchiveTest.java | 147 ++--- .../BonsaiWorldStateKeyValueStorageTest.java | 3 +- .../bonsai/CachedMerkleTrieLoaderTest.java | 2 + .../bonsai/LayeredWorldStateTests.java | 131 ---- .../besu/ethereum/bonsai/LogRollingTests.java | 49 +- .../besu/ethereum/bonsai/RollingImport.java | 19 +- .../mainnet/ProtocolScheduleBuilderTest.java | 22 +- .../mainnet/TimestampScheduleBuilderTest.java | 21 +- .../mainnet/WithdrawalsProcessorTest.java | 5 +- .../transaction/TransactionSimulatorTest.java | 173 +++--- .../ethereum/vm/AbstractRetryingTest.java | 6 +- .../worldstate/MarkSweepPrunerTest.java | 7 +- .../eth/sync/DefaultSynchronizer.java | 19 +- .../eth/sync/backwardsync/BackwardChain.java | 103 +++- ...orithm.java => BackwardSyncAlgorithm.java} | 11 +- .../backwardsync/BackwardSyncContext.java | 4 +- .../sync/backwardsync/BackwardSyncStep.java | 2 +- .../sync/checkpointsync/CheckpointSource.java | 10 +- ...CheckpointSyncDownloadPipelineFactory.java | 8 +- .../eth/sync/fastsync/FastSyncDownloader.java | 2 +- .../AccountTrieNodeDataRequest.java | 2 +- .../worldstate/FastDownloaderFactory.java | 2 +- .../StorageTrieNodeDataRequest.java | 2 +- .../request/AccountTrieNodeDataRequest.java | 2 +- .../request/StorageTrieNodeDataRequest.java | 2 +- .../eth/sync/tasks/CompleteBlocksTask.java | 32 +- .../eth/transactions/TransactionPool.java | 16 +- .../transactions/TransactionPoolFactory.java | 17 +- .../eth/manager/EthProtocolManagerTest.java | 17 +- .../backwardsync/BackwardSyncAlgSpec.java | 6 +- .../backwardsync/BackwardSyncContextTest.java | 8 +- .../backwardsync/BackwardSyncStepTest.java | 10 +- .../backwardsync/ForwardSyncStepTest.java | 10 +- .../InMemoryBackwardChainTest.java | 9 +- .../checkpointsync/CheckPointSourceTest.java | 9 +- .../FastWorldDownloadStateTest.java | 2 +- .../snapsync/SnapWorldDownloadStateTest.java | 2 +- .../sync/tasks/CompleteBlocksTaskTest.java | 124 ++++ .../TransactionPoolFactoryTest.java | 65 ++ ethereum/evmtool/build.gradle | 2 - .../besu/evmtool/B11rSubCommand.java | 2 + .../besu/evmtool/CodeValidateSubCommand.java | 2 + .../besu/evmtool/EvmToolCommand.java | 9 +- .../besu/evmtool/StateTestSubCommand.java | 11 +- .../besu/evmtool/T8nSubCommand.java | 141 +++-- .../t8n/berlin-negative-cli-reward.json | 96 +++ .../besu/evmtool/t8n/london-hex.json | 5 +- .../besu/evmtool/t8n/shanghai-invalidTx.json | 87 +++ .../src/reference-test/external-resources | 2 +- .../vm/BlockchainReferenceTestTools.java | 2 +- .../NoRewardProtocolScheduleWrapper.java | 13 + .../ethereum/retesteth/RetestethContext.java | 6 +- gradle.properties | 4 +- gradle/verification-metadata.xml | 52 ++ gradle/versions.gradle | 1 + gradlew | 2 +- plugin-api/build.gradle | 3 +- .../storage/SnappedKeyValueStorage.java | 7 - .../RocksDBColumnarKeyValueSnapshot.java | 36 +- .../segmented/RocksDBSnapshotTransaction.java | 1 - .../unsegmented/RocksDBKeyValueStorage.java | 4 + .../kvstore/InMemoryKeyValueStorage.java | 42 +- .../kvstore/LayeredKeyValueStorage.java | 158 +++++ .../besu/util/Log4j2ConfiguratorUtil.java | 28 +- .../besu/util/LogConfigurator.java | 54 ++ 227 files changed, 3988 insertions(+), 4071 deletions(-) create mode 100644 .github/workflows/dco-merge-group.yml create mode 100644 .github/workflows/pr-checklist-on-open.yml create mode 100644 .github/workflows/sonarcloud.yml delete mode 100644 ISSUE_TEMPLATE.md delete mode 100644 PULL_REQUEST_TEMPLATE.md create mode 100644 acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/LoggingTest.java create mode 100644 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/Tracer.java delete mode 100644 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/util/TraceUtil.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldStateKeyValueStorage.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldState.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateKeyValueStorage.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/{BonsaiWorldStateArchive.java => BonsaiWorldStateProvider.java} (65%) delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/LayeredTrieLogManager.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/SnapshotTrieLogManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedBonsaiWorldView.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/{ => cache}/CachedMerkleTrieLoader.java (96%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/{ => storage}/BonsaiWorldStateKeyValueStorage.java (90%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/{ => trielog}/AbstractTrieLogManager.java (56%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/{ => trielog}/TrieLogLayer.java (98%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/{ => trielog}/TrieLogManager.java (55%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/{BonsaiWorldStateUpdater.java => worldview/BonsaiWorldStateUpdateAccumulator.java} (91%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/{ => worldview}/BonsaiWorldView.java (80%) delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/SnapshotMutableWorldState.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryIsolationTests.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateArchiveTest.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LayeredWorldStateTests.java rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/{BackwardsSyncAlgorithm.java => BackwardSyncAlgorithm.java} (94%) create mode 100644 ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-negative-cli-reward.json create mode 100644 ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-invalidTx.json create mode 100644 services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java create mode 100644 util/src/main/java/org/hyperledger/besu/util/LogConfigurator.java diff --git a/.circleci/config.yml b/.circleci/config.yml index f9fbade930..6a48f22929 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -219,7 +219,7 @@ jobs: - capture_test_results acceptanceTests: - parallelism: 6 + parallelism: 4 executor: xl_machine_executor steps: - prepare diff --git a/.github/issue_template.md b/.github/issue_template.md index 13e7ecf306..e6a7a5accb 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -32,6 +32,7 @@ As an [Actor], I want [feature] so that [why]. * Virtual Machine software & version: [`vmware -v`] * Docker Version: [`docker version`] * Cloud VM, type, size: [Amazon Web Services I3-large] +* Consensus Client & Version if using Proof of Stake: [e.g. Teku, Lighthouse, Prysm, Nimbus, Lodestar] ### Smart contract information (If you're reporting an issue arising from deploying or calling a smart contract, please supply related information) * Solidity version [`solc --version`] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 74a1cdcc57..11bac0fb92 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,17 +5,4 @@ ## Fixed Issue(s) - - -## Documentation - -- [ ] I thought about documentation and added the `doc-change-required` label to this PR if - [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation). - -## Acceptance Tests (Non Mainnet) - -- [ ] I have considered running `./gradlew acceptanceTestNonMainnet` locally if my PR affects non-mainnet modules. - -## Changelog - -- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog). \ No newline at end of file + \ No newline at end of file diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d9c7dafba0..4da01d6f68 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -4,10 +4,11 @@ on: branches: [ main ] pull_request: workflow_dispatch: + merge_group: jobs: spotless: - runs-on: ubuntu-latest + runs-on: [besu,Linux,self-hosted,X64] if: ${{ github.actor != 'dependabot[bot]' }} steps: - name: Checkout Repo @@ -21,7 +22,7 @@ jobs: - name: spotless run: ./gradlew --no-daemon --parallel clean spotlessCheck javadoc_17: - runs-on: ubuntu-latest + runs-on: [besu,Linux,self-hosted,X64] if: ${{ github.actor != 'dependabot[bot]' }} steps: - name: Checkout Repo diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 246c118d67..1116f52026 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,10 +21,11 @@ on: - '**/*.md' - '**/*.properties' - '**/*.txt' + merge_group: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: [besu,Linux,self-hosted,X64] permissions: actions: read contents: read diff --git a/.github/workflows/dco-merge-group.yml b/.github/workflows/dco-merge-group.yml new file mode 100644 index 0000000000..02b9c6bd13 --- /dev/null +++ b/.github/workflows/dco-merge-group.yml @@ -0,0 +1,10 @@ +name: dco +on: + merge_group: + +jobs: + dco: + runs-on: [besu,Linux,self-hosted] + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" \ No newline at end of file diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index c50fa6219a..49de075a1c 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -5,9 +5,10 @@ on: jobs: dco: - runs-on: ubuntu-latest + runs-on: [besu,Linux,self-hosted] if: ${{ github.actor != 'dependabot[bot]' }} steps: + - run: echo "This DCO job runs on pull_request event and workflow_dispatch" - name: Get PR Commits id: 'get-pr-commits' uses: tim-actions/get-pr-commits@v1.2.0 diff --git a/.github/workflows/gradle-wrapper-validation.yml b/.github/workflows/gradle-wrapper-validation.yml index b397f1c6b4..bfc098ab66 100644 --- a/.github/workflows/gradle-wrapper-validation.yml +++ b/.github/workflows/gradle-wrapper-validation.yml @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 name: "Validate Gradle Wrapper" -on: [push, pull_request] +on: [push, pull_request, merge_group] jobs: validation: name: "Gradle Wrapper Validation" - runs-on: ubuntu-latest + runs-on: [besu,Linux,self-hosted] steps: - uses: actions/checkout@v2 - uses: gradle/wrapper-validation-action@v1 diff --git a/.github/workflows/pr-checklist-on-open.yml b/.github/workflows/pr-checklist-on-open.yml new file mode 100644 index 0000000000..678c11705c --- /dev/null +++ b/.github/workflows/pr-checklist-on-open.yml @@ -0,0 +1,20 @@ +name: "comment on pr with checklist" +on: + pull_request_target: + types: [ opened ] + branches: [ main ] +jobs: + checklist: + name: "add checklist as a comment on newly opened PRs" + runs-on: [besu,Linux,self-hosted] + steps: + - uses: actions/github-script@v5 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I have considered running `./gradlew acceptanceTestNonMainnet` locally if my PR affects non-mainnet modules.\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).' + }) \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 35b74bbfd0..54bcc7fc42 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,7 @@ on: types: released jobs: dockerPromoteX64: - runs-on: ubuntu-latest + runs-on: [besu,Linux,self-hosted] steps: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 diff --git a/.github/workflows/repolinter.yml b/.github/workflows/repolinter.yml index feca74edb8..94b1b3a2a8 100644 --- a/.github/workflows/repolinter.yml +++ b/.github/workflows/repolinter.yml @@ -12,10 +12,11 @@ on: branches: - master - main + merge_group: jobs: build: - runs-on: ubuntu-latest + runs-on: [besu,Linux,self-hosted,X64] container: ghcr.io/todogroup/repolinter:v0.10.1 steps: - name: Checkout Code diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml new file mode 100644 index 0000000000..a94fe9404a --- /dev/null +++ b/.github/workflows/sonarcloud.yml @@ -0,0 +1,42 @@ + +name: SonarCloud analysis + +on: + schedule: + # * is a special character in YAML so you have to quote this string + # expression evaluates to midnight on Tuesdays UTC + - cron: '0 0 * * 2' + +permissions: + pull-requests: read # allows SonarCloud to decorate PRs with analysis results + +jobs: + Analysis: + runs-on: ubuntu-latest + + steps: + - name: checkout + uses: actions/checkout@v3 + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - name: Cache SonarCloud packages + uses: actions/cache@v1 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + - name: Cache Gradle packages + uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: ${{ runner.os }}-gradle + - name: Build and analyze + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: ./gradlew build sonarqube --info + diff --git a/CHANGELOG.md b/CHANGELOG.md index e0ab6d3357..6fe79e363a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 23.1.2 +This update is a mainnet-compatible Shanghai/Capella upgrade and is recommended for all Mainnet users. + +### Breaking Changes + +### Additions and Improvements +- Schedule Shanghai (Shapella) fork for Mainnet [#5230](https://github.com/hyperledger/besu/pull/5230) +- Increase default from 1000 to 5000 for `--rpc-max-logs-range` [#5209](https://github.com/hyperledger/besu/pull/5209) +- Bonsai-safe refactor [#5123](https://github.com/hyperledger/besu/pull/5123) +- Safe tracing [#5197](https://github.com/hyperledger/besu/pull/5197) + +### Bug Fixes +- Persist backward sync status to support resuming across restarts [#5182](https://github.com/hyperledger/besu/pull/5182) + +### Download Links +https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.1.2/besu-23.1.2.tar.gz / sha256: tba +https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.1.2/besu-23.1.2.zip / sha256: tba + ## 23.1.1 This update is required for the Goerli Shanghai/Capella upgrade and recommended for all Mainnet users. If you use Besu on Goerli, update to 23.1.1. If you previously used 23.1.1-RC1, update to test 23.1.1 on Goerli. @@ -16,8 +34,13 @@ This update is required for the Goerli Shanghai/Capella upgrade and recommended - Goerli configs for shapella [#5151](https://github.com/hyperledger/besu/pull/5151) ### Bug Fixes -- Fix engine_getPayloadV2 block value calculation https://github.com/hyperledger/besu/issues/5040 -- Moves check for init code length before balance check https://github.com/hyperledger/besu/pull/5077 +- Fix engine_getPayloadV2 block value calculation [#5040](https://github.com/hyperledger/besu/issues/5040) +- Moves check for init code length before balance check [#5077](https://github.com/hyperledger/besu/pull/5077) +- Address concurrency problems with eth_call [#5179](https://github.com/hyperledger/besu/pull/5179) + +### Download Links +https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.1.1/besu-23.1.1.tar.gz / sha256: 11c3e5cdbc06df16a690e7ee9f98eefa46848f9fa280824b6e4c896d88f6b975 +https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.1.1/besu-23.1.1.zip / sha256: afcf852f193adb8e82d187aa4f02e4669f12cc680270624d37101b94cf37adec ## 23.1.1-RC1 ### Sepolia Shanghai Release aka Sepolia Shapella aka Shapolia @@ -33,15 +56,15 @@ Sepolia Shanghai hardfork scheduled for: **Tue Feb 28 2023 04:04:48 UTC** This release has everything from [23.1.0](https://github.com/hyperledger/besu/releases/tag/23.1.0) and in addition the following: ### Additions and Improvements -* Add support for Shanghai in Sepolia https://github.com/hyperledger/besu/pull/5088 -* Add implementation for engine_getPayloadBodiesByRangeV1 and engine_getPayloadBodiesByHashV1 https://github.com/hyperledger/besu/pull/4980 -* If a PoS block creation repetition takes less than a configurable duration, then waits before next repetition https://github.com/hyperledger/besu/pull/5048 -* Allow other users to read the /opt/besu dir when using docker https://github.com/hyperledger/besu/pull/5092 -* Invalid params - add some error detail #5066 +- Add support for Shanghai in Sepolia https://github.com/hyperledger/besu/pull/5088 +- Add implementation for engine_getPayloadBodiesByRangeV1 and engine_getPayloadBodiesByHashV1 https://github.com/hyperledger/besu/pull/4980 +- If a PoS block creation repetition takes less than a configurable duration, then waits before next repetition https://github.com/hyperledger/besu/pull/5048 +- Allow other users to read the /opt/besu dir when using docker https://github.com/hyperledger/besu/pull/5092 +- Invalid params - add some error detail #5066 ### Bug fixes -* Fix engine_getPayloadV2 block value calculation https://github.com/hyperledger/besu/issues/5040 -* Moves check for init code length before balance check https://github.com/hyperledger/besu/pull/5077 +- Fix engine_getPayloadV2 block value calculation https://github.com/hyperledger/besu/issues/5040 +- Moves check for init code length before balance check https://github.com/hyperledger/besu/pull/5077 ### Download Links https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.1.1-RC1/besu-23.1.1-RC1.tar.gz / sha256: 82cff41f3eace02006b0e670605848e0e77e045892f8fa9aad66cbd84a88221e diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index edfac2bb41..0000000000 --- a/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,39 +0,0 @@ -## Status -**READY/IN DEVELOPMENT/HOLD** - -## Description -A few sentences describing the overall goals of the pull request's commits. - -## Related PRs -List related PRs against other branches: - -branch | PR ------- | ------ -other_pr_production | [link]() -other_pr_master | [link]() - - -## Todos -- [ ] Tests -- [ ] Documentation - - -## Deploy Notes -Notes regarding deployment the contained body of work. These should note any -db migrations, etc. - -## Steps to Test or Reproduce -Outline the steps to test or reproduce the PR here. - -```sh -git pull --prune -git checkout -bundle; script/server -``` - -1. - -## Impacted Areas in Application -List general components of the application that this PR will affect: - -* \ No newline at end of file diff --git a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestRpcEndpointServicePlugin.java b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestRpcEndpointServicePlugin.java index 60cfe955a8..fee061f544 100644 --- a/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestRpcEndpointServicePlugin.java +++ b/acceptance-tests/test-plugins/src/main/java/org/hyperledger/besu/tests/acceptance/plugins/TestRpcEndpointServicePlugin.java @@ -28,6 +28,8 @@ import com.google.auto.service.AutoService; @AutoService(BesuPlugin.class) public class TestRpcEndpointServicePlugin implements BesuPlugin { + private static final String NAMESPACE_ENABLED = "tests"; + private static final String NAMESPACE_NOT_ENABLED = "notEnabled"; private final AtomicReference stringStorage = new AtomicReference<>("InitialValue"); private final AtomicReference arrayStorage = new AtomicReference<>(); @@ -54,13 +56,14 @@ public class TestRpcEndpointServicePlugin implements BesuPlugin { .getService(RpcEndpointService.class) .ifPresent( rpcEndpointService -> { - rpcEndpointService.registerRPCEndpoint("tests", "getValue", this::getValue); - rpcEndpointService.registerRPCEndpoint("tests", "setValue", this::setValue); + rpcEndpointService.registerRPCEndpoint(NAMESPACE_ENABLED, "getValue", this::getValue); + rpcEndpointService.registerRPCEndpoint(NAMESPACE_ENABLED, "setValue", this::setValue); rpcEndpointService.registerRPCEndpoint( - "tests", "replaceValueList", this::replaceValueList); + NAMESPACE_ENABLED, "replaceValueList", this::replaceValueList); rpcEndpointService.registerRPCEndpoint( - "tests", "throwException", this::throwException); - rpcEndpointService.registerRPCEndpoint("notEnabled", "getValue", this::getValue); + NAMESPACE_ENABLED, "throwException", this::throwException); + rpcEndpointService.registerRPCEndpoint( + NAMESPACE_NOT_ENABLED, "getValue", this::getValue); }); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/CreateAccountAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/CreateAccountAcceptanceTest.java index caa9683795..c0af67c494 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/CreateAccountAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/CreateAccountAcceptanceTest.java @@ -19,14 +19,14 @@ import org.hyperledger.besu.tests.acceptance.dsl.account.Account; import org.hyperledger.besu.tests.acceptance.dsl.blockchain.Amount; import org.hyperledger.besu.tests.acceptance.dsl.node.Node; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class CreateAccountAcceptanceTest extends AcceptanceTestBase { private Node minerNode; - @Before + @BeforeEach public void setUp() throws Exception { minerNode = besu.createMinerNode("minerNode"); cluster.start(minerNode, besu.createArchiveNode("archiveNode")); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/GossipTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/GossipTransactionAcceptanceTest.java index 8dcd4175dc..1e6ca01891 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/GossipTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/GossipTransactionAcceptanceTest.java @@ -21,14 +21,14 @@ import org.hyperledger.besu.tests.acceptance.dsl.blockchain.Amount; import org.hyperledger.besu.tests.acceptance.dsl.node.Node; import org.hyperledger.besu.tests.acceptance.dsl.transaction.account.TransferTransaction; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class GossipTransactionAcceptanceTest extends AcceptanceTestBase { private Node archiveNode1; - @Before + @BeforeEach public void setUp() throws Exception { archiveNode1 = besu.createArchiveNode("archiveNode1"); cluster.start(archiveNode1, besu.createArchiveNode("archiveNode2")); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/LoggingTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/LoggingTest.java new file mode 100644 index 0000000000..e0276842f0 --- /dev/null +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/LoggingTest.java @@ -0,0 +1,38 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.tests.acceptance; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase; +import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode; + +import java.io.IOException; + +import org.junit.Test; + +public class LoggingTest extends AcceptanceTestBase { + + @Test + public void testDefaultLoggingIsAtLeastInfo() throws IOException { + final BesuNode node = besu.createArchiveNode("logger"); + cluster.startConsoleCapture(); + cluster.runNodeStart(node); + + node.verify(net.awaitPeerCount(0)); + + assertThat(cluster.getConsoleContents()).contains("| INFO |"); + } +} diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/MetricsAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/MetricsAcceptanceTest.java index 4fe6ffa3b7..2f7aee17b6 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/MetricsAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/MetricsAcceptanceTest.java @@ -26,15 +26,15 @@ import okhttp3.Call; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class MetricsAcceptanceTest extends AcceptanceTestBase { private BesuNode metricsNode; private OkHttpClient client; - @Before + @BeforeEach public void setUp() throws Exception { metricsNode = besu.create( diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/OpenTelemetryAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/OpenTelemetryAcceptanceTest.java index 8265efe9d7..46c5c9a35b 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/OpenTelemetryAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/OpenTelemetryAcceptanceTest.java @@ -59,9 +59,9 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class OpenTelemetryAcceptanceTest extends AcceptanceTestBase { @@ -125,7 +125,7 @@ public class OpenTelemetryAcceptanceTest extends AcceptanceTestBase { private BesuNode metricsNode; - @Before + @BeforeEach public void setUp() throws Exception { System.setProperty("root.log.level", "DEBUG"); Server server = @@ -163,7 +163,7 @@ public class OpenTelemetryAcceptanceTest extends AcceptanceTestBase { cluster.start(metricsNode); } - @After + @AfterEach public void tearDown() throws Exception { closer.close(); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RpcApisTogglesAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RpcApisTogglesAcceptanceTest.java index 3eb9821f8e..821d440b71 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RpcApisTogglesAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RpcApisTogglesAcceptanceTest.java @@ -22,8 +22,8 @@ import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase; import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode; import org.java_websocket.exceptions.WebsocketNotConnectedException; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class RpcApisTogglesAcceptanceTest extends AcceptanceTestBase { @@ -31,7 +31,7 @@ public class RpcApisTogglesAcceptanceTest extends AcceptanceTestBase { private BesuNode rpcDisabledNode; private BesuNode ethApiDisabledNode; - @Before + @BeforeEach public void before() throws Exception { rpcEnabledNode = besu.createArchiveNode("rpc-enabled"); rpcDisabledNode = besu.createArchiveNodeWithRpcDisabled("rpc-disabled"); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RunHelpTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RunHelpTest.java index 78dc19aef0..dabb277b04 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RunHelpTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RunHelpTest.java @@ -22,7 +22,7 @@ import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode; import java.io.IOException; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class RunHelpTest extends AcceptanceTestBase { diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/AbstractJsonRpcTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/AbstractJsonRpcTest.java index 29f44e0da4..56c43da355 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/AbstractJsonRpcTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/AbstractJsonRpcTest.java @@ -95,7 +95,10 @@ abstract class AbstractJsonRpcTest { final ObjectNode actualBody = JsonUtil.objectNodeFromString(response.body().string()); final ObjectNode expectedBody = JsonUtil.objectNodeFromString(testCase.getResponse().toString()); - assertThat(actualBody.toPrettyString()).isEqualTo(expectedBody.toPrettyString()); + assertThat(actualBody) + .withFailMessage( + "%s\ndid not equal\n %s", actualBody.toPrettyString(), expectedBody.toPrettyString()) + .isEqualTo(expectedBody); } private String getRpcUrl(final String rpcMethod) { diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/DeploySmartContractAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/DeploySmartContractAcceptanceTest.java index 6a06674df4..06bd5cbfda 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/DeploySmartContractAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/DeploySmartContractAcceptanceTest.java @@ -18,14 +18,14 @@ import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase; import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode; import org.hyperledger.besu.tests.web3j.generated.SimpleStorage; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class DeploySmartContractAcceptanceTest extends AcceptanceTestBase { private BesuNode minerNode; - @Before + @BeforeEach public void setUp() throws Exception { minerNode = besu.createMinerNode("miner-node"); cluster.start(minerNode); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/DeployTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/DeployTransactionAcceptanceTest.java index 451418fa5f..909d40beb3 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/DeployTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/DeployTransactionAcceptanceTest.java @@ -19,15 +19,15 @@ import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase; import org.hyperledger.besu.tests.acceptance.dsl.account.Account; import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class DeployTransactionAcceptanceTest extends AcceptanceTestBase { private BesuNode minerNode; private Account recipient; - @Before + @BeforeEach public void setUp() throws Exception { recipient = accounts.createAccount("recipient"); minerNode = besu.createMinerNode("node"); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthGetWorkAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthGetWorkAcceptanceTest.java index 2e9dd20619..c4842348aa 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthGetWorkAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthGetWorkAcceptanceTest.java @@ -17,16 +17,16 @@ package org.hyperledger.besu.tests.acceptance.jsonrpc; import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase; import org.hyperledger.besu.tests.acceptance.dsl.node.Node; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; public class EthGetWorkAcceptanceTest extends AcceptanceTestBase { private Node minerNode; private Node fullNode; - @Before + @BeforeEach public void setUp() throws Exception { minerNode = besu.createMinerNode("node1"); fullNode = besu.createArchiveNode("node2"); @@ -34,7 +34,7 @@ public class EthGetWorkAcceptanceTest extends AcceptanceTestBase { } @Test - @Ignore("Genuinely Flakey") + @Disabled("Genuinely Flakey") public void shouldReturnSuccessResponseWhenMining() { minerNode.verify(eth.getWork()); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionTest.java index c16ce22e49..e614e794c7 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionTest.java @@ -24,8 +24,8 @@ import org.hyperledger.besu.tests.acceptance.dsl.transaction.account.TransferTra import java.math.BigInteger; import java.util.function.UnaryOperator; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class EthSendRawTransactionTest extends AcceptanceTestBase { private static final long CHAIN_ID = 20211; @@ -36,7 +36,7 @@ public class EthSendRawTransactionTest extends AcceptanceTestBase { private Node strictNode; private Node miningNode; - @Before + @BeforeEach public void setUp() throws Exception { sender = accounts.getPrimaryBenefactor(); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EventEmitterAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EventEmitterAcceptanceTest.java index 9644cc2b62..de4bfdb646 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EventEmitterAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EventEmitterAcceptanceTest.java @@ -26,8 +26,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import io.reactivex.Flowable; import io.reactivex.disposables.Disposable; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.web3j.protocol.core.methods.request.EthFilter; import org.web3j.protocol.core.methods.response.TransactionReceipt; @@ -38,7 +38,7 @@ public class EventEmitterAcceptanceTest extends AcceptanceTestBase { private BesuNode node; - @Before + @BeforeEach public void setUp() throws Exception { node = besu.createMinerNode("node1"); cluster.start(node); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/JsonRpcHttpAuthenticationAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/JsonRpcHttpAuthenticationAcceptanceTest.java index 7a612be683..3ef383a526 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/JsonRpcHttpAuthenticationAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/JsonRpcHttpAuthenticationAcceptanceTest.java @@ -25,8 +25,8 @@ import java.net.URISyntaxException; import java.util.Arrays; import java.util.List; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class JsonRpcHttpAuthenticationAcceptanceTest extends AcceptanceTestBase { private Cluster authenticatedCluster; @@ -52,7 +52,7 @@ public class JsonRpcHttpAuthenticationAcceptanceTest extends AcceptanceTestBase private static final List NO_AUTH_API_METHODS = Arrays.asList("net_services"); - @Before + @BeforeEach public void setUp() throws IOException, URISyntaxException { final ClusterConfiguration clusterConfiguration = diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/JsonRpcWebsocketAuthenticationAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/JsonRpcWebsocketAuthenticationAcceptanceTest.java index f5c3d35078..47d236e5b3 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/JsonRpcWebsocketAuthenticationAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/JsonRpcWebsocketAuthenticationAcceptanceTest.java @@ -25,8 +25,8 @@ import java.net.URISyntaxException; import java.util.Arrays; import java.util.List; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class JsonRpcWebsocketAuthenticationAcceptanceTest extends AcceptanceTestBase { private BesuNode nodeUsingAuthFile; @@ -52,7 +52,7 @@ public class JsonRpcWebsocketAuthenticationAcceptanceTest extends AcceptanceTest + "c2lvbnMiOlsibmV0OnBlZXJDb3VudCJdfQ.pWXniN6XQ7G8b1nawy8sviPCMxrfbcI6c7UFzeXm26CMGMUEZxiC" + "JjRntB8ueuZcsxnGlEhCHt-KngpFEmx5TA"; - @Before + @BeforeEach public void setUp() throws IOException, URISyntaxException { final ClusterConfiguration clusterConfiguration = new ClusterConfigurationBuilder().awaitPeerDiscovery(false).build(); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/NetServicesAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/NetServicesAcceptanceTest.java index 218382bcc1..dbff6f151a 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/NetServicesAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/NetServicesAcceptanceTest.java @@ -20,9 +20,9 @@ import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.Cluster; import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.ClusterConfiguration; import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.ClusterConfigurationBuilder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class NetServicesAcceptanceTest extends AcceptanceTestBase { @@ -31,7 +31,7 @@ public class NetServicesAcceptanceTest extends AcceptanceTestBase { private Node nodeA; private Node nodeB; - @Before + @BeforeEach public void setup() { final ClusterConfiguration clusterConfiguration = new ClusterConfigurationBuilder().awaitPeerDiscovery(false).build(); @@ -58,7 +58,7 @@ public class NetServicesAcceptanceTest extends AcceptanceTestBase { nodeB.verify(net.netServicesOnlyJsonRpcEnabled()); } - @After + @AfterEach public void closeDown() throws Exception { noDiscoveryCluster.close(); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/RevertReasonAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/RevertReasonAcceptanceTest.java index bccbe6d9e0..17b82cd20f 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/RevertReasonAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/RevertReasonAcceptanceTest.java @@ -21,15 +21,15 @@ import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase; import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode; import org.hyperledger.besu.tests.web3j.generated.RevertReason; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.web3j.protocol.core.methods.response.EthSendTransaction; public class RevertReasonAcceptanceTest extends AcceptanceTestBase { private BesuNode minerNode; - @Before + @BeforeEach public void setUp() throws Exception { minerNode = besu.createMinerNodeWithRevertReasonEnabled("miner-node-withRevertReason"); cluster.start(minerNode); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/Web3Sha3AcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/Web3Sha3AcceptanceTest.java index 9de4c0bf2b..a925a27661 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/Web3Sha3AcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/Web3Sha3AcceptanceTest.java @@ -17,14 +17,14 @@ package org.hyperledger.besu.tests.acceptance.jsonrpc; import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase; import org.hyperledger.besu.tests.acceptance.dsl.node.Node; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class Web3Sha3AcceptanceTest extends AcceptanceTestBase { private Node node; - @Before + @BeforeEach public void setUp() throws Exception { node = besu.createArchiveNode("node1"); cluster.start(node); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/admin/AdminAddPeerAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/admin/AdminAddPeerAcceptanceTest.java index cce9846384..6c1bd5813b 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/admin/AdminAddPeerAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/admin/AdminAddPeerAcceptanceTest.java @@ -20,9 +20,9 @@ import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.Cluster; import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.ClusterConfiguration; import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.ClusterConfigurationBuilder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class AdminAddPeerAcceptanceTest extends AcceptanceTestBase { private Cluster noDiscoveryCluster; @@ -30,7 +30,7 @@ public class AdminAddPeerAcceptanceTest extends AcceptanceTestBase { private Node nodeA; private Node nodeB; - @Before + @BeforeEach public void setUp() throws Exception { final ClusterConfiguration clusterConfiguration = new ClusterConfigurationBuilder().awaitPeerDiscovery(false).build(); @@ -40,7 +40,7 @@ public class AdminAddPeerAcceptanceTest extends AcceptanceTestBase { noDiscoveryCluster.start(nodeA, nodeB); } - @After + @AfterEach public void tearDown() { noDiscoveryCluster.stop(); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/BadCLIOptionsPluginTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/BadCLIOptionsPluginTest.java index be533b7054..02299f4fdb 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/BadCLIOptionsPluginTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/BadCLIOptionsPluginTest.java @@ -28,15 +28,15 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.awaitility.Awaitility; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; public class BadCLIOptionsPluginTest extends AcceptanceTestBase { private BesuNode node; - @Before + @BeforeEach public void setUp() throws Exception { System.setProperty("TEST_BAD_CLI", "true"); @@ -46,7 +46,7 @@ public class BadCLIOptionsPluginTest extends AcceptanceTestBase { cluster.start(node); } - @After + @AfterEach public void tearDown() { System.setProperty("TEST_BAD_CLI", "false"); } @@ -68,7 +68,7 @@ public class BadCLIOptionsPluginTest extends AcceptanceTestBase { } @Test - @Ignore("No way to do a graceful shutdown of Besu at the moment.") + @Disabled("No way to do a graceful shutdown of Besu at the moment.") public void shouldNotStop() { cluster.stopNode(node); waitForFile(node.homeDirectory().resolve("plugins/pluginLifecycle.stopped")); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/BesuEventsPluginTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/BesuEventsPluginTest.java index 09deffb551..4906193362 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/BesuEventsPluginTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/BesuEventsPluginTest.java @@ -25,14 +25,14 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.awaitility.Awaitility; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class BesuEventsPluginTest extends AcceptanceTestBase { private BesuNode pluginNode; private BesuNode minerNode; - @Before + @BeforeEach public void setUp() throws Exception { minerNode = besu.createMinerNode("minerNode"); pluginNode = @@ -42,7 +42,7 @@ public class BesuEventsPluginTest extends AcceptanceTestBase { } @Test - public void blockIsAnnounded() { + public void blockIsAnnounced() { waitForFile(pluginNode.homeDirectory().resolve("plugins/newBlock.2")); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PermissioningPluginTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PermissioningPluginTest.java index 4bfb857a47..af6a3f7a16 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PermissioningPluginTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PermissioningPluginTest.java @@ -25,8 +25,8 @@ import org.hyperledger.besu.tests.acceptance.dsl.transaction.account.TransferTra import java.util.List; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class PermissioningPluginTest extends AcceptanceTestBase { private BesuNode minerNode; @@ -35,7 +35,7 @@ public class PermissioningPluginTest extends AcceptanceTestBase { private BesuNode bobNode; private BesuNode charlieNode; - @Before + @BeforeEach public void setUp() throws Exception { final BesuNodeConfigurationBuilder builder = new BesuNodeConfigurationBuilder() diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PicoCLIOptionsPluginTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PicoCLIOptionsPluginTest.java index d1a1154de6..b3a7a1ed55 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PicoCLIOptionsPluginTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PicoCLIOptionsPluginTest.java @@ -28,9 +28,9 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.awaitility.Awaitility; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; public class PicoCLIOptionsPluginTest extends AcceptanceTestBase { private BesuNode node; @@ -38,7 +38,7 @@ public class PicoCLIOptionsPluginTest extends AcceptanceTestBase { // context: https://en.wikipedia.org/wiki/The_Magic_Words_are_Squeamish_Ossifrage private static final String MAGIC_WORDS = "Squemish Ossifrage"; - @Before + @BeforeEach public void setUp() throws Exception { node = besu.createPluginsNode( @@ -70,7 +70,7 @@ public class PicoCLIOptionsPluginTest extends AcceptanceTestBase { } @Test - @Ignore("No way to do a graceful shutdown of Besu at the moment.") + @Disabled("No way to do a graceful shutdown of Besu at the moment.") public void shouldStop() { cluster.stopNode(node); waitForFile(node.homeDirectory().resolve("plugins/pluginLifecycle.stopped")); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/RpcEndpointServicePluginTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/RpcEndpointServicePluginTest.java index 47081c7454..09da91663a 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/RpcEndpointServicePluginTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/RpcEndpointServicePluginTest.java @@ -30,8 +30,8 @@ import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class RpcEndpointServicePluginTest extends AcceptanceTestBase { @@ -40,7 +40,7 @@ public class RpcEndpointServicePluginTest extends AcceptanceTestBase { private OkHttpClient client; protected static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - @Before + @BeforeEach public void setUp() throws Exception { node = besu.createPluginsNode("node1", List.of("testPlugins"), List.of("--rpc-http-api=TESTS")); cluster.start(node); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/PrivacyGroupAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/PrivacyGroupAcceptanceTest.java index ab1bd91d0e..a1739b1fc8 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/PrivacyGroupAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/PrivacyGroupAcceptanceTest.java @@ -24,7 +24,7 @@ import org.hyperledger.besu.tests.acceptance.dsl.privacy.PrivacyAcceptanceTestBa import org.hyperledger.besu.tests.acceptance.dsl.privacy.PrivacyNode; import org.hyperledger.besu.tests.acceptance.dsl.privacy.account.PrivacyAccountResolver; import org.hyperledger.besu.tests.web3j.generated.EventEmitter; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import org.hyperledger.enclave.testutil.EnclaveEncryptorType; import org.hyperledger.enclave.testutil.EnclaveType; @@ -34,7 +34,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Optional; -import org.apache.logging.log4j.Level; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -99,7 +98,7 @@ public class PrivacyGroupAcceptanceTest extends PrivacyAcceptanceTestBase { @Test public void nodeCanCreatePrivacyGroup() { - Log4j2ConfiguratorUtil.setLevel("", Level.DEBUG); + LogConfigurator.setLevel("", "DEBUG"); final String privacyGroupId = alice.execute( privacyTransactions.createPrivacyGroup( diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/contracts/PrivacyGroupTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/contracts/PrivacyGroupTest.java index 2e5d6ec6de..b805fa43c4 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/contracts/PrivacyGroupTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/contracts/PrivacyGroupTest.java @@ -25,8 +25,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.web3j.protocol.core.RemoteFunctionCall; import org.web3j.protocol.core.methods.response.TransactionReceipt; import org.web3j.protocol.exceptions.TransactionException; @@ -55,7 +55,7 @@ public class PrivacyGroupTest extends AcceptanceTestBase { private BesuNode minerNode; - @Before + @BeforeEach public void setUp() throws Exception { minerNode = besu.createMinerNode("node"); cluster.start(minerNode); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/contracts/PrivacyProxyTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/contracts/PrivacyProxyTest.java index 1c6763f4c9..931770ee32 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/contracts/PrivacyProxyTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/contracts/PrivacyProxyTest.java @@ -27,8 +27,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.web3j.crypto.Credentials; import org.web3j.protocol.Web3j; import org.web3j.protocol.exceptions.TransactionException; @@ -55,7 +55,7 @@ public class PrivacyProxyTest extends AcceptanceTestBase { private DefaultFlexiblePrivacyGroupManagementContract defaultFlexiblePrivacyGroupManagementContract; - @Before + @BeforeEach public void setUp() throws Exception { minerNode = besu.createMinerNode("node"); cluster.start(minerNode); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java index 95f4df08ec..4ae94924b2 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java @@ -22,9 +22,9 @@ import org.hyperledger.besu.tests.acceptance.dsl.pubsub.Subscription; import org.hyperledger.besu.tests.acceptance.dsl.pubsub.WebSocket; import io.vertx.core.Vertx; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class NewPendingTransactionAcceptanceTest extends AcceptanceTestBase { @@ -35,7 +35,7 @@ public class NewPendingTransactionAcceptanceTest extends AcceptanceTestBase { private BesuNode minerNode; private BesuNode archiveNode; - @Before + @BeforeEach public void setUp() throws Exception { vertx = Vertx.vertx(); minerNode = besu.createMinerNode("miner-node1"); @@ -46,7 +46,7 @@ public class NewPendingTransactionAcceptanceTest extends AcceptanceTestBase { archiveWebSocket = new WebSocket(vertx, archiveNode.getConfiguration()); } - @After + @AfterEach public void tearDown() { vertx.close(); } diff --git a/besu/build.gradle b/besu/build.gradle index f52ccf6813..dd0765a777 100644 --- a/besu/build.gradle +++ b/besu/build.gradle @@ -99,6 +99,7 @@ dependencies { testImplementation 'org.awaitility:awaitility' testImplementation 'org.junit.jupiter:junit-jupiter' testImplementation 'org.mockito:mockito-core' + testImplementation 'org.mockito:mockito-junit-jupiter' testImplementation 'org.testcontainers:testcontainers' testImplementation 'tech.pegasys.discovery:discovery' diff --git a/besu/src/main/java/org/hyperledger/besu/Besu.java b/besu/src/main/java/org/hyperledger/besu/Besu.java index 993f7b3a93..35682cd8ae 100644 --- a/besu/src/main/java/org/hyperledger/besu/Besu.java +++ b/besu/src/main/java/org/hyperledger/besu/Besu.java @@ -62,7 +62,13 @@ public final class Besu { } private static Logger setupLogging() { - InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE); + try { + InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE); + } catch (Throwable t) { + System.out.printf( + "Could not set netty log4j logger factory: %s - %s%n", + t.getClass().getSimpleName(), t.getMessage()); + } try { System.setProperty( "vertx.logger-delegate-factory-class-name", @@ -70,10 +76,10 @@ public final class Besu { System.setProperty( "log4j.configurationFactory", BesuLoggingConfigurationFactory.class.getName()); System.setProperty("log4j.skipJansi", String.valueOf(false)); - } catch (SecurityException e) { - System.out.println( - "Could not set logging system property as the security manager prevented it:" - + e.getMessage()); + } catch (Throwable t) { + System.out.printf( + "Could not set logging system property: %s - %s%n", + t.getClass().getSimpleName(), t.getMessage()); } final Logger logger = LoggerFactory.getLogger(Besu.class); Thread.setDefaultUncaughtExceptionHandler(slf4jExceptionHandler(logger)); diff --git a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java index a9a8f6fea5..df94ac1bc7 100644 --- a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java @@ -67,6 +67,7 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.api.query.PrivacyQueries; import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Synchronizer; @@ -762,11 +763,7 @@ public class RunnerBuilder { .getBlockchain() .observeBlockAdded( blockAddedEvent -> { - if (protocolSchedule - .streamMilestoneBlocks() - .anyMatch( - blockNumber -> - blockNumber == blockAddedEvent.getBlock().getHeader().getNumber())) { + if (protocolSchedule.isOnMilestoneBoundary(blockAddedEvent.getBlock().getHeader())) { network.updateNodeRecord(); } }); @@ -796,7 +793,8 @@ public class RunnerBuilder { .build(); vertx.deployVerticle(filterManager); - createPrivateTransactionObserver(filterManager, privacyParameters); + createPrivateTransactionObserver( + filterManager, privacyParameters, context.getBlockchain().getGenesisBlockHeader()); final P2PNetwork peerNetwork = networkRunner.getNetwork(); @@ -1032,7 +1030,8 @@ public class RunnerBuilder { DefaultAuthenticationService.create(vertx, webSocketConfiguration), metricsSystem)); - createPrivateTransactionObserver(subscriptionManager, privacyParameters); + createPrivateTransactionObserver( + subscriptionManager, privacyParameters, context.getBlockchain().getGenesisBlockHeader()); } final Optional metricsService = @@ -1344,7 +1343,8 @@ public class RunnerBuilder { private void createPrivateTransactionObserver( final PrivateTransactionObserver privateTransactionObserver, - final PrivacyParameters privacyParameters) { + final PrivacyParameters privacyParameters, + final BlockHeader genesisBlockHeader) { // register privateTransactionObserver as observer of events fired by the flexible precompile. if (privacyParameters.isFlexiblePrivacyGroupsEnabled() && privacyParameters.isMultiTenancyEnabled()) { @@ -1352,9 +1352,10 @@ public class RunnerBuilder { (FlexiblePrivacyPrecompiledContract) besuController .getProtocolSchedule() - .getByBlockNumber(1) + .getByBlockHeader(genesisBlockHeader) .getPrecompileContractRegistry() .get(FLEXIBLE_PRIVACY); + flexiblePrivacyPrecompiledContract.addPrivateTransactionObserver(privateTransactionObserver); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 6dd4649d8a..4fad078baa 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -186,7 +186,7 @@ import org.hyperledger.besu.services.SecurityModuleServiceImpl; import org.hyperledger.besu.services.StorageServiceImpl; import org.hyperledger.besu.services.kvstore.InMemoryStoragePlugin; import org.hyperledger.besu.util.InvalidConfigurationException; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import org.hyperledger.besu.util.NetworkUtility; import org.hyperledger.besu.util.PermissioningConfigurationValidator; import org.hyperledger.besu.util.number.Fraction; @@ -240,7 +240,6 @@ import net.consensys.quorum.mainnet.launcher.LauncherManager; import net.consensys.quorum.mainnet.launcher.config.ImmutableLauncherConfig; import net.consensys.quorum.mainnet.launcher.exception.LauncherException; import net.consensys.quorum.mainnet.launcher.util.ParseArgsHelper; -import org.apache.logging.log4j.Level; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; @@ -1313,7 +1312,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { names = {"--rpc-max-logs-range"}, description = "Specifies the maximum number of blocks to retrieve logs from via RPC. Must be >=0. 0 specifies no limit (default: ${DEFAULT-VALUE})") - private final Long rpcMaxLogsRange = 1000L; + private final Long rpcMaxLogsRange = 5000L; @Mixin private P2PTLSConfigOptions p2pTLSConfigOptions; @@ -1440,7 +1439,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { * @param args arguments to Besu command * @return success or failure exit code. */ - @VisibleForTesting public int parse( final IExecutionStrategy resultHandler, final BesuParameterExceptionHandler parameterExceptionHandler, @@ -1553,7 +1551,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private void registerConverters() { commandLine.registerConverter(Address.class, Address::fromHexStringStrict); commandLine.registerConverter(Bytes.class, Bytes::fromHexString); - commandLine.registerConverter(Level.class, Level::valueOf); commandLine.registerConverter(MetricsProtocol.class, MetricsProtocol::fromString); commandLine.registerConverter(UInt256.class, (arg) -> UInt256.valueOf(new BigInteger(arg))); commandLine.registerConverter(Wei.class, (arg) -> Wei.of(Long.parseUnsignedLong(arg))); @@ -1796,14 +1793,14 @@ public class BesuCommand implements DefaultCommandValues, Runnable { */ public void configureLogging(final boolean announce) { // To change the configuration if color was enabled/disabled - Log4j2ConfiguratorUtil.reconfigure(); + LogConfigurator.reconfigure(); // set log level per CLI flags - final Level logLevel = loggingLevelOption.getLogLevel(); + final String logLevel = loggingLevelOption.getLogLevel(); if (logLevel != null) { if (announce) { - System.out.println("Setting logging level to " + logLevel.name()); + System.out.println("Setting logging level to " + logLevel); } - Log4j2ConfiguratorUtil.setAllLevels("", logLevel); + LogConfigurator.setLevel("", logLevel); } } @@ -3109,7 +3106,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { try { besuPluginContext.stopPlugins(); runner.close(); - Log4j2ConfiguratorUtil.shutdown(); + LogConfigurator.shutdown(); } catch (final Exception e) { logger.error("Failed to stop Besu"); } @@ -3306,7 +3303,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { * * @return instance of BesuParameterExceptionHandler */ - @VisibleForTesting public BesuParameterExceptionHandler parameterExceptionHandler() { return new BesuParameterExceptionHandler(this::getLogLevel); } @@ -3454,7 +3450,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { } @VisibleForTesting - Level getLogLevel() { + String getLogLevel() { return loggingLevelOption.getLogLevel(); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/error/BesuParameterExceptionHandler.java b/besu/src/main/java/org/hyperledger/besu/cli/error/BesuParameterExceptionHandler.java index 54fa2504b9..6b7d8d6d6c 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/error/BesuParameterExceptionHandler.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/error/BesuParameterExceptionHandler.java @@ -17,21 +17,20 @@ package org.hyperledger.besu.cli.error; import java.io.PrintWriter; import java.util.function.Supplier; -import org.apache.logging.log4j.Level; import picocli.CommandLine; import picocli.CommandLine.Model.CommandSpec; /** The custom parameter exception handler for Besu PicoCLI. */ public class BesuParameterExceptionHandler implements CommandLine.IParameterExceptionHandler { - private final Supplier levelSupplier; + private final Supplier levelSupplier; /** * Instantiates a new Besu parameter exception handler. * * @param levelSupplier the logging level supplier */ - public BesuParameterExceptionHandler(final Supplier levelSupplier) { + public BesuParameterExceptionHandler(final Supplier levelSupplier) { this.levelSupplier = levelSupplier; } @@ -39,8 +38,9 @@ public class BesuParameterExceptionHandler implements CommandLine.IParameterExce public int handleParseException(final CommandLine.ParameterException ex, final String[] args) { final CommandLine cmd = ex.getCommandLine(); final PrintWriter err = cmd.getErr(); - final Level logLevel = levelSupplier.get(); - if (logLevel != null && Level.DEBUG.isMoreSpecificThan(logLevel)) { + final String logLevel = levelSupplier.get(); + if (logLevel != null + && (logLevel.equals("DEBUG") || logLevel.equals("TRACE") || logLevel.equals("ALL"))) { ex.printStackTrace(err); } else { err.println(ex.getMessage()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java index 4eba5b74c4..ad119ea61a 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.cli.options.stable; import java.util.Set; -import org.apache.logging.log4j.Level; import picocli.CommandLine; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Spec; @@ -38,7 +37,7 @@ public class LoggingLevelOption { /** The Picocli CommandSpec. Visible for testing. Injected by Picocli framework at runtime. */ @Spec CommandSpec spec; - private Level logLevel; + private String logLevel; /** * Sets log level. @@ -52,9 +51,9 @@ public class LoggingLevelOption { public void setLogLevel(final String logLevel) { if ("FATAL".equalsIgnoreCase(logLevel)) { System.out.println("FATAL level is deprecated"); - this.logLevel = Level.ERROR; + this.logLevel = "ERROR"; } else if (ACCEPTED_VALUES.contains(logLevel.toUpperCase())) { - this.logLevel = Level.getLevel(logLevel.toUpperCase()); + this.logLevel = logLevel.toUpperCase(); } else { throw new CommandLine.ParameterException( spec.commandLine(), "Unknown logging value: " + logLevel); @@ -66,7 +65,7 @@ public class LoggingLevelOption { * * @return the log level */ - public Level getLogLevel() { + public String getLogLevel() { return logLevel; } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/RetestethSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/RetestethSubCommand.java index 9fea1eab17..36fe7ffab4 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/RetestethSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/RetestethSubCommand.java @@ -24,12 +24,11 @@ import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration; import org.hyperledger.besu.ethereum.retesteth.RetestethConfiguration; import org.hyperledger.besu.ethereum.retesteth.RetestethService; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import java.net.InetAddress; import java.nio.file.Path; -import org.apache.logging.log4j.Level; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; @@ -106,10 +105,10 @@ public class RetestethSubCommand implements Runnable { private void prepareLogging() { // set log level per CLI flags - final Level logLevel = loggingLevelOption.getLogLevel(); + final String logLevel = loggingLevelOption.getLogLevel(); if (logLevel != null) { - System.out.println("Setting logging level to " + logLevel.name()); - Log4j2ConfiguratorUtil.setAllLevels("", logLevel); + System.out.println("Setting logging level to " + logLevel); + LogConfigurator.setLevel("", logLevel); } } @@ -132,7 +131,7 @@ public class RetestethSubCommand implements Runnable { () -> { try { retestethService.close(); - Log4j2ConfiguratorUtil.shutdown(); + LogConfigurator.shutdown(); } catch (final Exception e) { LOG.error("Failed to stop Besu Retesteth"); } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index db700e2126..f3fe3c4647 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -30,9 +30,9 @@ import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods; import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.bonsai.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.BlockchainStorage; import org.hyperledger.besu.ethereum.chain.ChainDataPruner; @@ -658,6 +658,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides createAdditionalJsonRpcMethodFactory(protocolContext); final List closeables = new ArrayList<>(); + closeables.add(protocolContext.getWorldStateArchive()); closeables.add(storageProvider); if (privacyParameters.getPrivateStorageProvider() != null) { closeables.add(privacyParameters.getPrivateStorageProvider()); @@ -934,17 +935,16 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides new SnapProtocolManager(peerValidators, ethPeers, snapMessages, worldStateArchive)); } - private WorldStateArchive createWorldStateArchive( + WorldStateArchive createWorldStateArchive( final WorldStateStorage worldStateStorage, final Blockchain blockchain, final CachedMerkleTrieLoader cachedMerkleTrieLoader) { switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI: - return new BonsaiWorldStateArchive( + return new BonsaiWorldStateProvider( (BonsaiWorldStateKeyValueStorage) worldStateStorage, blockchain, Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), - dataStorageConfiguration.useBonsaiSnapshots(), cachedMerkleTrieLoader); case FOREST: diff --git a/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java b/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java index 5483a6173d..5133299651 100644 --- a/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java +++ b/besu/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java @@ -118,8 +118,9 @@ public class ForkIdsNetworkConfigTest { new ForkId(Bytes.ofUnsignedInt(0xeb440f6L), 12965000L), new ForkId(Bytes.ofUnsignedInt(0xb715077dL), 13773000L), new ForkId(Bytes.ofUnsignedInt(0x20c327fcL), 15050000L), - new ForkId(Bytes.ofUnsignedInt(0xf0afd0e3L), 0L), - new ForkId(Bytes.ofUnsignedInt(0xf0afd0e3L), 0L)) + new ForkId(Bytes.ofUnsignedInt(0xf0afd0e3L), 1681338455L), + new ForkId(Bytes.ofUnsignedInt(0xdce96c2dL), 0L), + new ForkId(Bytes.ofUnsignedInt(0xdce96c2dL), 0L)) }, new Object[] { NetworkName.MORDOR, diff --git a/besu/src/test/java/org/hyperledger/besu/RunnerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/RunnerBuilderTest.java index 77e349819f..9a95f9af04 100644 --- a/besu/src/test/java/org/hyperledger/besu/RunnerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/RunnerBuilderTest.java @@ -49,6 +49,7 @@ import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; @@ -63,6 +64,7 @@ import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.nat.NatMethod; @@ -74,7 +76,6 @@ import org.hyperledger.besu.services.RpcEndpointServiceImpl; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.Collections; -import java.util.stream.Stream; import io.vertx.core.Vertx; import org.apache.tuweni.bytes.Bytes; @@ -97,6 +98,7 @@ public final class RunnerBuilderTest { @Mock BesuController besuController; @Mock ProtocolSchedule protocolSchedule; @Mock ProtocolContext protocolContext; + @Mock WorldStateArchive worldstateArchive; @Mock Vertx vertx; private NodeKey nodeKey; @@ -124,7 +126,7 @@ public final class RunnerBuilderTest { final Block block = mock(Block.class); when(blockchain.getGenesisBlock()).thenReturn(block); when(block.getHash()).thenReturn(Hash.ZERO); - + when(protocolContext.getWorldStateArchive()).thenReturn(worldstateArchive); when(besuController.getProtocolManager()).thenReturn(ethProtocolManager); when(besuController.getSubProtocolConfiguration()).thenReturn(subProtocolConfiguration); when(besuController.getProtocolContext()).thenReturn(protocolContext); @@ -214,7 +216,7 @@ public final class RunnerBuilderTest { .build(); runner.startEthereumMainLoop(); - when(protocolSchedule.streamMilestoneBlocks()).thenAnswer(__ -> Stream.of(1L, 2L)); + when(protocolSchedule.isOnMilestoneBoundary(any(BlockHeader.class))).thenReturn(true); for (int i = 0; i < 2; ++i) { final Block block = diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index b82b7ae29e..2e5169cdff 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -123,7 +123,6 @@ import com.google.common.io.Resources; import io.vertx.core.json.JsonObject; import org.apache.commons.io.FileUtils; import org.apache.commons.text.StringEscapeUtils; -import org.apache.logging.log4j.Level; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.toml.Toml; import org.apache.tuweni.toml.TomlParseResult; @@ -258,7 +257,7 @@ public class BesuCommandTest extends CommandTestAbstract { verify(mockRunnerBuilder).metricsConfiguration(eq(DEFAULT_METRICS_CONFIGURATION)); verify(mockRunnerBuilder).ethNetworkConfig(ethNetworkArg.capture()); verify(mockRunnerBuilder).autoLogBloomCaching(eq(true)); - verify(mockRunnerBuilder).rpcMaxLogsRange(eq(1000L)); + verify(mockRunnerBuilder).rpcMaxLogsRange(eq(5000L)); verify(mockRunnerBuilder).build(); verify(mockControllerBuilderFactory) @@ -4979,7 +4978,7 @@ public class BesuCommandTest extends CommandTestAbstract { public void logLevelIsSetByLoggingOption() { final TestBesuCommand command = parseCommand("--logging", "WARN"); - assertThat(command.getLogLevel()).isEqualTo(Level.WARN); + assertThat(command.getLogLevel()).isEqualTo("WARN"); } @Test diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOptionTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOptionTest.java index f63136b2cb..4d0a67e75c 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOptionTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOptionTest.java @@ -39,7 +39,7 @@ public class LoggingLevelOptionTest { @Test public void fatalLevelEqualsToError() { levelOption.setLogLevel("fatal"); - assertThat(levelOption.getLogLevel()).isEqualTo(Level.ERROR); + assertThat(levelOption.getLogLevel()).isEqualTo("ERROR"); } @Test @@ -49,7 +49,7 @@ public class LoggingLevelOptionTest { .forEach( level -> { levelOption.setLogLevel(level.name()); - assertThat(levelOption.getLogLevel()).isEqualTo(level); + assertThat(levelOption.getLogLevel()).isEqualTo(level.name()); }); } diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index e320ca2688..0b396b5b04 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -16,8 +16,10 @@ package org.hyperledger.besu.controller; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -29,7 +31,10 @@ import org.hyperledger.besu.config.Keccak256ConfigOptions; import org.hyperledger.besu.crypto.NodeKey; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.GasLimitCalculator; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; @@ -43,11 +48,11 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; @@ -61,6 +66,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; +import org.mockito.Answers; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; @@ -85,6 +91,7 @@ public class BesuControllerBuilderTest { @Mock StorageProvider storageProvider; @Mock GasLimitCalculator gasLimitCalculator; @Mock WorldStateStorage worldStateStorage; + @Mock WorldStateArchive worldStateArchive; @Mock BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorage; @Mock WorldStatePreimageStorage worldStatePreimageStorage; @@ -132,14 +139,7 @@ public class BesuControllerBuilderTest { when(worldStatePreimageStorage.updater()) .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class)); - BonsaiWorldStateKeyValueStorage.BonsaiUpdater bonsaiUpdater = - mock(BonsaiWorldStateKeyValueStorage.BonsaiUpdater.class); - when(bonsaiUpdater.getTrieLogStorageTransaction()) - .thenReturn(mock(KeyValueStorageTransaction.class)); - when(bonsaiUpdater.getTrieBranchStorageTransaction()) - .thenReturn(mock(KeyValueStorageTransaction.class)); - when(bonsaiWorldStateStorage.updater()).thenReturn(bonsaiUpdater); - besuControllerBuilder = visitWithMockConfigs(new MainnetBesuControllerBuilder()); + besuControllerBuilder = spy(visitWithMockConfigs(new MainnetBesuControllerBuilder())); } BesuControllerBuilder visitWithMockConfigs(final BesuControllerBuilder builder) { @@ -162,6 +162,13 @@ public class BesuControllerBuilderTest { @Test public void shouldDisablePruningIfBonsaiIsEnabled() { + BonsaiWorldState mockWorldState = mock(BonsaiWorldState.class, Answers.RETURNS_DEEP_STUBS); + doReturn(worldStateArchive) + .when(besuControllerBuilder) + .createWorldStateArchive( + any(WorldStateStorage.class), any(Blockchain.class), any(CachedMerkleTrieLoader.class)); + doReturn(mockWorldState).when(worldStateArchive).getMutable(); + when(storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI)) .thenReturn(bonsaiWorldStateStorage); besuControllerBuilder diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index 5a29193e4d..e631e0b57f 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -121,6 +121,8 @@ public class MergeBesuControllerBuilderTest { .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new MainnetBlockHeaderFunctions())); + when(storageProvider.getStorageBySegmentIdentifier(any())) + .thenReturn(new InMemoryKeyValueStorage()); when(synchronizerConfiguration.getDownloaderParallelism()).thenReturn(1); when(synchronizerConfiguration.getTransactionsParallelism()).thenReturn(1); when(synchronizerConfiguration.getComputationParallelism()).thenReturn(1); diff --git a/besu/src/test/java/org/hyperledger/besu/ethereum/p2p/config/DefaultDiscoveryConfiguration.java b/besu/src/test/java/org/hyperledger/besu/ethereum/p2p/config/DefaultDiscoveryConfiguration.java index 113bdabe0c..066f179668 100644 --- a/besu/src/test/java/org/hyperledger/besu/ethereum/p2p/config/DefaultDiscoveryConfiguration.java +++ b/besu/src/test/java/org/hyperledger/besu/ethereum/p2p/config/DefaultDiscoveryConfiguration.java @@ -36,10 +36,6 @@ public class DefaultDiscoveryConfiguration { // Ethereum Foundation Bootnodes "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", // Singapore AWS "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", // Virginia AWS - "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", // Australia Azure - "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", // Brazil Azure - "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", // South Korea Azure - "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", // West US Azure "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", // Helsinki Hetzner "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", // Falkenstein Hetzner diff --git a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java index 59a14b9384..2c7127c2b1 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java @@ -131,8 +131,7 @@ public class BesuEventsImplTest { .thenReturn(ValidationResult.valid()); when(mockTransactionValidator.validateForSender(any(), any(), any())) .thenReturn(ValidationResult.valid()); - when(mockWorldState.copy()).thenReturn(mockWorldState); - when(mockWorldStateArchive.getMutable(any(), any(), anyBoolean())) + when(mockWorldStateArchive.getMutable(any(), anyBoolean())) .thenReturn(Optional.of(mockWorldState)); blockBroadcaster = new BlockBroadcaster(mockEthContext); diff --git a/besu/src/test/java/org/hyperledger/besu/util/LocalPermissioningConfigurationValidatorTest.java b/besu/src/test/java/org/hyperledger/besu/util/LocalPermissioningConfigurationValidatorTest.java index bbd1a05e92..d4e916d390 100644 --- a/besu/src/test/java/org/hyperledger/besu/util/LocalPermissioningConfigurationValidatorTest.java +++ b/besu/src/test/java/org/hyperledger/besu/util/LocalPermissioningConfigurationValidatorTest.java @@ -34,11 +34,11 @@ import java.util.List; import com.google.common.collect.Lists; import com.google.common.io.Resources; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class LocalPermissioningConfigurationValidatorTest { static final String PERMISSIONING_CONFIG_SEPOLIA_BOOTNODES = @@ -50,7 +50,7 @@ public class LocalPermissioningConfigurationValidatorTest { "/permissioning_config_unknown_hostname.toml"; @Test - public void spoliaWithNodesAllowlistOptionWhichDoesIncludeRopstenBootnodesMustNotError() + public void sepoliaWithNodesAllowlistOptionWhichDoesIncludeRopstenBootnodesMustNotError() throws Exception { EthNetworkConfig ethNetworkConfig = EthNetworkConfig.getNetworkConfig(NetworkName.SEPOLIA); diff --git a/besu/src/test/java/org/hyperledger/besu/util/PrivateStorageMigrationServiceTest.java b/besu/src/test/java/org/hyperledger/besu/util/PrivateStorageMigrationServiceTest.java index fdccf5b874..b54d8ffdfa 100644 --- a/besu/src/test/java/org/hyperledger/besu/util/PrivateStorageMigrationServiceTest.java +++ b/besu/src/test/java/org/hyperledger/besu/util/PrivateStorageMigrationServiceTest.java @@ -31,12 +31,12 @@ import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMig import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigrationException; import org.hyperledger.besu.ethereum.privacy.storage.migration.PrivateStorageMigrationService; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class PrivateStorageMigrationServiceTest { @Mock private PrivateStateStorage privateStateStorage; diff --git a/besu/src/test/java/org/hyperledger/besu/util/StringUtilsTest.java b/besu/src/test/java/org/hyperledger/besu/util/StringUtilsTest.java index 3fd0fdc085..1c3e6f2c95 100644 --- a/besu/src/test/java/org/hyperledger/besu/util/StringUtilsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/util/StringUtilsTest.java @@ -22,11 +22,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class StringUtilsTest { @Test diff --git a/besu/src/test/resources/invalid_post_merge_checkpoint_total_difficulty_same_as_TTD.json b/besu/src/test/resources/invalid_post_merge_checkpoint_total_difficulty_same_as_TTD.json index 116063bcba..8910740006 100644 --- a/besu/src/test/resources/invalid_post_merge_checkpoint_total_difficulty_same_as_TTD.json +++ b/besu/src/test/resources/invalid_post_merge_checkpoint_total_difficulty_same_as_TTD.json @@ -16,25 +16,6 @@ "terminalTotalDifficulty": 58750000000000000000000, "ethash": { }, - "discovery": { - "dns": "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.mainnet.ethdisco.net", - "bootnodes": [ - "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", - "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", - "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", - "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", - "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", - "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", - "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", - "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", - "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", - "enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303", - "enode://78de8a0916848093c73790ead81d1928bec737d565119932b98c6b100d944b7a95e94f847f689fc723399d2e31129d182f7ef3863f2b4c820abbf3ab2722344d@191.235.84.50:30303", - "enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303", - "enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303", - "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303" - ] - }, "checkpoint": { "hash": "0x186642d6084eb7799cb01c7eb69c1a7f3b2c32cd141e62d309f30081a1ea3c91", "number": 16015954, diff --git a/besu/src/test/resources/invalid_post_merge_merge_at_genesis.json b/besu/src/test/resources/invalid_post_merge_merge_at_genesis.json index 10e70cf443..8869186410 100644 --- a/besu/src/test/resources/invalid_post_merge_merge_at_genesis.json +++ b/besu/src/test/resources/invalid_post_merge_merge_at_genesis.json @@ -16,25 +16,6 @@ "terminalTotalDifficulty": 0, "ethash": { }, - "discovery": { - "dns": "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.mainnet.ethdisco.net", - "bootnodes": [ - "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", - "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", - "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", - "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", - "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", - "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", - "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", - "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", - "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", - "enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303", - "enode://78de8a0916848093c73790ead81d1928bec737d565119932b98c6b100d944b7a95e94f847f689fc723399d2e31129d182f7ef3863f2b4c820abbf3ab2722344d@191.235.84.50:30303", - "enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303", - "enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303", - "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303" - ] - }, "checkpoint": { "hash": "0x186642d6084eb7799cb01c7eb69c1a7f3b2c32cd141e62d309f30081a1ea3c91", "number": 1, diff --git a/besu/src/test/resources/invalid_post_merge_near_head_checkpoint.json b/besu/src/test/resources/invalid_post_merge_near_head_checkpoint.json index cc5f4c054f..c940505a9e 100644 --- a/besu/src/test/resources/invalid_post_merge_near_head_checkpoint.json +++ b/besu/src/test/resources/invalid_post_merge_near_head_checkpoint.json @@ -15,25 +15,6 @@ "grayGlacierBlock": 15050000, "ethash": { }, - "discovery": { - "dns": "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.mainnet.ethdisco.net", - "bootnodes": [ - "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", - "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", - "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", - "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", - "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", - "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", - "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", - "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", - "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", - "enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303", - "enode://78de8a0916848093c73790ead81d1928bec737d565119932b98c6b100d944b7a95e94f847f689fc723399d2e31129d182f7ef3863f2b4c820abbf3ab2722344d@191.235.84.50:30303", - "enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303", - "enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303", - "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303" - ] - }, "checkpoint": { "hash": "0x186642d6084eb7799cb01c7eb69c1a7f3b2c32cd141e62d309f30081a1ea3c91", "number": 16015954, diff --git a/besu/src/test/resources/valid_pos_checkpoint_pos_TD_equals_TTD.json b/besu/src/test/resources/valid_pos_checkpoint_pos_TD_equals_TTD.json index eb7c419fed..eb69f82486 100644 --- a/besu/src/test/resources/valid_pos_checkpoint_pos_TD_equals_TTD.json +++ b/besu/src/test/resources/valid_pos_checkpoint_pos_TD_equals_TTD.json @@ -16,25 +16,6 @@ "terminalTotalDifficulty": 10, "ethash": { }, - "discovery": { - "dns": "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.mainnet.ethdisco.net", - "bootnodes": [ - "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", - "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", - "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", - "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", - "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", - "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", - "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", - "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", - "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", - "enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303", - "enode://78de8a0916848093c73790ead81d1928bec737d565119932b98c6b100d944b7a95e94f847f689fc723399d2e31129d182f7ef3863f2b4c820abbf3ab2722344d@191.235.84.50:30303", - "enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303", - "enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303", - "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303" - ] - }, "checkpoint": { "hash": "0x186642d6084eb7799cb01c7eb69c1a7f3b2c32cd141e62d309f30081a1ea3c91", "number": 3, diff --git a/besu/src/test/resources/valid_post_merge_near_head_checkpoint.json b/besu/src/test/resources/valid_post_merge_near_head_checkpoint.json index 074e857748..29d33e1c04 100644 --- a/besu/src/test/resources/valid_post_merge_near_head_checkpoint.json +++ b/besu/src/test/resources/valid_post_merge_near_head_checkpoint.json @@ -16,25 +16,6 @@ "terminalTotalDifficulty": 58750000000000000000000, "ethash": { }, - "discovery": { - "dns": "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.mainnet.ethdisco.net", - "bootnodes": [ - "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", - "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", - "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", - "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", - "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", - "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", - "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", - "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", - "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", - "enode://3f1d12044546b76342d59d4a05532c14b85aa669704bfe1f864fe079415aa2c02d743e03218e57a33fb94523adb54032871a6c51b2cc5514cb7c7e35b3ed0a99@13.93.211.84:30303", - "enode://78de8a0916848093c73790ead81d1928bec737d565119932b98c6b100d944b7a95e94f847f689fc723399d2e31129d182f7ef3863f2b4c820abbf3ab2722344d@191.235.84.50:30303", - "enode://158f8aab45f6d19c6cbf4a089c2670541a8da11978a2f90dbf6a502a4a3bab80d288afdbeb7ec0ef6d92de563767f3b1ea9e8e334ca711e9f8e2df5a0385e8e6@13.75.154.138:30303", - "enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303", - "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303" - ] - }, "checkpoint": { "hash": "0x186642d6084eb7799cb01c7eb69c1a7f3b2c32cd141e62d309f30081a1ea3c91", "number": 16015954, diff --git a/build.gradle b/build.gradle index ee5f488d7b..a4dca8ef1d 100644 --- a/build.gradle +++ b/build.gradle @@ -28,8 +28,21 @@ plugins { id 'me.champeau.jmh' version '0.6.6' apply false id 'net.ltgt.errorprone' version '2.0.2' id 'maven-publish' + id 'org.sonarqube' version '3.4.0.2513' } +sonarqube { + properties { + property "sonar.projectKey", "hyperledger_besu" + property "sonar.organization", "hyperledger" + property "sonar.host.url", "https://sonarcloud.io" + property "sonar.coverage.jacoco.xmlReportPaths", "${buildDir}/reports/jacoco/jacocoRootReport/jacocoRootReport.xml" + property "sonar.coverage.exclusions", "acceptance-tests/**/*" + } +} + +project.tasks["sonarqube"].dependsOn "jacocoRootReport" + if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)) { throw new GradleException("Java 17 or later is required to build Besu.\n" + " Detected version ${JavaVersion.current()}") @@ -862,7 +875,7 @@ task checkSpdxHeader(type: CheckSpdxHeader) { "(.*/.idea/.*)", "(.*/out/.*)", "(.*/build/.*)", - "(.*/src/[^/]+/generated/.*)", + "(.*/src/[^/]+/generated/.*)" ].join("|") } diff --git a/config/src/main/resources/mainnet.json b/config/src/main/resources/mainnet.json index 57273c6436..00668a8cf0 100644 --- a/config/src/main/resources/mainnet.json +++ b/config/src/main/resources/mainnet.json @@ -14,6 +14,7 @@ "arrowGlacierBlock": 13773000, "grayGlacierBlock": 15050000, "terminalTotalDifficulty": 58750000000000000000000, + "shanghaiTime": 1681338455, "ethash": { }, "discovery": { @@ -21,10 +22,6 @@ "bootnodes": [ "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", - "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", - "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", - "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", - "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", "enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303", diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/CombinedProtocolScheduleFactory.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/CombinedProtocolScheduleFactory.java index 5ef87092bf..1fe75cad9a 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/CombinedProtocolScheduleFactory.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/CombinedProtocolScheduleFactory.java @@ -49,7 +49,7 @@ public class CombinedProtocolScheduleFactory { Optional.ofNullable(forkSpecs.higher(spec)).map(ForkSpec::getBlock); protocolSchedule.getScheduledProtocolSpecs().stream() .filter(protocolSpecMatchesConsensusBlockRange(spec.getBlock(), endBlock)) - .forEach(s -> combinedProtocolSchedule.putMilestone(s.getBlock(), s.getSpec())); + .forEach(s -> combinedProtocolSchedule.putMilestone(s.milestone(), s.spec())); // When moving to a new consensus mechanism we want to use the last milestone but created by // our consensus mechanism's BesuControllerBuilder so any additional rules are applied @@ -64,7 +64,7 @@ public class CombinedProtocolScheduleFactory { private Predicate protocolSpecMatchesConsensusBlockRange( final long startBlock, final Optional endBlock) { return scheduledProtocolSpec -> - scheduledProtocolSpec.getBlock() >= startBlock - && endBlock.map(b -> scheduledProtocolSpec.getBlock() < b).orElse(true); + scheduledProtocolSpec.milestone() >= startBlock + && endBlock.map(b -> scheduledProtocolSpec.milestone() < b).orElse(true); } } diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionProtocolSchedule.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionProtocolSchedule.java index 0d7146444b..42a46f33c7 100644 --- a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionProtocolSchedule.java +++ b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionProtocolSchedule.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.consensus.merge; import org.hyperledger.besu.config.GenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; @@ -24,11 +25,13 @@ import org.hyperledger.besu.ethereum.mainnet.HeaderBasedProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.ScheduledProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.TimestampSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.math.BigInteger; import java.util.Optional; +import java.util.function.Predicate; import java.util.stream.Stream; import org.slf4j.Logger; @@ -211,6 +214,20 @@ public class TransitionProtocolSchedule implements ProtocolSchedule { return Stream.concat(milestoneBlockNumbers, timestampSchedule.streamMilestoneBlocks()); } + @Override + public boolean anyMatch(final Predicate predicate) { + return timestampSchedule.anyMatch(predicate) + || transitionUtils.dispatchFunctionAccordingToMergeState( + schedule -> schedule.anyMatch(predicate)); + } + + @Override + public boolean isOnMilestoneBoundary(final BlockHeader blockHeader) { + return timestampSchedule.isOnMilestoneBoundary(blockHeader) + || transitionUtils.dispatchFunctionAccordingToMergeState( + schedule -> schedule.isOnMilestoneBoundary(blockHeader)); + } + /** * Gets chain id. * diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/TransitionProtocolScheduleTest.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/TransitionProtocolScheduleTest.java index 4030bae6c6..ece9ab706e 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/TransitionProtocolScheduleTest.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/TransitionProtocolScheduleTest.java @@ -224,6 +224,68 @@ public class TransitionProtocolScheduleTest { verifyPostMergeProtocolScheduleReturnedUsingBlockHeader(); } + @Test + public void anyMatch_usesTimestampSchedule() { + when(timestampSchedule.anyMatch(any())).thenReturn(true); + + assertThat(transitionProtocolSchedule.anyMatch(__ -> true)).isTrue(); + verifyNoInteractions(preMergeProtocolSchedule); + verifyNoInteractions(postMergeProtocolSchedule); + } + + @Test + public void anyMatch_delegatesToPreMergeSchedule() { + when(mergeContext.isPostMerge()).thenReturn(false); + + when(timestampSchedule.anyMatch(any())).thenReturn(false); + when(preMergeProtocolSchedule.anyMatch(any())).thenReturn(true); + + assertThat(transitionProtocolSchedule.anyMatch(__ -> true)).isTrue(); + verifyNoInteractions(postMergeProtocolSchedule); + } + + @Test + public void anyMatch_delegatesToPostMergeSchedule() { + when(mergeContext.isPostMerge()).thenReturn(true); + + when(timestampSchedule.anyMatch(any())).thenReturn(false); + when(postMergeProtocolSchedule.anyMatch(any())).thenReturn(true); + + assertThat(transitionProtocolSchedule.anyMatch(__ -> true)).isTrue(); + verifyNoInteractions(preMergeProtocolSchedule); + } + + @Test + public void isOnMilestoneBoundary_usesTimestampSchedule() { + when(timestampSchedule.isOnMilestoneBoundary(any(BlockHeader.class))).thenReturn(true); + + assertThat(transitionProtocolSchedule.isOnMilestoneBoundary(blockHeader)).isTrue(); + verifyNoInteractions(preMergeProtocolSchedule); + verifyNoInteractions(postMergeProtocolSchedule); + } + + @Test + public void isOnMilestoneBoundary_delegatesToPreMergeSchedule() { + when(mergeContext.isPostMerge()).thenReturn(false); + + when(timestampSchedule.isOnMilestoneBoundary(any(BlockHeader.class))).thenReturn(false); + when(preMergeProtocolSchedule.isOnMilestoneBoundary(any(BlockHeader.class))).thenReturn(true); + + assertThat(transitionProtocolSchedule.isOnMilestoneBoundary(blockHeader)).isTrue(); + verifyNoInteractions(postMergeProtocolSchedule); + } + + @Test + public void isOnMilestoneBoundary_delegatesToPostMergeSchedule() { + when(mergeContext.isPostMerge()).thenReturn(true); + + when(timestampSchedule.isOnMilestoneBoundary(any(BlockHeader.class))).thenReturn(false); + when(postMergeProtocolSchedule.isOnMilestoneBoundary(any(BlockHeader.class))).thenReturn(true); + + assertThat(transitionProtocolSchedule.isOnMilestoneBoundary(blockHeader)).isTrue(); + verifyNoInteractions(preMergeProtocolSchedule); + } + private void verifyPreMergeProtocolScheduleReturnedUsingBlockNumber() { verify(preMergeProtocolSchedule).getByBlockNumber(anyLong()); verifyNoInteractions(postMergeProtocolSchedule); diff --git a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java index de8873df35..de75bffd14 100644 --- a/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java +++ b/consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java @@ -41,7 +41,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket; import org.hyperledger.besu.ethereum.mainnet.feemarket.LondonFeeMarket; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import java.util.ArrayList; import java.util.List; @@ -109,7 +109,7 @@ public class MergeReorgTest implements MergeGenesisConfigHelper { @Test public void reorgsAcrossTDDToDifferentTargetsWhenNotFinal() { // Add N blocks to chain from genesis, where total diff is < TTD - Log4j2ConfiguratorUtil.setLevelDebug(BlockHeaderValidator.class.getName()); + LogConfigurator.setLevel(BlockHeaderValidator.class.getName(), "DEBUG"); List endOfWork = subChain(genesisState.getBlock().getHeader(), 10, Difficulty.of(100L)); endOfWork.stream().forEach(this::appendBlock); assertThat(blockchain.getChainHead().getHeight()).isEqualTo(10L); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLDataFetchers.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLDataFetchers.java index dd96c68eec..fbf3b602c0 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLDataFetchers.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLDataFetchers.java @@ -224,9 +224,9 @@ public class GraphQLDataFetchers { ws -> { final Account account = ws.get(addr); if (account == null) { - return new EmptyAccountAdapter(addr); + return Optional.of(new EmptyAccountAdapter(addr)); } - return new AccountAdapter(account); + return Optional.of(new AccountAdapter(account)); }) .or( () -> { @@ -246,9 +246,9 @@ public class GraphQLDataFetchers { ws -> { final Account account = ws.get(addr); if (account == null) { - return new EmptyAccountAdapter(addr); + return Optional.of(new EmptyAccountAdapter(addr)); } - return new AccountAdapter(account); + return Optional.of(new AccountAdapter(account)); }); } }; diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java index 957a6e9855..afa63fe862 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java @@ -96,7 +96,7 @@ public class BlockAdapterBase extends AdapterBase { } return query - .getAndMapWorldState(blockNumber, ws -> ws.get(header.getCoinbase())) + .getAndMapWorldState(blockNumber, ws -> Optional.ofNullable(ws.get(header.getCoinbase()))) .map(account -> (AdapterBase) new AccountAdapter(account)) .or(() -> Optional.of(new EmptyAccountAdapter(header.getCoinbase()))); } @@ -150,7 +150,7 @@ public class BlockAdapterBase extends AdapterBase { bn, ws -> { final Address address = environment.getArgument("address"); - return new AccountAdapter(ws.get(address)); + return Optional.of(new AccountAdapter(ws.get(address))); }); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/LogAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/LogAdapter.java index 52f55ea3ff..20c1e31feb 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/LogAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/LogAdapter.java @@ -64,6 +64,6 @@ public class LogAdapter extends AdapterBase { } return query.getAndMapWorldState( - blockNumber, ws -> new AccountAdapter(ws.get(logWithMetadata.getLogger()))); + blockNumber, ws -> Optional.of(new AccountAdapter(ws.get(logWithMetadata.getLogger())))); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/PendingStateAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/PendingStateAdapter.java index 61300b3fe2..f3462f4209 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/PendingStateAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/PendingStateAdapter.java @@ -66,7 +66,7 @@ public class PendingStateAdapter extends AdapterBase { final Long blockNumber = dataFetchingEnvironment.getArgument("blockNumber"); final long latestBlockNumber = blockchainQuery.latestBlock().get().getHeader().getNumber(); return blockchainQuery - .getAndMapWorldState(latestBlockNumber, ws -> ws.get(addr)) + .getAndMapWorldState(latestBlockNumber, ws -> Optional.ofNullable(ws.get(addr))) .map(AccountAdapter::new); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java index 454590539d..72f2f3b8ab 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java @@ -85,8 +85,9 @@ public class TransactionAdapter extends AdapterBase { return query.getAndMapWorldState( blockNumber, mutableWorldState -> - new AccountAdapter( - mutableWorldState.get(transactionWithMetadata.getTransaction().getSender()))); + Optional.of( + new AccountAdapter( + mutableWorldState.get(transactionWithMetadata.getTransaction().getSender())))); } public Optional getTo(final DataFetchingEnvironment environment) { @@ -102,9 +103,7 @@ public class TransactionAdapter extends AdapterBase { transactionWithMetadata .getTransaction() .getTo() - .map(address -> new AccountAdapter(address, ws.get(address))) - // safe because mapWorldState returns Optional.ofNullable - .orElse(null)); + .map(address -> new AccountAdapter(address, ws.get(address)))); } public Optional getValue() { @@ -174,7 +173,8 @@ public class TransactionAdapter extends AdapterBase { return Optional.empty(); } final long blockNumber = bn.orElseGet(txBlockNumber::get); - return query.getAndMapWorldState(blockNumber, ws -> new AccountAdapter(ws.get(addr.get()))); + return query.getAndMapWorldState( + blockNumber, ws -> Optional.of(new AccountAdapter(ws.get(addr.get())))); } } return Optional.empty(); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractTraceByHash.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractTraceByHash.java index 4cb1fd1de0..281d724810 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractTraceByHash.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractTraceByHash.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTraceGenerator; @@ -28,6 +29,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import java.util.Collections; +import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Stream; @@ -62,19 +64,33 @@ public abstract class AbstractTraceByHash implements JsonRpcMethod { if (block == null || block.getBody().getTransactions().isEmpty()) { return Stream.empty(); } - final TransactionTrace transactionTrace = getTransactionTrace(block, transactionHash); - return getTraceStream(transactionTrace, block); + return Tracer.processTracing( + blockchainQueries, + Optional.of(block.getHeader()), + mutableWorldState -> { + final TransactionTrace transactionTrace = getTransactionTrace(block, transactionHash); + return Optional.ofNullable(getTraceStream(transactionTrace, block)); + }) + .orElse(Stream.empty()); } private TransactionTrace getTransactionTrace(final Block block, final Hash transactionHash) { - return blockTracerSupplier - .get() - .trace(block, new DebugOperationTracer(new TraceOptions(false, false, true))) - .map(BlockTrace::getTransactionTraces) - .orElse(Collections.emptyList()) - .stream() - .filter(trxTrace -> trxTrace.getTransaction().getHash().equals(transactionHash)) - .findFirst() + return Tracer.processTracing( + blockchainQueries, + Optional.of(block.getHeader()), + mutableWorldState -> { + return blockTracerSupplier + .get() + .trace( + mutableWorldState, + block, + new DebugOperationTracer(new TraceOptions(false, false, true))) + .map(BlockTrace::getTransactionTraces) + .orElse(Collections.emptyList()) + .stream() + .filter(trxTrace -> trxTrace.getTransaction().getHash().equals(transactionHash)) + .findFirst(); + }) .orElseThrow(); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminChangeLogLevel.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminChangeLogLevel.java index fe4958104e..8a6dc7168f 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminChangeLogLevel.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminChangeLogLevel.java @@ -21,13 +21,12 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import java.util.Arrays; import java.util.Optional; import java.util.Set; -import org.apache.logging.log4j.Level; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,12 +44,11 @@ public class AdminChangeLogLevel implements JsonRpcMethod { @Override public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { try { - final String rawLogLevel = requestContext.getRequiredParameter(0, String.class); - if (!VALID_PARAMS.contains(rawLogLevel)) { + final String logLevel = requestContext.getRequiredParameter(0, String.class); + if (!VALID_PARAMS.contains(logLevel)) { return new JsonRpcErrorResponse( requestContext.getRequest().getId(), JsonRpcError.INVALID_PARAMS); } - final Level logLevel = Level.toLevel(rawLogLevel); final Optional optionalLogFilters = requestContext.getOptionalParameter(1, String[].class); optionalLogFilters.ifPresentOrElse( @@ -64,8 +62,8 @@ public class AdminChangeLogLevel implements JsonRpcMethod { } } - private void setLogLevel(final String logFilter, final Level logLevel) { - LOG.debug("Setting {} logging level to {} ", logFilter, logLevel.name()); - Log4j2ConfiguratorUtil.setAllLevels(logFilter, logLevel); + private void setLogLevel(final String logFilter, final String logLevel) { + LOG.debug("Setting {} logging level to {} ", logFilter, logLevel); + LogConfigurator.setLevel(logFilter, logLevel); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountAt.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountAt.java index bea7efb9ae..a6cde35bb4 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountAt.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountAt.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameterOrBlockHash; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; @@ -87,42 +88,56 @@ public class DebugAccountAt extends AbstractBlockParameterOrBlockHashMethod { requestContext.getRequest().getId(), JsonRpcError.INVALID_PARAMS); } - final Optional transactionTrace = - blockTracerSupplier - .get() - .trace(blockHash, new DebugOperationTracer(new TraceOptions(false, true, true))) - .map(BlockTrace::getTransactionTraces) - .orElse(Collections.emptyList()) - .stream() - .filter( - trxTrace -> - trxTrace - .getTransaction() - .getHash() - .equals(transactions.get(txIndex).getTransaction().getHash())) - .findFirst(); - - if (transactionTrace.isEmpty()) { - return new JsonRpcErrorResponse( - requestContext.getRequest().getId(), JsonRpcError.TRANSACTION_NOT_FOUND); - } - - Optional account = - transactionTrace.get().getTraceFrames().stream() - .map(traceFrame -> traceFrame.getWorldUpdater().get(address)) - .filter(Objects::nonNull) - .filter(a -> a.getAddress().equals(address)) - .findFirst(); - if (account.isEmpty()) { - return new JsonRpcErrorResponse( - requestContext.getRequest().getId(), JsonRpcError.NO_ACCOUNT_FOUND); - } - - return debugAccountAtResult( - account.get().getCode(), - Quantity.create(account.get().getNonce()), - Quantity.create(account.get().getBalance()), - Quantity.create(account.get().getCodeHash())); + return Tracer.processTracing( + blockchainQueries.get(), + Optional.of(block.get().getHeader()), + mutableWorldState -> { + final Optional transactionTrace = + blockTracerSupplier + .get() + .trace( + mutableWorldState, + blockHash, + new DebugOperationTracer(new TraceOptions(false, true, true))) + .map(BlockTrace::getTransactionTraces) + .orElse(Collections.emptyList()) + .stream() + .filter( + trxTrace -> + trxTrace + .getTransaction() + .getHash() + .equals(transactions.get(txIndex).getTransaction().getHash())) + .findFirst(); + + if (transactionTrace.isEmpty()) { + return Optional.of( + new JsonRpcErrorResponse( + requestContext.getRequest().getId(), JsonRpcError.TRANSACTION_NOT_FOUND)); + } + + Optional account = + transactionTrace.get().getTraceFrames().stream() + .map(traceFrame -> traceFrame.getWorldUpdater().get(address)) + .filter(Objects::nonNull) + .filter(a -> a.getAddress().equals(address)) + .findFirst(); + if (account.isEmpty()) { + return Optional.of( + new JsonRpcErrorResponse( + requestContext.getRequest().getId(), JsonRpcError.NO_ACCOUNT_FOUND)); + } + + return Optional.of( + debugAccountAtResult( + account.get().getCode(), + Quantity.create(account.get().getNonce()), + Quantity.create(account.get().getBalance()), + Quantity.create(account.get().getCodeHash()))); + }) + .orElse( + new JsonRpcErrorResponse( + requestContext.getRequest().getId(), JsonRpcError.WORLD_STATE_UNAVAILABLE)); } protected ImmutableDebugAccountAtResult debugAccountAtResult( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountRange.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountRange.java index 777cc3f98b..4ca5cb0685 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountRange.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountRange.java @@ -88,15 +88,17 @@ public class DebugAccountRange implements JsonRpcMethod { accounts.remove(maxResults); } - return new JsonRpcSuccessResponse( - requestContext.getRequest().getId(), - new DebugAccountRangeAtResult( - accounts.stream() - .collect( - Collectors.toMap( - account -> account.getAddressHash().toString(), - account -> account.getAddress().orElse(Address.ZERO).toString())), - nextKey.toString())); + return Optional.of( + new JsonRpcSuccessResponse( + requestContext.getRequest().getId(), + new DebugAccountRangeAtResult( + accounts.stream() + .collect( + Collectors.toMap( + account -> account.getAddressHash().toString(), + account -> + account.getAddress().orElse(Address.ZERO).toString())), + nextKey.toString()))); }) .orElse(emptyResponse(requestContext)); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFile.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFile.java index 6c3d904bc0..d9cab6769e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFile.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFile.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; @@ -29,11 +30,11 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.core.Block; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Supplier; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Suppliers; public class DebugStandardTraceBlockToFile implements JsonRpcMethod { @@ -79,14 +80,18 @@ public class DebugStandardTraceBlockToFile implements JsonRpcMethod { protected List traceBlock( final Block block, final Optional transactionTraceParams) { - return transactionTracerSupplier - .get() - .traceTransactionToFile( - block.getHash(), transactionTraceParams, dataDir.resolve(TRACE_PATH)); - } - - protected Object emptyResult() { - final ObjectMapper mapper = new ObjectMapper(); - return mapper.createArrayNode(); + return Tracer.processTracing( + blockchainQueries.get(), + Optional.of(block.getHeader()), + mutableWorldState -> + Optional.of( + transactionTracerSupplier + .get() + .traceTransactionToFile( + mutableWorldState, + block.getHash(), + transactionTraceParams, + dataDir.resolve(TRACE_PATH)))) + .orElse(new ArrayList<>()); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAt.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAt.java index 3c5db065ee..0ea50b41ac 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAt.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAt.java @@ -20,6 +20,8 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameterOrBlockHash; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockReplay; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer.TraceableState; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugStorageRangeAtResult; @@ -29,7 +31,6 @@ import org.hyperledger.besu.ethereum.api.query.TransactionWithMetadata; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountStorageEntry; -import org.hyperledger.besu.evm.worldstate.WorldState; import java.util.Collections; import java.util.NavigableMap; @@ -85,36 +86,38 @@ public class DebugStorageRangeAt implements JsonRpcMethod { return emptyResponse(requestContext); } - final Optional optional = + final Optional maybeTransactionIndex = blockchainQueries.get().transactionByBlockHashAndIndex(blockHash, transactionIndex); - return optional - .map( - transactionWithMetadata -> - (blockReplay + return Tracer.processTracing( + blockchainQueries.get(), + Optional.of(blockHeaderOptional.get()), + mutableWorldState -> { + if (maybeTransactionIndex.isEmpty()) { + return Optional.of( + extractStorageAt( + requestContext, accountAddress, startKey, limit, mutableWorldState)); + } else { + return blockReplay .get() .afterTransactionInBlock( + mutableWorldState, blockHash, - transactionWithMetadata.getTransaction().getHash(), + maybeTransactionIndex.get().getTransaction().getHash(), (transaction, blockHeader, blockchain, - worldState, transactionProcessor, protocolSpec) -> extractStorageAt( - requestContext, accountAddress, startKey, limit, worldState)) - .orElseGet(() -> emptyResponse(requestContext)))) - .orElseGet( - () -> - blockchainQueries - .get() - .getAndMapWorldState( - blockHeaderOptional.get().getNumber(), - worldState -> - extractStorageAt( - requestContext, accountAddress, startKey, limit, worldState)) - .orElseGet(() -> emptyResponse(requestContext))); + requestContext, + accountAddress, + startKey, + limit, + mutableWorldState)); + } + }) + .orElse(emptyResponse(requestContext)); } private Optional hashFromParameter(final BlockParameterOrBlockHash blockParameter) { @@ -137,7 +140,7 @@ public class DebugStorageRangeAt implements JsonRpcMethod { final Address accountAddress, final Hash startKey, final int limit, - final WorldState worldState) { + final TraceableState worldState) { final Account account = worldState.get(accountAddress); final NavigableMap entries = account.storageEntriesFrom(startKey, limit + 1); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java index 97892ebda9..762728edbe 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; @@ -33,6 +34,7 @@ import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; import java.util.Collection; +import java.util.Optional; import java.util.function.Supplier; import org.apache.tuweni.bytes.Bytes; @@ -79,11 +81,16 @@ public class DebugTraceBlock implements JsonRpcMethod { if (this.blockchain.blockByHash(block.getHeader().getParentHash()).isPresent()) { final Collection results = - blockTracerSupplier - .get() - .trace(block, new DebugOperationTracer(traceOptions)) - .map(BlockTrace::getTransactionTraces) - .map(DebugTraceTransactionResult::of) + Tracer.processTracing( + blockchain, + Optional.of(block.getHeader()), + mutableWorldState -> { + return blockTracerSupplier + .get() + .trace(mutableWorldState, block, new DebugOperationTracer(traceOptions)) + .map(BlockTrace::getTransactionTraces) + .map(DebugTraceTransactionResult::of); + }) .orElse(null); return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results); } else { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHash.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHash.java index f600b090c7..3b5f5e734f 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHash.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHash.java @@ -20,9 +20,11 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.debug.TraceOptions; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; @@ -32,9 +34,13 @@ import java.util.function.Supplier; public class DebugTraceBlockByHash implements JsonRpcMethod { private final Supplier blockTracerSupplier; + private final Supplier blockchainQueries; - public DebugTraceBlockByHash(final Supplier blockTracerSupplier) { + public DebugTraceBlockByHash( + final Supplier blockTracerSupplier, + final Supplier blockchainQueriesSupplier) { this.blockTracerSupplier = blockTracerSupplier; + this.blockchainQueries = blockchainQueriesSupplier; } @Override @@ -52,11 +58,15 @@ public class DebugTraceBlockByHash implements JsonRpcMethod { .orElse(TraceOptions.DEFAULT); final Collection results = - blockTracerSupplier - .get() - .trace(blockHash, new DebugOperationTracer(traceOptions)) - .map(BlockTrace::getTransactionTraces) - .map(DebugTraceTransactionResult::of) + Tracer.processTracing( + blockchainQueries.get(), + blockHash, + mutableWorldState -> + blockTracerSupplier + .get() + .trace(mutableWorldState, blockHash, new DebugOperationTracer(traceOptions)) + .map(BlockTrace::getTransactionTraces) + .map(DebugTraceTransactionResult::of)) .orElse(null); return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumber.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumber.java index 95f8a65732..977a1d8793 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumber.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumber.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParame import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.debug.TraceOptions; @@ -62,11 +63,16 @@ public class DebugTraceBlockByNumber extends AbstractBlockParameterMethod { return blockHash .flatMap( hash -> - blockTracerSupplier - .get() - .trace(hash, new DebugOperationTracer(traceOptions)) - .map(BlockTrace::getTransactionTraces) - .map(DebugTraceTransactionResult::of)) + Tracer.processTracing( + blockchainQueriesSupplier.get(), + hash, + mutableWorldState -> { + return blockTracerSupplier + .get() + .trace(mutableWorldState, hash, new DebugOperationTracer(traceOptions)) + .map(BlockTrace::getTransactionTraces) + .map(DebugTraceTransactionResult::of); + })) .orElse(null); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceTransaction.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceTransaction.java index 4ca20c77cd..1ce30e243e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceTransaction.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceTransaction.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; @@ -74,9 +75,13 @@ public class DebugTraceTransaction implements JsonRpcMethod { final DebugOperationTracer execTracer = new DebugOperationTracer(traceOptions); - return transactionTracer - .traceTransaction(blockHash, hash, execTracer) - .map(DebugTraceTransactionResult::new) + return Tracer.processTracing( + blockchain, + blockHash, + mutableWorldState -> + transactionTracer + .traceTransaction(mutableWorldState, blockHash, hash, execTracer) + .map(DebugTraceTransactionResult::new)) .orElse(null); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetProof.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetProof.java index b2d5007579..6075f2be28 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetProof.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetProof.java @@ -72,9 +72,12 @@ public class EthGetProof extends AbstractBlockParameterOrBlockHashMethod { new JsonRpcSuccessResponse( requestContext.getRequest().getId(), GetProofResult.buildGetProofResult(address, proof))) - .orElse( - new JsonRpcErrorResponse( - requestContext.getRequest().getId(), JsonRpcError.NO_ACCOUNT_FOUND)); + .or( + () -> + Optional.of( + new JsonRpcErrorResponse( + requestContext.getRequest().getId(), + JsonRpcError.NO_ACCOUNT_FOUND))); }) .orElse( new JsonRpcErrorResponse( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java index 64e4b421d5..0658fe6c94 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.FilterParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTraceGenerator; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.RewardTraceGenerator; @@ -89,15 +90,27 @@ public class TraceBlock extends AbstractBlockParameterMethod { } final ArrayNodeWrapper resultArrayNode = new ArrayNodeWrapper(MAPPER.createArrayNode()); - blockTracerSupplier - .get() - .trace(block, new DebugOperationTracer(new TraceOptions(false, false, true))) - .ifPresent( - blockTrace -> - generateTracesFromTransactionTraceAndBlock( - filterParameter, blockTrace.getTransactionTraces(), block, resultArrayNode)); - - generateRewardsFromBlock(filterParameter, block, resultArrayNode); + Tracer.processTracing( + blockchainQueriesSupplier.get(), + Optional.of(block.getHeader()), + mutableWorldState -> { + blockTracerSupplier + .get() + .trace( + mutableWorldState, + block, + new DebugOperationTracer(new TraceOptions(false, false, true))) + .ifPresent( + blockTrace -> + generateTracesFromTransactionTraceAndBlock( + filterParameter, + blockTrace.getTransactionTraces(), + block, + resultArrayNode)); + + generateRewardsFromBlock(filterParameter, block, resultArrayNode); + return Optional.empty(); + }); return resultArrayNode; } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java index 6f78d28ab1..d863caf0ca 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceCallMany.java @@ -130,20 +130,20 @@ public class TraceCallMany extends TraceCall implements JsonRpcMethod { }); } catch (final TransactionInvalidException e) { LOG.error("Invalid transaction simulator result"); - return new JsonRpcErrorResponse( - requestContext.getRequest().getId(), INTERNAL_ERROR); + return Optional.of( + new JsonRpcErrorResponse(requestContext.getRequest().getId(), INTERNAL_ERROR)); } catch (final EmptySimulatorResultException e) { LOG.error( "Empty simulator result, call params: {}, blockHeader: {} ", JsonCallParameterUtil.validateAndGetCallParams(requestContext), blockHeader); - return new JsonRpcErrorResponse( - requestContext.getRequest().getId(), INTERNAL_ERROR); + return Optional.of( + new JsonRpcErrorResponse(requestContext.getRequest().getId(), INTERNAL_ERROR)); } catch (final Exception e) { - return new JsonRpcErrorResponse( - requestContext.getRequest().getId(), INTERNAL_ERROR); + return Optional.of( + new JsonRpcErrorResponse(requestContext.getRequest().getId(), INTERNAL_ERROR)); } - return traceCallResults; + return Optional.of(traceCallResults); }); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java index 22c88e1db1..1aed9088dd 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceReplayBlockTransactions.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TraceTypePa import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TraceTypeParameter.TraceType; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.TraceFormatter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.TraceWriter; @@ -113,11 +114,16 @@ public class TraceReplayBlockTransactions extends AbstractBlockParameterMethod { final TraceOptions traceOptions = new TraceOptions(false, false, traceTypes.contains(VM_TRACE) || traceTypes.contains(TRACE)); - return blockTracerSupplier - .get() - .trace(block, new DebugOperationTracer(traceOptions)) - .map(BlockTrace::getTransactionTraces) - .map((traces) -> generateTracesFromTransactionTrace(traces, block, traceTypes)) + return Tracer.processTracing( + blockchainQueriesSupplier.get(), + Optional.of(block.getHeader()), + mutableWorldState -> { + return blockTracerSupplier + .get() + .trace(mutableWorldState, block, new DebugOperationTracer(traceOptions)) + .map(BlockTrace::getTransactionTraces) + .map((traces) -> generateTracesFromTransactionTrace(traces, block, traceTypes)); + }) .orElse(null); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockReplay.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockReplay.java index 98e44c822e..252771a7ed 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockReplay.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockReplay.java @@ -17,11 +17,11 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor; import org.hyperledger.besu.datatypes.DataGas; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer.TraceableState; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -29,7 +29,6 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.List; import java.util.Optional; @@ -38,15 +37,10 @@ public class BlockReplay { private final ProtocolSchedule protocolSchedule; private final Blockchain blockchain; - private final WorldStateArchive worldStateArchive; - public BlockReplay( - final ProtocolSchedule protocolSchedule, - final Blockchain blockchain, - final WorldStateArchive worldStateArchive) { + public BlockReplay(final ProtocolSchedule protocolSchedule, final Blockchain blockchain) { this.protocolSchedule = protocolSchedule; this.blockchain = blockchain; - this.worldStateArchive = worldStateArchive; } public Optional block( @@ -54,7 +48,7 @@ public class BlockReplay { return performActionWithBlock( block.getHeader(), block.getBody(), - (body, header, blockchain, mutableWorldState, transactionProcessor, protocolSpec) -> { + (body, header, blockchain, transactionProcessor, protocolSpec) -> { final Wei dataGasPrice = protocolSpec .getFeeMarket() @@ -69,12 +63,7 @@ public class BlockReplay { .map( transaction -> action.performAction( - transaction, - header, - blockchain, - mutableWorldState, - transactionProcessor, - dataGasPrice)) + transaction, header, blockchain, transactionProcessor, dataGasPrice)) .toList(); return Optional.of(new BlockTrace(transactionTraces)); }); @@ -86,10 +75,13 @@ public class BlockReplay { } public Optional beforeTransactionInBlock( - final Hash blockHash, final Hash transactionHash, final TransactionAction action) { + final TraceableState mutableWorldState, + final Hash blockHash, + final Hash transactionHash, + final TransactionAction action) { return performActionWithBlock( blockHash, - (body, header, blockchain, mutableWorldState, transactionProcessor, protocolSpec) -> { + (body, header, blockchain, transactionProcessor, protocolSpec) -> { final BlockHashLookup blockHashLookup = new CachingBlockHashLookup(header, blockchain); final Wei dataGasPrice = protocolSpec @@ -104,12 +96,7 @@ public class BlockReplay { if (transaction.getHash().equals(transactionHash)) { return Optional.of( action.performAction( - transaction, - header, - blockchain, - mutableWorldState, - transactionProcessor, - dataGasPrice)); + transaction, header, blockchain, transactionProcessor, dataGasPrice)); } else { transactionProcessor.processTransaction( blockchain, @@ -128,15 +115,19 @@ public class BlockReplay { } public Optional afterTransactionInBlock( - final Hash blockHash, final Hash transactionHash, final TransactionAction action) { + final TraceableState mutableWorldState, + final Hash blockHash, + final Hash transactionHash, + final TransactionAction action) { return beforeTransactionInBlock( + mutableWorldState, blockHash, transactionHash, - (transaction, blockHeader, blockchain, worldState, transactionProcessor, dataGasPrice) -> { + (transaction, blockHeader, blockchain, transactionProcessor, dataGasPrice) -> { final ProtocolSpec spec = protocolSchedule.getByBlockHeader(blockHeader); transactionProcessor.processTransaction( blockchain, - worldState.updater(), + mutableWorldState.updater(), blockHeader, transaction, spec.getMiningBeneficiaryCalculator().calculateBeneficiary(blockHeader), @@ -145,7 +136,7 @@ public class BlockReplay { TransactionValidationParams.blockReplay(), dataGasPrice); return action.performAction( - transaction, blockHeader, blockchain, worldState, transactionProcessor, dataGasPrice); + transaction, blockHeader, blockchain, transactionProcessor, dataGasPrice); }); } @@ -168,30 +159,8 @@ public class BlockReplay { } final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(header); final MainnetTransactionProcessor transactionProcessor = protocolSpec.getTransactionProcessor(); - final BlockHeader previous = blockchain.getBlockHeader(header.getParentHash()).orElse(null); - if (previous == null) { - return Optional.empty(); - } - try (final MutableWorldState worldState = - worldStateArchive - .getMutable(previous.getStateRoot(), previous.getBlockHash(), false) - .map( - ws -> { - if (!ws.isPersistable()) { - return ws.copy(); - } - return ws; - }) - .orElseThrow( - () -> - new IllegalArgumentException( - "Missing worldstate for stateroot " - + previous.getStateRoot().toShortHexString()))) { - return action.perform( - body, header, blockchain, worldState, transactionProcessor, protocolSpec); - } catch (Exception ex) { - return Optional.empty(); - } + + return action.perform(body, header, blockchain, transactionProcessor, protocolSpec); } private Optional getBlock(final Hash blockHash) { @@ -217,7 +186,6 @@ public class BlockReplay { BlockBody body, BlockHeader blockHeader, Blockchain blockchain, - MutableWorldState worldState, MainnetTransactionProcessor transactionProcessor, ProtocolSpec protocolSpec); } @@ -228,7 +196,6 @@ public class BlockReplay { Transaction transaction, BlockHeader blockHeader, Blockchain blockchain, - MutableWorldState worldState, MainnetTransactionProcessor transactionProcessor, Wei dataGasPrice); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockTracer.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockTracer.java index 599980e5c6..bfccaac9e2 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockTracer.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/BlockTracer.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup; @@ -37,22 +38,23 @@ public class BlockTracer { this.blockReplay = blockReplay; } - public Optional trace(final Hash blockHash, final DebugOperationTracer tracer) { - return blockReplay.block(blockHash, prepareReplayAction(tracer)); + public Optional trace( + final Tracer.TraceableState mutableWorldState, + final Hash blockHash, + final DebugOperationTracer tracer) { + return blockReplay.block(blockHash, prepareReplayAction(mutableWorldState, tracer)); } - public Optional trace(final Block block, final DebugOperationTracer tracer) { - return blockReplay.block(block, prepareReplayAction(tracer)); + public Optional trace( + final Tracer.TraceableState mutableWorldState, + final Block block, + final DebugOperationTracer tracer) { + return blockReplay.block(block, prepareReplayAction(mutableWorldState, tracer)); } private BlockReplay.TransactionAction prepareReplayAction( - final DebugOperationTracer tracer) { - return (transaction, - header, - blockchain, - mutableWorldState, - transactionProcessor, - dataGasPrice) -> { + final MutableWorldState mutableWorldState, final DebugOperationTracer tracer) { + return (transaction, header, blockchain, transactionProcessor, dataGasPrice) -> { // if we have no prior updater, it must be the first TX, so use the block's initial state if (chainedUpdater == null) { chainedUpdater = mutableWorldState.updater(); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/Tracer.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/Tracer.java new file mode 100644 index 0000000000..c6bbeceb74 --- /dev/null +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/Tracer.java @@ -0,0 +1,101 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; + +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes32; + +public class Tracer { + + public static Optional processTracing( + final BlockchainQueries blockchainQueries, + final Hash blockHash, + final Function> mapper) { + return processTracing( + blockchainQueries, blockchainQueries.getBlockHeaderByHash(blockHash), mapper); + } + + public static Optional processTracing( + final BlockchainQueries blockchainQueries, + final Optional blockHeader, + final Function> mapper) { + return blockHeader + .map(BlockHeader::getParentHash) + .flatMap( + parentHash -> + blockchainQueries.getAndMapWorldState( + parentHash, + mutableWorldState -> mapper.apply(new TraceableState(mutableWorldState)))); + } + + /** + * This class force the use of the processTracing method to do tracing. processTracing allows you + * to cleanly manage the worldstate, to close it etc + */ + public static class TraceableState implements MutableWorldState { + private final MutableWorldState mutableWorldState; + + private TraceableState(final MutableWorldState mutableWorldState) { + this.mutableWorldState = mutableWorldState; + } + + @Override + public void persist(final BlockHeader blockHeader) { + mutableWorldState.persist(blockHeader); + } + + @Override + public WorldUpdater updater() { + return mutableWorldState.updater(); + } + + @Override + public Hash rootHash() { + return mutableWorldState.rootHash(); + } + + @Override + public Hash frontierRootHash() { + return mutableWorldState.rootHash(); + } + + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + return mutableWorldState.streamAccounts(startKeyHash, limit); + } + + @Override + public Account get(final Address address) { + return mutableWorldState.get(address); + } + + @Override + public void close() throws Exception { + mutableWorldState.close(); + } + } +} diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracer.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracer.java index ef35a07244..6375b7befe 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracer.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracer.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.debug.TraceOptions; import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams; @@ -63,25 +64,32 @@ public class TransactionTracer { } public Optional traceTransaction( - final Hash blockHash, final Hash transactionHash, final DebugOperationTracer tracer) { - return blockReplay.beforeTransactionInBlock( - blockHash, - transactionHash, - (transaction, header, blockchain, worldState, transactionProcessor, dataGasPrice) -> { - final TransactionProcessingResult result = - processTransaction( - header, - blockchain, - worldState.updater(), - transaction, - transactionProcessor, - tracer, - dataGasPrice); - return new TransactionTrace(transaction, result, tracer.getTraceFrames()); - }); + final Tracer.TraceableState mutableWorldState, + final Hash blockHash, + final Hash transactionHash, + final DebugOperationTracer tracer) { + Optional transactionTrace = + blockReplay.beforeTransactionInBlock( + mutableWorldState, + blockHash, + transactionHash, + (transaction, header, blockchain, transactionProcessor, dataGasPrice) -> { + final TransactionProcessingResult result = + processTransaction( + header, + blockchain, + mutableWorldState.updater(), + transaction, + transactionProcessor, + tracer, + dataGasPrice); + return new TransactionTrace(transaction, result, tracer.getTraceFrames()); + }); + return transactionTrace; } public List traceTransactionToFile( + final MutableWorldState mutableWorldState, final Hash blockHash, final Optional transactionTraceParams, final Path traceDir) { @@ -104,8 +112,8 @@ public class TransactionTracer { return blockReplay .performActionWithBlock( blockHash, - (body, header, blockchain, worldState, transactionProcessor, protocolSpec) -> { - WorldUpdater stackedUpdater = worldState.updater().updater(); + (body, header, blockchain, transactionProcessor, protocolSpec) -> { + WorldUpdater stackedUpdater = mutableWorldState.updater().updater(); final List traces = new ArrayList<>(); final Wei dataGasPrice = protocolSpec @@ -180,7 +188,6 @@ public class TransactionTracer { final MainnetTransactionProcessor transactionProcessor, final OperationTracer tracer, final Wei dataGasPrice) { - return transactionProcessor.processTransaction( blockchain, worldUpdater, diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/DebugJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/DebugJsonRpcMethods.java index 4a337137fa..cce9ada72a 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/DebugJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/DebugJsonRpcMethods.java @@ -85,10 +85,7 @@ public class DebugJsonRpcMethods extends ApiGroupJsonRpcMethods { @Override protected Map create() { final BlockReplay blockReplay = - new BlockReplay( - protocolSchedule, - blockchainQueries.getBlockchain(), - blockchainQueries.getWorldStateArchive()); + new BlockReplay(protocolSchedule, blockchainQueries.getBlockchain()); return mapOf( new DebugTraceTransaction(blockchainQueries, new TransactionTracer(blockReplay)), @@ -103,7 +100,7 @@ public class DebugJsonRpcMethods extends ApiGroupJsonRpcMethods { new DebugSetHead(blockchainQueries, protocolContext), new DebugReplayBlock(blockchainQueries, protocolContext, protocolSchedule), new DebugTraceBlockByNumber(() -> new BlockTracer(blockReplay), blockchainQueries), - new DebugTraceBlockByHash(() -> new BlockTracer(blockReplay)), + new DebugTraceBlockByHash(() -> new BlockTracer(blockReplay), () -> blockchainQueries), new DebugBatchSendRawTransaction(transactionPool), new DebugGetBadBlocks(blockchainQueries, protocolSchedule, blockResult), new DebugStandardTraceBlockToFile( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java index 3aabdee372..eac87abef8 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java @@ -56,10 +56,7 @@ public class TraceJsonRpcMethods extends ApiGroupJsonRpcMethods { @Override protected Map create() { final BlockReplay blockReplay = - new BlockReplay( - protocolSchedule, - blockchainQueries.getBlockchain(), - blockchainQueries.getWorldStateArchive()); + new BlockReplay(protocolSchedule, blockchainQueries.getBlockchain()); return mapOf( new TraceReplayBlockTransactions( () -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries), diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java index c0f14dbf6b..f854f48692 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java @@ -303,7 +303,8 @@ public class BlockchainQueries { * @return The number of transactions sent from the given address. */ public long getTransactionCount(final Address address, final Hash blockHash) { - return getAndMapWorldState(blockHash, worldState -> worldState.get(address)) + return getAndMapWorldState( + blockHash, worldState -> Optional.ofNullable(worldState.get(address))) .map(Account::getNonce) .orElse(0L); } @@ -859,24 +860,15 @@ public class BlockchainQueries { * @return the world state at the block number */ public Optional getAndMapWorldState( - final Hash blockHash, final Function mapper) { + final Hash blockHash, final Function> mapper) { + return blockchain .getBlockHeader(blockHash) .flatMap( blockHeader -> { - try (final var worldState = - worldStateArchive - .getMutable(blockHeader.getStateRoot(), blockHeader.getHash(), false) - .map( - ws -> { - if (!ws.isPersistable()) { - return ws.copy(); - } - return ws; - }) - .orElse(null)) { - if (worldState != null) { - return Optional.ofNullable(mapper.apply(worldState)); + try (var ws = worldStateArchive.getMutable(blockHeader, false).orElse(null)) { + if (ws != null) { + return mapper.apply(ws); } } catch (Exception ex) { LOG.error("failed worldstate query for " + blockHash.toShortHexString(), ex); @@ -894,7 +886,7 @@ public class BlockchainQueries { * @return the world state at the block number */ public Optional getAndMapWorldState( - final long blockNumber, final Function mapper) { + final long blockNumber, final Function> mapper) { final Hash blockHash = getBlockHeaderByNumber(blockNumber).map(BlockHeader::getHash).orElse(Hash.EMPTY); return getAndMapWorldState(blockHash, mapper); @@ -965,7 +957,9 @@ public class BlockchainQueries { return getAndMapWorldState( blockHash, worldState -> - Optional.ofNullable(worldState.get(address)).map(getter).orElse(noAccountValue)); + Optional.ofNullable(worldState.get(address)) + .map(getter) + .or(() -> Optional.ofNullable(noAccountValue))); } private List formatTransactions( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/util/TraceUtil.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/util/TraceUtil.java deleted file mode 100644 index efdb3e8928..0000000000 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/util/TraceUtil.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.api.util; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTrace; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTraceGenerator; -import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.api.query.TransactionWithMetadata; -import org.hyperledger.besu.ethereum.core.Block; -import org.hyperledger.besu.ethereum.debug.TraceOptions; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; - -import java.util.Collections; -import java.util.function.Supplier; -import java.util.stream.Stream; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; - -public class TraceUtil { - - private TraceUtil() { - throw new IllegalStateException("Utility class"); - } - - public static Stream resultByTransactionHash( - final Hash transactionHash, - final BlockchainQueries blockchainQueries, - final Supplier blockTracerSupplier, - final ProtocolSchedule protocolSchedule) { - return blockchainQueries - .transactionByHash(transactionHash) - .flatMap(TransactionWithMetadata::getBlockNumber) - .flatMap(blockNumber -> blockchainQueries.getBlockchain().getBlockByNumber(blockNumber)) - .map(block -> getTraceBlock(block, transactionHash, blockTracerSupplier, protocolSchedule)) - .orElse(Stream.empty()); - } - - private static Stream getTraceBlock( - final Block block, - final Hash transactionHash, - final Supplier blockTracerSupplier, - final ProtocolSchedule protocolSchedule) { - if (block == null || block.getBody().getTransactions().isEmpty()) { - return Stream.empty(); - } - final TransactionTrace transactionTrace = - getTransactionTrace(block, transactionHash, blockTracerSupplier); - return getTraceStream(protocolSchedule, transactionTrace, block); - } - - private static TransactionTrace getTransactionTrace( - final Block block, - final Hash transactionHash, - final Supplier blockTracerSupplier) { - return blockTracerSupplier - .get() - .trace(block, new DebugOperationTracer(new TraceOptions(false, false, true))) - .map(BlockTrace::getTransactionTraces) - .orElse(Collections.emptyList()) - .stream() - .filter(trxTrace -> trxTrace.getTransaction().getHash().equals(transactionHash)) - .findFirst() - .orElseThrow(); - } - - private static Stream getTraceStream( - final ProtocolSchedule protocolSchedule, - final TransactionTrace transactionTrace, - final Block block) { - return FlatTraceGenerator.generateFromTransactionTraceAndBlock( - protocolSchedule, transactionTrace, block) - .map(FlatTrace.class::cast); - } - - public static JsonNode arrayNodeFromTraceStream(final Stream traceStream) { - final ObjectMapper mapper = new ObjectMapper(); - final ArrayNode resultArrayNode = mapper.createArrayNode(); - traceStream.forEachOrdered(resultArrayNode::addPOJO); - return resultArrayNode; - } -} diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminChangeLogLevelTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminChangeLogLevelTest.java index e0a5ed7a6f..eeefcbc045 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminChangeLogLevelTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminChangeLogLevelTest.java @@ -22,7 +22,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import org.apache.logging.log4j.Level; import org.junit.Before; @@ -40,7 +40,7 @@ public class AdminChangeLogLevelTest { @Before public void before() { adminChangeLogLevel = new AdminChangeLogLevel(); - Log4j2ConfiguratorUtil.setAllLevels("", Level.INFO); + LogConfigurator.setLevel("", "INFO"); } @Test diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountAtTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountAtTest.java index 03f247c393..cc97c40bb3 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountAtTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountAtTest.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; import static org.hyperledger.besu.evm.account.Account.MAX_NONCE; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -24,6 +25,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; @@ -34,6 +36,7 @@ import org.hyperledger.besu.ethereum.api.query.BlockWithMetadata; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.api.query.TransactionWithMetadata; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.evm.account.Account; @@ -41,12 +44,14 @@ import org.hyperledger.besu.evm.worldstate.WorldUpdater; import java.util.Collections; import java.util.Optional; +import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @@ -55,7 +60,10 @@ import org.mockito.junit.jupiter.MockitoExtension; class DebugAccountAtTest { @Mock private BlockTracer blockTracer; @Mock private BlockchainQueries blockchainQueries; - @Mock private BlockWithMetadata blockWithMetadata; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private BlockWithMetadata blockWithMetadata; + @Mock private BlockHeader blockHeader; @Mock private TransactionWithMetadata transactionWithMetadata; @Mock private BlockTrace blockTrace; @@ -63,6 +71,8 @@ class DebugAccountAtTest { @Mock private TraceFrame traceFrame; @Mock private Transaction transaction; @Mock private WorldUpdater worldUpdater; + @Mock private MutableWorldState worldState; + @Mock private Account account; private static DebugAccountAt debugAccountAt; @@ -140,6 +150,15 @@ class DebugAccountAtTest { @Test void testTransactionNotFoundResponse() { + doAnswer( + invocation -> + invocation + .>>getArgument( + 1) + .apply(worldState)) + .when(blockchainQueries) + .getAndMapWorldState(any(), any()); + setupMockBlock(); Mockito.when(blockWithMetadata.getTransactions()) .thenReturn(Collections.singletonList(transactionWithMetadata)); @@ -156,6 +175,15 @@ class DebugAccountAtTest { @Test void testNoAccountFoundResponse() { + doAnswer( + invocation -> + invocation + .>>getArgument( + 1) + .apply(worldState)) + .when(blockchainQueries) + .getAndMapWorldState(any(), any()); + setupMockTransaction(); setupMockBlock(); @@ -172,6 +200,15 @@ class DebugAccountAtTest { @Test void shouldBeSuccessfulWhenTransactionsAndAccountArePresent() { + doAnswer( + invocation -> + invocation + .>>getArgument( + 1) + .apply(worldState)) + .when(blockchainQueries) + .getAndMapWorldState(any(), any()); + final String codeString = "0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063b27b880414610030575b"; final Bytes code = Bytes.fromHexString(codeString); @@ -214,7 +251,8 @@ class DebugAccountAtTest { Mockito.when(blockchainQueries.blockByHash(any())).thenReturn(Optional.of(blockWithMetadata)); Mockito.when(blockWithMetadata.getTransactions()) .thenReturn(Collections.singletonList(transactionWithMetadata)); - Mockito.when(blockTracer.trace(any(Hash.class), any())).thenReturn(Optional.of(blockTrace)); + Mockito.when(blockTracer.trace(any(Tracer.TraceableState.class), any(Hash.class), any())) + .thenReturn(Optional.of(blockTrace)); Mockito.when(blockTrace.getTransactionTraces()) .thenReturn(Collections.singletonList(transactionTrace)); Mockito.when(transactionTrace.getTransaction()).thenReturn(transaction); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBadBlockToFileTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBadBlockToFileTest.java index 27e48157c8..1c749df131 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBadBlockToFileTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBadBlockToFileTest.java @@ -17,12 +17,14 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.chain.BadBlockManager; @@ -31,6 +33,7 @@ import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; @@ -38,7 +41,9 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.function.Function; +import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -49,6 +54,8 @@ public class DebugStandardTraceBadBlockToFileTest { private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); private final Blockchain blockchain = mock(Blockchain.class); + private final MutableWorldState mutableWorldState = mock(MutableWorldState.class); + private final ProtocolSchedule protocolSchedule = mock(ProtocolSchedule.class); private final TransactionTracer transactionTracer = mock(TransactionTracer.class); private final ProtocolSpec protocolSpec = mock(ProtocolSpec.class); @@ -57,6 +64,18 @@ public class DebugStandardTraceBadBlockToFileTest { new DebugStandardTraceBadBlockToFile( () -> transactionTracer, blockchainQueries, protocolSchedule, folder.getRoot().toPath()); + @Before + public void setup() { + doAnswer( + invocation -> + invocation + .>>getArgument( + 1) + .apply(mutableWorldState)) + .when(blockchainQueries) + .getAndMapWorldState(any(), any()); + } + @Test public void nameShouldBeDebugTraceTransaction() { assertThat(debugStandardTraceBadBlockToFile.getName()) @@ -89,7 +108,8 @@ public class DebugStandardTraceBadBlockToFileTest { when(blockchainQueries.getBlockchain()).thenReturn(blockchain); when(blockchain.getChainHeadHeader()).thenReturn(new BlockHeaderTestFixture().buildHeader()); when(protocolSchedule.getByBlockHeader(blockHeader)).thenReturn(protocolSpec); - when(transactionTracer.traceTransactionToFile(eq(block.getHash()), any(), any())) + when(transactionTracer.traceTransactionToFile( + any(MutableWorldState.class), eq(block.getHash()), any(), any())) .thenReturn(paths); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugStandardTraceBadBlockToFile.response(request); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java index b49cb44048..ddae8a94ff 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; @@ -28,6 +29,8 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.ArrayList; import java.util.HashMap; @@ -37,13 +40,17 @@ import java.util.Optional; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Answers; public class DebugStandardTraceBlockToFileTest { @ClassRule public static final TemporaryFolder folder = new TemporaryFolder(); - private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); + private final WorldStateArchive archive = + mock(WorldStateArchive.class, Answers.RETURNS_DEEP_STUBS); private final Blockchain blockchain = mock(Blockchain.class); + private final BlockchainQueries blockchainQueries = + spy(new BlockchainQueries(blockchain, archive)); private final TransactionTracer transactionTracer = mock(TransactionTracer.class); private final DebugStandardTraceBlockToFile debugStandardTraceBlockToFile = new DebugStandardTraceBlockToFile( @@ -75,8 +82,10 @@ public class DebugStandardTraceBlockToFileTest { when(blockchainQueries.getBlockchain()).thenReturn(blockchain); when(blockchain.getBlockByHash(block.getHash())).thenReturn(Optional.of(block)); + when(blockchain.getBlockHeader(genesis.getHash())).thenReturn(Optional.of(genesis.getHeader())); - when(transactionTracer.traceTransactionToFile(eq(block.getHash()), any(), any())) + when(transactionTracer.traceTransactionToFile( + any(MutableWorldState.class), eq(block.getHash()), any(), any())) .thenReturn(paths); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugStandardTraceBlockToFile.response(request); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAtTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAtTest.java index 488f0d05fe..5767b1b008 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAtTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAtTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -27,6 +28,8 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockReplay; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugStorageRangeAtResult; import org.hyperledger.besu.ethereum.api.query.BlockWithMetadata; @@ -48,11 +51,13 @@ import java.util.List; import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; +import java.util.function.Function; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; import org.junit.Before; import org.junit.Test; +import org.mockito.Answers; import org.mockito.invocation.InvocationOnMock; public class DebugStorageRangeAtTest { @@ -61,16 +66,16 @@ public class DebugStorageRangeAtTest { private static final Bytes32 START_KEY_HASH = Bytes32.fromHexString("0x22"); private final Blockchain blockchain = mock(Blockchain.class); private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); - private final BlockReplay blockReplay = mock(BlockReplay.class); + private final BlockReplay blockReplay = mock(BlockReplay.class, Answers.RETURNS_DEEP_STUBS); private final DebugStorageRangeAt debugStorageRangeAt = new DebugStorageRangeAt(blockchainQueries, blockReplay); - private final MutableWorldState worldState = mock(MutableWorldState.class); + private final Tracer.TraceableState worldState = mock(Tracer.TraceableState.class); private final Account account = mock(Account.class); private final MainnetTransactionProcessor transactionProcessor = mock(MainnetTransactionProcessor.class); private final Transaction transaction = mock(Transaction.class); - private final BlockHeader blockHeader = mock(BlockHeader.class); + private final BlockHeader blockHeader = mock(BlockHeader.class, Answers.RETURNS_DEEP_STUBS); private final Hash blockHash = Hash.fromHexString("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); private final Hash transactionHash = @@ -113,10 +118,19 @@ public class DebugStorageRangeAtTest { })); when(blockchainQueries.blockByHash(blockHash)).thenReturn(Optional.of(blockWithMetadata)); + doAnswer( + invocation -> + invocation + .>>getArgument( + 1) + .apply(worldState)) + .when(blockchainQueries) + .getAndMapWorldState(any(), any()); when(blockchainQueries.transactionByBlockHashAndIndex(blockHash, TRANSACTION_INDEX)) .thenReturn(Optional.of(transactionWithMetadata)); when(worldState.get(accountAddress)).thenReturn(account); - when(blockReplay.afterTransactionInBlock(eq(blockHash), eq(transactionHash), any())) + when(blockReplay.afterTransactionInBlock( + any(Tracer.TraceableState.class), any(Hash.class), eq(transactionHash), any())) .thenAnswer(this::callAction); final List entries = new ArrayList<>(); @@ -155,8 +169,7 @@ public class DebugStorageRangeAtTest { private Object callAction(final InvocationOnMock invocation) { //noinspection rawtypes return Optional.of( - ((BlockReplay.TransactionAction) invocation.getArgument(2)) - .performAction( - transaction, blockHeader, blockchain, worldState, transactionProcessor, Wei.ZERO)); + ((BlockReplay.TransactionAction) invocation.getArgument(3)) + .performAction(transaction, blockHeader, blockchain, transactionProcessor, Wei.ZERO)); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHashTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHashTest.java index 4c255ab6f2..253dffb1dc 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHashTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHashTest.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -26,8 +27,13 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; @@ -35,19 +41,41 @@ import java.util.Arrays; import java.util.Collection; import java.util.Optional; import java.util.OptionalLong; +import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; +import org.junit.Before; import org.junit.Test; +import org.mockito.Answers; public class DebugTraceBlockByHashTest { private final BlockTracer blockTracer = mock(BlockTracer.class); + + private final BlockchainQueries blockchainQueries = + mock(BlockchainQueries.class, Answers.RETURNS_DEEP_STUBS); + private final MutableWorldState mutableWorldState = mock(MutableWorldState.class); + private final BlockHeader blockHeader = mock(BlockHeader.class, Answers.RETURNS_DEEP_STUBS); private final DebugTraceBlockByHash debugTraceBlockByHash = - new DebugTraceBlockByHash(() -> blockTracer); + new DebugTraceBlockByHash(() -> blockTracer, () -> blockchainQueries); private final Hash blockHash = Hash.fromHexString("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + @Before + public void setUp() { + doAnswer( + invocation -> + invocation + .>>getArgument( + 1) + .apply(mutableWorldState)) + .when(blockchainQueries) + .getAndMapWorldState(any(), any()); + when(blockchainQueries.getBlockHeaderByHash(any(Hash.class))) + .thenReturn(Optional.of(blockHeader)); + } + @Test public void nameShouldBeDebugTraceBlockByHash() { assertThat(debugTraceBlockByHash.getName()).isEqualTo("debug_traceBlockByHash"); @@ -99,7 +127,8 @@ public class DebugTraceBlockByHashTest { when(transaction2Trace.getResult()).thenReturn(transaction2Result); when(transaction1Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); when(transaction2Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); - when(blockTracer.trace(eq(blockHash), any())).thenReturn(Optional.of(blockTrace)); + when(blockTracer.trace(any(Tracer.TraceableState.class), eq(blockHash), any())) + .thenReturn(Optional.of(blockTrace)); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugTraceBlockByHash.response(request); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumberTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumberTest.java index dc0f67daa0..2b78ec562d 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumberTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumberTest.java @@ -18,7 +18,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -28,26 +28,35 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import java.util.Collection; import java.util.Optional; import java.util.OptionalLong; +import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; import org.junit.Test; +import org.mockito.Answers; public class DebugTraceBlockByNumberTest { - private final BlockchainQueries blockchain = mock(BlockchainQueries.class); - private final BlockTracer blockTracer = mock(BlockTracer.class); + private final BlockchainQueries blockchainQueries = + mock(BlockchainQueries.class, Answers.RETURNS_DEEP_STUBS); + + private final Tracer.TraceableState worldState = mock(Tracer.TraceableState.class); + private final BlockTracer blockTracer = mock(BlockTracer.class, Answers.RETURNS_DEEP_STUBS); private final DebugTraceBlockByNumber debugTraceBlockByNumber = - new DebugTraceBlockByNumber(() -> blockTracer, blockchain); + new DebugTraceBlockByNumber(() -> blockTracer, blockchainQueries); private final Hash blockHash = Hash.fromHexString("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); @@ -104,8 +113,20 @@ public class DebugTraceBlockByNumberTest { when(transaction2Trace.getResult()).thenReturn(transaction2Result); when(transaction1Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); when(transaction2Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); - when(blockchain.getBlockHashByNumber(blockNumber)).thenReturn(Optional.of(blockHash)); - when(blockTracer.trace(eq(blockHash), any())).thenReturn(Optional.of(blockTrace)); + when(blockchainQueries.getBlockHashByNumber(blockNumber)).thenReturn(Optional.of(blockHash)); + when(blockchainQueries.getBlockHeaderByHash(any(Hash.class))) + .thenReturn(Optional.of(mock(BlockHeader.class, Answers.RETURNS_DEEP_STUBS))); + + doAnswer( + invocation -> + invocation + .>> + getArgument(1) + .apply(worldState)) + .when(blockchainQueries) + .getAndMapWorldState(any(), any()); + when(blockTracer.trace(any(Tracer.TraceableState.class), any(Hash.class), any())) + .thenReturn(Optional.of(blockTrace)); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugTraceBlockByNumber.response(request); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java index cf7256435b..df337f44f2 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java @@ -18,7 +18,9 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Wei; @@ -26,17 +28,20 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockWithMetadata; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.Collection; import java.util.Collections; @@ -45,12 +50,17 @@ import java.util.OptionalLong; import org.apache.tuweni.bytes.Bytes; import org.junit.Test; +import org.mockito.Answers; import org.mockito.Mockito; public class DebugTraceBlockTest { private final BlockTracer blockTracer = mock(BlockTracer.class); - private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); + private final WorldStateArchive archive = + mock(WorldStateArchive.class, Answers.RETURNS_DEEP_STUBS); + private final Blockchain blockchain = mock(Blockchain.class); + private final BlockchainQueries blockchainQueries = + spy(new BlockchainQueries(blockchain, archive)); private final DebugTraceBlock debugTraceBlock = new DebugTraceBlock(() -> blockTracer, new MainnetBlockHeaderFunctions(), blockchainQueries); @@ -117,17 +127,22 @@ public class DebugTraceBlockTest { when(transaction2Trace.getResult()).thenReturn(transaction2Result); when(transaction1Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); when(transaction2Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); - when(blockTracer.trace(Mockito.eq(block), any())).thenReturn(Optional.of(blockTrace)); - - when(blockchainQueries.blockByHash(parentBlock.getHash())) - .thenReturn( - Optional.of( - new BlockWithMetadata<>( - parentBlock.getHeader(), - Collections.emptyList(), - Collections.emptyList(), - parentBlock.getHeader().getDifficulty(), - parentBlock.calculateSize()))); + when(blockTracer.trace(any(Tracer.TraceableState.class), Mockito.eq(block), any())) + .thenReturn(Optional.of(blockTrace)); + + when(blockchain.getBlockHeader(parentBlock.getHash())) + .thenReturn(Optional.of(parentBlock.getHeader())); + doAnswer( + invocation -> + Optional.of( + new BlockWithMetadata<>( + parentBlock.getHeader(), + Collections.emptyList(), + Collections.emptyList(), + parentBlock.getHeader().getDifficulty(), + parentBlock.calculateSize()))) + .when(blockchainQueries) + .blockByHash(parentBlock.getHash()); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugTraceBlock.response(request); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceTransactionTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceTransactionTest.java index afa43a0220..c7c0aca44a 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceTransactionTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceTransactionTest.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -24,13 +25,17 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.StructLog; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.api.query.TransactionWithMetadata; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; @@ -42,17 +47,23 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.junit.Before; import org.junit.Test; +import org.mockito.Answers; public class DebugTraceTransactionTest { - private final BlockchainQueries blockchain = mock(BlockchainQueries.class); + private final BlockchainQueries blockchainQueries = + mock(BlockchainQueries.class, Answers.RETURNS_DEEP_STUBS); + private final BlockHeader blockHeader = mock(BlockHeader.class, Answers.RETURNS_DEEP_STUBS); + private final MutableWorldState mutableWorldState = mock(MutableWorldState.class); private final TransactionTracer transactionTracer = mock(TransactionTracer.class); private final DebugTraceTransaction debugTraceTransaction = - new DebugTraceTransaction(blockchain, transactionTracer); + new DebugTraceTransaction(blockchainQueries, transactionTracer); private final Transaction transaction = mock(Transaction.class); private final Hash blockHash = @@ -60,6 +71,22 @@ public class DebugTraceTransactionTest { private final Hash transactionHash = Hash.fromHexString("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + @Before + public void setup() { + doAnswer(__ -> Optional.of(blockHeader)) + .when(blockchainQueries) + .getBlockHeaderByHash(any(Hash.class)); + + doAnswer( + invocation -> + invocation + .>>getArgument( + 1) + .apply(mutableWorldState)) + .when(blockchainQueries) + .getAndMapWorldState(any(), any()); + } + @Test public void nameShouldBeDebugTraceTransaction() { assertThat(debugTraceTransaction.getName()).isEqualTo("debug_traceTransaction"); @@ -116,11 +143,14 @@ public class DebugTraceTransactionTest { when(transaction.getGasLimit()).thenReturn(100L); when(result.getGasRemaining()).thenReturn(27L); when(result.getOutput()).thenReturn(Bytes.fromHexString("1234")); - when(blockchain.headBlockNumber()).thenReturn(12L); - when(blockchain.transactionByHash(transactionHash)) + when(blockchainQueries.headBlockNumber()).thenReturn(12L); + when(blockchainQueries.transactionByHash(transactionHash)) .thenReturn(Optional.of(transactionWithMetadata)); when(transactionTracer.traceTransaction( - eq(blockHash), eq(transactionHash), any(DebugOperationTracer.class))) + any(Tracer.TraceableState.class), + eq(blockHash), + eq(transactionHash), + any(DebugOperationTracer.class))) .thenReturn(Optional.of(transactionTrace)); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugTraceTransaction.response(request); @@ -180,10 +210,13 @@ public class DebugTraceTransactionTest { when(transaction.getGasLimit()).thenReturn(100L); when(result.getGasRemaining()).thenReturn(27L); when(result.getOutput()).thenReturn(Bytes.fromHexString("1234")); - when(blockchain.headBlockNumber()).thenReturn(12L); - when(blockchain.transactionByHash(transactionHash)).thenReturn(Optional.empty()); + when(blockchainQueries.headBlockNumber()).thenReturn(12L); + when(blockchainQueries.transactionByHash(transactionHash)).thenReturn(Optional.empty()); when(transactionTracer.traceTransaction( - eq(blockHash), eq(transactionHash), any(DebugOperationTracer.class))) + any(Tracer.TraceableState.class), + eq(blockHash), + eq(transactionHash), + any(DebugOperationTracer.class))) .thenReturn(Optional.of(transactionTrace)); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugTraceTransaction.response(request); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetProofTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetProofTest.java index 0aff535b5a..e361a5b688 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetProofTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthGetProofTest.java @@ -17,10 +17,12 @@ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.evm.account.Account.MAX_NONCE; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Address; @@ -53,6 +55,7 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -62,8 +65,13 @@ import org.mockito.quality.Strictness; @MockitoSettings(strictness = Strictness.LENIENT) class EthGetProofTest { @Mock private Blockchain blockchain; - @Mock private BlockchainQueries blockchainQueries; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private WorldStateArchive archive; + @Mock private ChainHead chainHead; + @Mock private BlockHeader blockHeader; + private BlockchainQueries blockchainQueries; private EthGetProof method; private final String JSON_RPC_VERSION = "2.0"; @@ -77,7 +85,14 @@ class EthGetProofTest { @BeforeEach public void setUp() { - method = new EthGetProof(blockchainQueries); + blockchainQueries = spy(new BlockchainQueries(blockchain, archive)); + when(blockchainQueries.getBlockchain()).thenReturn(blockchain); + when(blockchainQueries.headBlockNumber()).thenReturn(14L); + when(blockchain.getChainHead()).thenReturn(chainHead); + when(chainHead.getBlockHeader()).thenReturn(blockHeader); + when(blockHeader.getBlockHash()).thenReturn(Hash.ZERO); + when(blockchain.getBlockHeader(Hash.ZERO)).thenReturn(Optional.of(mock(BlockHeader.class))); + method = spy(new EthGetProof(blockchainQueries)); } @Test @@ -88,11 +103,6 @@ class EthGetProofTest { @Test void errorWhenNoAddressAccountSupplied() { final JsonRpcRequestContext request = requestWithParams(null, null, "latest"); - when(blockchainQueries.getBlockchain()).thenReturn(blockchain); - when(blockchain.getChainHead()).thenReturn(chainHead); - final BlockHeader blockHeader = mock(BlockHeader.class); - when(chainHead.getBlockHeader()).thenReturn(blockHeader); - when(blockHeader.getBlockHash()).thenReturn(Hash.ZERO); Assertions.assertThatThrownBy(() -> method.response(request)) .isInstanceOf(InvalidJsonRpcParameters.class) @@ -102,11 +112,6 @@ class EthGetProofTest { @Test void errorWhenNoStorageKeysSupplied() { final JsonRpcRequestContext request = requestWithParams(address.toString(), null, "latest"); - when(blockchainQueries.getBlockchain()).thenReturn(blockchain); - when(blockchain.getChainHead()).thenReturn(chainHead); - final BlockHeader blockHeader = mock(BlockHeader.class); - when(chainHead.getBlockHeader()).thenReturn(blockHeader); - when(blockHeader.getBlockHash()).thenReturn(Hash.ZERO); Assertions.assertThatThrownBy(() -> method.response(request)) .isInstanceOf(InvalidJsonRpcParameters.class) @@ -124,14 +129,12 @@ class EthGetProofTest { @Test void errorWhenAccountNotFound() { - generateWorldState(); - + when(archive.getAccountProof(any(Hash.class), any(Address.class), any())) + .thenReturn(Optional.empty()); final JsonRpcErrorResponse expectedResponse = new JsonRpcErrorResponse(null, JsonRpcError.NO_ACCOUNT_FOUND); - when(blockchainQueries.headBlockNumber()).thenReturn(14L); - final JsonRpcRequestContext request = requestWithParams( Address.fromHexString("0x0000000000000000000000000000000000000000"), @@ -146,11 +149,9 @@ class EthGetProofTest { @Test void errorWhenWorldStateUnavailable() { - when(blockchainQueries.headBlockNumber()).thenReturn(14L); - when(blockchainQueries.getAndMapWorldState(any(), any())).thenReturn(Optional.empty()); - final JsonRpcErrorResponse expectedResponse = new JsonRpcErrorResponse(null, JsonRpcError.WORLD_STATE_UNAVAILABLE); + when(archive.getMutable(any(BlockHeader.class), anyBoolean())).thenReturn(Optional.empty()); final JsonRpcRequestContext request = requestWithParams( @@ -168,8 +169,6 @@ class EthGetProofTest { final GetProofResult expectedResponse = generateWorldState(); - when(blockchainQueries.headBlockNumber()).thenReturn(14L); - final JsonRpcRequestContext request = requestWithParams( address.toString(), new String[] {storageKey.toString()}, String.valueOf(blockNumber)); @@ -200,9 +199,7 @@ class EthGetProofTest { final Hash storageRoot = Hash.fromHexString("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"); - final WorldStateArchive worldStateArchive = mock(WorldStateArchive.class); - - when(blockchainQueries.getWorldStateArchive()).thenReturn(worldStateArchive); + when(blockchainQueries.getWorldStateArchive()).thenReturn(archive); final StateTrieAccountValue stateTrieAccountValue = mock(StateTrieAccountValue.class); when(stateTrieAccountValue.getBalance()).thenReturn(balance); @@ -225,17 +222,17 @@ class EthGetProofTest { "0x2222222222222222222222222222222222222222222222222222222222222222"))); when(worldStateProof.getStorageValue(storageKey)).thenReturn(UInt256.ZERO); - when(worldStateArchive.getAccountProof(eq(rootHash), eq(address), anyList())) + when(archive.getAccountProof(eq(rootHash), eq(address), anyList())) .thenReturn(Optional.of(worldStateProof)); final MutableWorldState mutableWorldState = mock(MutableWorldState.class); when(mutableWorldState.rootHash()).thenReturn(rootHash); doAnswer( invocation -> - Optional.of( - invocation - .>getArgument(1) - .apply(mutableWorldState))) + invocation + .>>getArgument( + 1) + .apply(mutableWorldState)) .when(blockchainQueries) .getAndMapWorldState(any(), any()); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracerTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracerTest.java index 0728e53f2f..e2124a67c3 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracerTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracerTest.java @@ -27,7 +27,6 @@ import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; @@ -36,7 +35,6 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.tracing.StandardJsonTracer; import org.hyperledger.besu.evm.worldstate.StackedUpdater; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -67,8 +65,6 @@ public class TransactionTracerTest { @Mock private ProtocolSchedule protocolSchedule; @Mock private Blockchain blockchain; - @Mock private WorldStateArchive worldStateArchive; - @Mock private BlockHeader blockHeader; @Mock private BlockBody blockBody; @@ -83,7 +79,7 @@ public class TransactionTracerTest { @Mock private ProtocolSpec protocolSpec; - @Mock private MutableWorldState mutableWorldState; + @Mock private Tracer.TraceableState mutableWorldState; @Mock private MainnetTransactionProcessor transactionProcessor; @@ -104,29 +100,25 @@ public class TransactionTracerTest { @Before public void setUp() throws Exception { - transactionTracer = - new TransactionTracer(new BlockReplay(protocolSchedule, blockchain, worldStateArchive)); + transactionTracer = new TransactionTracer(new BlockReplay(protocolSchedule, blockchain)); when(transaction.getHash()).thenReturn(transactionHash); when(otherTransaction.getHash()).thenReturn(otherTransactionHash); when(blockHeader.getNumber()).thenReturn(12L); when(blockHeader.getHash()).thenReturn(blockHash); when(blockHeader.getParentHash()).thenReturn(previousBlockHash); - when(previousBlockHeader.getStateRoot()).thenReturn(Hash.ZERO); - when(worldStateArchive.getMutable(Hash.ZERO, null, false)) - .thenReturn(Optional.of(mutableWorldState)); when(protocolSchedule.getByBlockHeader(blockHeader)).thenReturn(protocolSpec); when(protocolSpec.getTransactionProcessor()).thenReturn(transactionProcessor); when(protocolSpec.getMiningBeneficiaryCalculator()).thenReturn(BlockHeader::getCoinbase); when(protocolSpec.getFeeMarket()).thenReturn(FeeMarket.london(0L)); when(blockchain.getChainHeadHeader()).thenReturn(blockHeader); when(protocolSpec.getBadBlocksManager()).thenReturn(new BadBlockManager()); - when(mutableWorldState.copy()).thenReturn(mutableWorldState); } @Test public void traceTransactionShouldReturnNoneWhenBlockHeaderNotFound() { final Optional transactionTrace = - transactionTracer.traceTransaction(invalidBlockHash, transactionHash, tracer); + transactionTracer.traceTransaction( + mutableWorldState, invalidBlockHash, transactionHash, tracer); assertThat(transactionTrace).isEmpty(); } @@ -140,7 +132,7 @@ public class TransactionTracerTest { when(tracer.getTraceFrames()).thenReturn(traceFrames); final Optional transactionTrace = - transactionTracer.traceTransaction(blockHash, transactionHash, tracer); + transactionTracer.traceTransaction(mutableWorldState, blockHash, transactionHash, tracer); assertThat(transactionTrace.map(TransactionTrace::getTraceFrames)).contains(traceFrames); } @@ -157,7 +149,7 @@ public class TransactionTracerTest { when(tracer.getTraceFrames()).thenReturn(traceFrames); final Optional transactionTrace = - transactionTracer.traceTransaction(blockHash, transactionHash, tracer); + transactionTracer.traceTransaction(mutableWorldState, blockHash, transactionHash, tracer); assertThat(transactionTrace.map(TransactionTrace::getTraceFrames)).contains(traceFrames); } @@ -188,7 +180,7 @@ public class TransactionTracerTest { .thenReturn(result); final Optional transactionTrace = - transactionTracer.traceTransaction(blockHash, transactionHash, tracer); + transactionTracer.traceTransaction(mutableWorldState, blockHash, transactionHash, tracer); assertThat(transactionTrace.map(TransactionTrace::getResult)).contains(result); } @@ -203,7 +195,7 @@ public class TransactionTracerTest { when(blockchain.getBlockBody(blockHash)).thenReturn(Optional.of(blockBody)); final Optional transactionTrace = - transactionTracer.traceTransaction(blockHash, transactionHash, tracer); + transactionTracer.traceTransaction(mutableWorldState, blockHash, transactionHash, tracer); assertThat(transactionTrace).isEmpty(); } @@ -215,7 +207,7 @@ public class TransactionTracerTest { when(blockchain.getBlockBody(blockHash)).thenReturn(Optional.empty()); final Optional transactionTrace = - transactionTracer.traceTransaction(blockHash, transactionHash, tracer); + transactionTracer.traceTransaction(mutableWorldState, blockHash, transactionHash, tracer); assertThat(transactionTrace).isEmpty(); } @@ -235,6 +227,7 @@ public class TransactionTracerTest { when(mutableWorldState.updater()).thenReturn(updater); final List transactionTraces = transactionTracer.traceTransactionToFile( + mutableWorldState, blockHash, Optional.of(ImmutableTransactionTraceParams.builder().build()), traceDir.getRoot().toPath()); @@ -277,6 +270,7 @@ public class TransactionTracerTest { final List transactionTraces = transactionTracer.traceTransactionToFile( + mutableWorldState, blockHash, Optional.of(ImmutableTransactionTraceParams.builder().build()), traceDir.getRoot().toPath()); diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java index d5377cb8a3..14ff51c11f 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java @@ -318,16 +318,7 @@ public abstract class AbstractBlockCreator implements AsyncBlockCreator { final Hash parentStateRoot = parentHeader.getStateRoot(); return protocolContext .getWorldStateArchive() - .getMutable(parentStateRoot, parentHeader.getHash(), false) - .map( - ws -> { - if (ws.isPersistable()) { - return ws; - } else { - // non-persistable worldstates should return a copy which is persistable: - return ws.copy(); - } - }) + .getMutable(parentHeader, false) .orElseThrow( () -> { LOG.info("Unable to create block because world state is not available"); diff --git a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java index cc1b569e75..56ac024591 100644 --- a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java +++ b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java @@ -41,6 +41,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -53,7 +54,7 @@ public class PrunerIntegrationTest { private final BlockDataGenerator gen = new BlockDataGenerator(); private final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); - private final Map hashValueStore = new HashMap<>(); + private final Map> hashValueStore = new HashMap<>(); private final InMemoryKeyValueStorage stateStorage = new TestInMemoryStorage(hashValueStore); private final WorldStateStorage worldStateStorage = new WorldStateKeyValueStorage(stateStorage); private final WorldStateArchive worldStateArchive = @@ -164,7 +165,7 @@ public class PrunerIntegrationTest { // Check that storage contains only the values we expect assertThat(hashValueStore.size()).isEqualTo(expectedNodes.size()); - assertThat(hashValueStore.values()) + assertThat(hashValueStore.values().stream().map(Optional::get)) .containsExactlyInAnyOrderElementsOf( expectedNodes.stream().map(Bytes::toArrayUnsafe).collect(Collectors.toSet())); } @@ -252,7 +253,7 @@ public class PrunerIntegrationTest { // Proxy class so that we have access to the constructor that takes our own map private static class TestInMemoryStorage extends InMemoryKeyValueStorage { - public TestInMemoryStorage(final Map hashValueStore) { + public TestInMemoryStorage(final Map> hashValueStore) { super(hashValueStore); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java index 1a95033d2a..c2bde19cd3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java @@ -122,19 +122,8 @@ public class MainnetBlockValidator implements BlockValidator { handleAndLogImportFailure(block, retval, shouldRecordBadBlock); return retval; } - try (final var worldState = - context - .getWorldStateArchive() - .getMutable(parentHeader.getStateRoot(), parentHeader.getBlockHash(), shouldPersist) - .map( - ws -> { - if (!ws.isPersistable()) { - return ws.copy(); - } - return ws; - }) - .orElse(null)) { + context.getWorldStateArchive().getMutable(parentHeader, shouldPersist).orElse(null)) { if (worldState == null) { var retval = diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java index b8133714a0..978dcf5df3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java @@ -19,6 +19,7 @@ package org.hyperledger.besu.ethereum.bonsai; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldView; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPException; @@ -53,7 +54,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount { private final Map updatedStorage = new HashMap<>(); - BonsaiAccount( + public BonsaiAccount( final BonsaiWorldView context, final Address address, final Hash addressHash, @@ -73,7 +74,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount { this.mutable = mutable; } - BonsaiAccount( + public BonsaiAccount( final BonsaiWorldView context, final Address address, final StateTrieAccountValue stateTrieAccount, @@ -89,11 +90,12 @@ public class BonsaiAccount implements MutableAccount, EvmAccount { mutable); } - BonsaiAccount(final BonsaiAccount toCopy) { + public BonsaiAccount(final BonsaiAccount toCopy) { this(toCopy, toCopy.context, false); } - BonsaiAccount(final BonsaiAccount toCopy, final BonsaiWorldView context, final boolean mutable) { + public BonsaiAccount( + final BonsaiAccount toCopy, final BonsaiWorldView context, final boolean mutable) { this.context = context; this.address = toCopy.address; this.addressHash = toCopy.addressHash; @@ -107,7 +109,8 @@ public class BonsaiAccount implements MutableAccount, EvmAccount { this.mutable = mutable; } - BonsaiAccount(final BonsaiWorldView context, final UpdateTrackingAccount tracked) { + public BonsaiAccount( + final BonsaiWorldView context, final UpdateTrackingAccount tracked) { this.context = context; this.address = tracked.getAddress(); this.addressHash = tracked.getAddressHash(); @@ -121,7 +124,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount { this.mutable = true; } - static BonsaiAccount fromRLP( + public static BonsaiAccount fromRLP( final BonsaiWorldView context, final Address address, final Bytes encoded, @@ -219,7 +222,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount { throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); } - Bytes serializeAccount() { + public Bytes serializeAccount() { final BytesValueRLPOutput out = new BytesValueRLPOutput(); out.startList(); @@ -294,7 +297,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount { * @param context a description to be added to the thrown exceptions * @throws IllegalStateException if the stored values differ */ - static void assertCloseEnoughForDiffing( + public static void assertCloseEnoughForDiffing( final BonsaiAccount source, final StateTrieAccountValue account, final String context) { if (source == null) { throw new IllegalStateException(context + ": source is null but target isn't"); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java deleted file mode 100644 index 7c0c2e74ec..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldState.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.hyperledger.besu.ethereum.bonsai; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; - -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.units.bigints.UInt256; - -public class BonsaiInMemoryWorldState extends BonsaiPersistedWorldState - implements BonsaiStorageSubscriber { - - private boolean isPersisted = false; - private final Long worldstateSubcriberId; - - public BonsaiInMemoryWorldState( - final BonsaiWorldStateArchive archive, - final BonsaiWorldStateKeyValueStorage worldStateStorage) { - super(archive, worldStateStorage); - worldstateSubcriberId = worldStateStorage.subscribe(this); - } - - @Override - public Hash rootHash() { - if (isPersisted) { - return worldStateRootHash; - } - return rootHash(updater.copy()); - } - - public Hash rootHash(final BonsaiWorldStateUpdater localUpdater) { - final Hash calculatedRootHash = calculateRootHash(localUpdater); - return Hash.wrap(calculatedRootHash); - } - - protected Hash calculateRootHash(final BonsaiWorldStateUpdater worldStateUpdater) { - - // second update account storage state. This must be done before updating the accounts so - // that we can get the storage state hash - - worldStateUpdater.getStorageToUpdate().entrySet().parallelStream() - .forEach( - addressMapEntry -> { - updateAccountStorage(worldStateUpdater, addressMapEntry); - }); - - // next walk the account trie - final StoredMerklePatriciaTrie accountTrie = - new StoredMerklePatriciaTrie<>( - (location, hash) -> - archive - .getCachedMerkleTrieLoader() - .getAccountStateTrieNode(worldStateStorage, location, hash), - worldStateRootHash, - Function.identity(), - Function.identity()); - - // for manicured tries and composting, collect branches here (not implemented) - - // now add the accounts - for (final Map.Entry> accountUpdate : - worldStateUpdater.getAccountsToUpdate().entrySet()) { - final Bytes accountKey = accountUpdate.getKey(); - final BonsaiValue bonsaiValue = accountUpdate.getValue(); - final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); - try { - if (updatedAccount == null) { - final Hash addressHash = Hash.hash(accountKey); - accountTrie.remove(addressHash); - } else { - final Hash addressHash = updatedAccount.getAddressHash(); - final Bytes accountValue = updatedAccount.serializeAccount(); - accountTrie.put(addressHash, accountValue); - } - } catch (MerkleTrieException e) { - // need to throw to trigger the heal - throw new MerkleTrieException( - e.getMessage(), Optional.of(Address.wrap(accountKey)), e.getHash(), e.getLocation()); - } - } - - // TODO write to a cache and then generate a layer update from that and the - // DB tx updates. Right now it is just DB updates. - return Hash.wrap(accountTrie.getRootHash()); - } - - private void updateAccountStorage( - final BonsaiWorldStateUpdater worldStateUpdater, - final Map.Entry>> - storageAccountUpdate) { - final Address updatedAddress = storageAccountUpdate.getKey(); - final Hash updatedAddressHash = Hash.hash(updatedAddress); - if (worldStateUpdater.getAccountsToUpdate().containsKey(updatedAddress)) { - final BonsaiValue accountValue = - worldStateUpdater.getAccountsToUpdate().get(updatedAddress); - final BonsaiAccount accountOriginal = accountValue.getPrior(); - final Hash storageRoot = - (accountOriginal == null) ? Hash.EMPTY_TRIE_HASH : accountOriginal.getStorageRoot(); - - final StoredMerklePatriciaTrie storageTrie = - new StoredMerklePatriciaTrie<>( - (location, key) -> - archive - .getCachedMerkleTrieLoader() - .getAccountStorageTrieNode( - worldStateStorage, updatedAddressHash, location, key), - storageRoot, - Function.identity(), - Function.identity()); - - // for manicured tries and composting, collect branches here (not implemented) - - for (final Map.Entry> storageUpdate : - storageAccountUpdate.getValue().entrySet()) { - final Hash keyHash = storageUpdate.getKey(); - final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); - try { - if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) { - storageTrie.remove(keyHash); - } else { - storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage)); - } - } catch (MerkleTrieException e) { - // need to throw to trigger the heal - throw new MerkleTrieException( - e.getMessage(), - Optional.of(Address.wrap(updatedAddress)), - e.getHash(), - e.getLocation()); - } - } - - final BonsaiAccount accountUpdated = accountValue.getUpdated(); - if (accountUpdated != null) { - final Hash newStorageRoot = Hash.wrap(storageTrie.getRootHash()); - accountUpdated.setStorageRoot(newStorageRoot); - } - } - } - - @Override - public void persist(final BlockHeader blockHeader) { - final BonsaiWorldStateUpdater localUpdater = updater.copy(); - final Hash newWorldStateRootHash = rootHash(localUpdater); - archive - .getTrieLogManager() - .saveTrieLog(archive, localUpdater, newWorldStateRootHash, blockHeader, this); - worldStateRootHash = newWorldStateRootHash; - worldStateBlockHash = blockHeader.getBlockHash(); - isPersisted = true; - } - - @Override - public void close() throws Exception { - // if storage is snapshot-based we need to close: - worldStateStorage.unSubscribe(worldstateSubcriberId); - worldStateStorage.close(); - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldStateKeyValueStorage.java deleted file mode 100644 index f539b899da..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryWorldStateKeyValueStorage.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.bonsai; - -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BonsaiInMemoryWorldStateKeyValueStorage extends BonsaiWorldStateKeyValueStorage - implements WorldStateStorage { - - private static final Logger LOG = - LoggerFactory.getLogger(BonsaiInMemoryWorldStateKeyValueStorage.class); - - public BonsaiInMemoryWorldStateKeyValueStorage( - final KeyValueStorage accountStorage, - final KeyValueStorage codeStorage, - final KeyValueStorage storageStorage, - final KeyValueStorage trieBranchStorage, - final KeyValueStorage trieLogStorage) { - super(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage); - } - - @Override - public InMemoryUpdater updater() { - return new InMemoryUpdater( - accountStorage.startTransaction(), - codeStorage.startTransaction(), - storageStorage.startTransaction(), - trieBranchStorage.startTransaction(), - trieLogStorage.startTransaction()); - } - - public static class InMemoryUpdater extends BonsaiWorldStateKeyValueStorage.Updater - implements WorldStateStorage.Updater { - - public InMemoryUpdater( - final KeyValueStorageTransaction accountStorageTransaction, - final KeyValueStorageTransaction codeStorageTransaction, - final KeyValueStorageTransaction storageStorageTransaction, - final KeyValueStorageTransaction trieBranchStorageTransaction, - final KeyValueStorageTransaction trieLogStorageTransaction) { - super( - accountStorageTransaction, - codeStorageTransaction, - storageStorageTransaction, - trieBranchStorageTransaction, - trieLogStorageTransaction); - } - - @Override - public void commit() { - LOG.trace("Cannot commit using an in memory key value storage"); - } - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java deleted file mode 100644 index 4a45def378..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.hyperledger.besu.ethereum.bonsai; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.core.SnapshotMutableWorldState; -import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.evm.account.Account; -import org.hyperledger.besu.evm.worldstate.WorldState; -import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.plugin.services.exception.StorageException; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Stream; - -import com.google.errorprone.annotations.MustBeClosed; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; - -/** A World State backed first by trie log layer and then by another world state. */ -public class BonsaiLayeredWorldState - implements MutableWorldState, - BonsaiWorldView, - WorldState, - BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { - private Optional nextWorldView = Optional.empty(); - private Optional newtWorldViewSubscribeId = Optional.empty(); - protected final long height; - protected final TrieLogLayer trieLog; - private final Hash worldStateRootHash; - - private final Blockchain blockchain; - private final BonsaiWorldStateArchive archive; - - BonsaiLayeredWorldState( - final Blockchain blockchain, - final BonsaiWorldStateArchive archive, - final Optional nextWorldView, - final long height, - final Hash worldStateRootHash, - final TrieLogLayer trieLog) { - this.blockchain = blockchain; - this.archive = archive; - this.setNextWorldView(nextWorldView); - this.height = height; - this.worldStateRootHash = worldStateRootHash; - this.trieLog = trieLog; - } - - public Optional getNextWorldView() { - if (nextWorldView.isEmpty()) { - final Optional blockHashByNumber = blockchain.getBlockHashByNumber(height + 1); - nextWorldView = - blockHashByNumber - .map(hash -> archive.getMutable(null, hash, false).map(BonsaiWorldView.class::cast)) - .orElseGet(() -> Optional.of(archive.getMutable()).map(BonsaiWorldView.class::cast)); - } - return nextWorldView; - } - - public void setNextWorldView(final Optional nextWorldView) { - maybeUnSubscribe(); // unsubscribe the old view - this.nextWorldView = nextWorldView; - maybeSubscribe(); // subscribe the next view - } - - private void maybeSubscribe() { - nextWorldView - .filter(BonsaiPersistedWorldState.class::isInstance) - .map(BonsaiPersistedWorldState.class::cast) - .ifPresent( - worldState -> { - newtWorldViewSubscribeId = Optional.of(worldState.worldStateStorage.subscribe(this)); - }); - } - - private void maybeUnSubscribe() { - nextWorldView - .filter(BonsaiPersistedWorldState.class::isInstance) - .map(BonsaiPersistedWorldState.class::cast) - .ifPresent( - worldState -> { - newtWorldViewSubscribeId.ifPresent(worldState.worldStateStorage::unSubscribe); - }); - } - - @Override - public void close() throws Exception { - maybeUnSubscribe(); - } - - public TrieLogLayer getTrieLog() { - return trieLog; - } - - public long getHeight() { - return height; - } - - @Override - public Optional getCode(final Address address, final Hash codeHash) { - BonsaiLayeredWorldState currentLayer = this; - while (currentLayer != null) { - final Optional maybeCode = currentLayer.trieLog.getCode(address); - final Optional maybePriorCode = currentLayer.trieLog.getPriorCode(address); - if (currentLayer == this && maybeCode.isPresent()) { - return maybeCode; - } else if (maybePriorCode.isPresent()) { - return maybePriorCode; - } else if (maybeCode.isPresent()) { - return Optional.empty(); - } - if (currentLayer.getNextWorldView().isEmpty()) { - currentLayer = null; - } else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) { - currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get(); - } else { - return currentLayer.getNextWorldView().get().getCode(address, codeHash); - } - } - return Optional.empty(); - } - - @Override - public Optional getStateTrieNode(final Bytes location) { - // this must be iterative and lambda light because the stack may blow up - // mainly because we don't have tail calls. - BonsaiLayeredWorldState currentLayer = this; - while (currentLayer != null) { - if (currentLayer.getNextWorldView().isEmpty()) { - currentLayer = null; - } else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) { - currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get(); - } else { - return currentLayer.getNextWorldView().get().getStateTrieNode(location); - } - } - return Optional.empty(); - } - - @Override - public UInt256 getStorageValue(final Address address, final UInt256 key) { - return getStorageValueBySlotHash(address, Hash.hash(key)).orElse(UInt256.ZERO); - } - - @Override - public Optional getStorageValueBySlotHash(final Address address, final Hash slotHash) { - // this must be iterative and lambda light because the stack may blow up - // mainly because we don't have tail calls. - BonsaiLayeredWorldState currentLayer = this; - while (currentLayer != null) { - final Optional maybeValue = - currentLayer.trieLog.getStorageBySlotHash(address, slotHash); - final Optional maybePriorValue = - currentLayer.trieLog.getPriorStorageBySlotHash(address, slotHash); - if (currentLayer == this && maybeValue.isPresent()) { - return maybeValue; - } else if (maybePriorValue.isPresent()) { - return maybePriorValue; - } else if (maybeValue.isPresent()) { - return Optional.empty(); - } - if (currentLayer.getNextWorldView().isEmpty()) { - currentLayer = null; - } else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) { - currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get(); - } else { - return currentLayer.getNextWorldView().get().getStorageValueBySlotHash(address, slotHash); - } - } - return Optional.empty(); - } - - @Override - public UInt256 getPriorStorageValue(final Address address, final UInt256 key) { - // This is the base layer for a block, all values are original. - return getStorageValue(address, key); - } - - @Override - public Map getAllAccountStorage(final Address address, final Hash rootHash) { - // this must be iterative and lambda light because the stack may blow up - // mainly because we don't have tail calls. - final Map results = new HashMap<>(); - BonsaiLayeredWorldState currentLayer = this; - while (currentLayer != null) { - if (currentLayer.trieLog.hasStorageChanges(address)) { - currentLayer - .trieLog - .streamStorageChanges(address) - .forEach( - entry -> { - if (!results.containsKey(entry.getKey())) { - final UInt256 value = entry.getValue().getUpdated(); - // yes, store the nulls. If it was deleted it should stay deleted - results.put(entry.getKey(), value); - } - }); - } - if (currentLayer.getNextWorldView().isEmpty()) { - currentLayer = null; - } else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) { - currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get(); - } else { - final Account account = currentLayer.getNextWorldView().get().get(address); - if (account != null) { - account - .storageEntriesFrom(Hash.ZERO, Integer.MAX_VALUE) - .forEach( - (k, v) -> { - if (!results.containsKey(k)) { - results.put(k, v.getValue()); - } - }); - } - currentLayer = null; - } - } - return results; - } - - @Override - public Account get(final Address address) { - // this must be iterative and lambda light because the stack may blow up - // mainly because we don't have tail calls. - BonsaiLayeredWorldState currentLayer = this; - while (currentLayer != null) { - final Optional maybeStateTrieAccount = - currentLayer.trieLog.getAccount(address); - final Optional maybePriorStateTrieAccount = - currentLayer.trieLog.getPriorAccount(address); - if (currentLayer == this && maybeStateTrieAccount.isPresent()) { - return new BonsaiAccount( - BonsaiLayeredWorldState.this, address, maybeStateTrieAccount.get(), false); - } else if (maybePriorStateTrieAccount.isPresent()) { - return new BonsaiAccount( - BonsaiLayeredWorldState.this, address, maybePriorStateTrieAccount.get(), false); - } else if (maybeStateTrieAccount.isPresent()) { - return null; - } - if (currentLayer.getNextWorldView().isEmpty()) { - currentLayer = null; - } else if (currentLayer.getNextWorldView().get() instanceof BonsaiLayeredWorldState) { - currentLayer = (BonsaiLayeredWorldState) currentLayer.getNextWorldView().get(); - } else { - return currentLayer.getNextWorldView().get().get(address); - } - } - return null; - } - - @Override - public Hash rootHash() { - return worldStateRootHash; - } - - @Override - public Hash frontierRootHash() { - // maybe throw? - return rootHash(); - } - - public Hash blockHash() { - return trieLog.getBlockHash(); - } - - @Override - public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { - throw new UnsupportedOperationException("Bonsai does not support pruning and debug RPCs"); - } - - @Override - @MustBeClosed - public MutableWorldState copy() { - // return an in-memory worldstate that is based on a persisted snapshot for this blockhash. - try (SnapshotMutableWorldState snapshot = - archive - .getMutableSnapshot(this.blockHash()) - .map(SnapshotMutableWorldState.class::cast) - .orElseThrow( - () -> - new StorageException( - "Unable to copy Layered Worldstate for " + blockHash().toHexString()))) { - return new BonsaiInMemoryWorldState(archive, snapshot.getWorldStateStorage()); - } catch (MerkleTrieException ex) { - throw ex; // need to throw to trigger the heal - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - @Override - public boolean isPersistable() { - return false; - } - - @Override - public void persist(final BlockHeader blockHeader) { - // no-op, layered worldstates do not persist, not even as a trielog. - } - - @Override - public WorldUpdater updater() { - return new BonsaiWorldStateUpdater(this); - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java deleted file mode 100644 index 3a3db38798..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.hyperledger.besu.ethereum.bonsai; - -import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.evm.account.Account; -import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.plugin.services.exception.StorageException; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; - -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Stream; -import javax.annotation.Nonnull; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorldView { - - private static final Logger LOG = LoggerFactory.getLogger(BonsaiPersistedWorldState.class); - - protected final BonsaiWorldStateKeyValueStorage worldStateStorage; - - protected final BonsaiWorldStateArchive archive; - protected final BonsaiWorldStateUpdater updater; - - protected Hash worldStateRootHash; - protected Hash worldStateBlockHash; - - public BonsaiPersistedWorldState( - final BonsaiWorldStateArchive archive, - final BonsaiWorldStateKeyValueStorage worldStateStorage) { - this.archive = archive; - this.worldStateStorage = worldStateStorage; - worldStateRootHash = - Hash.wrap( - Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); - worldStateBlockHash = - Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - updater = - new BonsaiWorldStateUpdater( - this, - (addr, value) -> - archive - .getCachedMerkleTrieLoader() - .preLoadAccount(getWorldStateStorage(), worldStateRootHash, addr), - (addr, value) -> - archive - .getCachedMerkleTrieLoader() - .preLoadStorageSlot(getWorldStateStorage(), addr, value)); - } - - public BonsaiWorldStateArchive getArchive() { - return archive; - } - - @Override - public MutableWorldState copy() { - BonsaiInMemoryWorldStateKeyValueStorage bonsaiInMemoryWorldStateKeyValueStorage = - new BonsaiInMemoryWorldStateKeyValueStorage( - worldStateStorage.accountStorage, - worldStateStorage.codeStorage, - worldStateStorage.storageStorage, - worldStateStorage.trieBranchStorage, - worldStateStorage.trieLogStorage); - - return new BonsaiInMemoryWorldState(archive, bonsaiInMemoryWorldStateKeyValueStorage); - } - - @Override - public Optional getCode(@Nonnull final Address address, final Hash codeHash) { - return worldStateStorage.getCode(codeHash, Hash.hash(address)); - } - - public void setArchiveStateUnSafe(final BlockHeader blockHeader) { - worldStateBlockHash = Hash.fromPlugin(blockHeader.getBlockHash()); - worldStateRootHash = Hash.fromPlugin(blockHeader.getStateRoot()); - } - - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { - return worldStateStorage; - } - - protected Hash calculateRootHash( - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, - final BonsaiWorldStateUpdater worldStateUpdater) { - clearStorage(stateUpdater, worldStateUpdater); - - // This must be done before updating the accounts so - // that we can get the storage state hash - updateAccountStorageState(stateUpdater, worldStateUpdater); - - // Third update the code. This has the side effect of ensuring a code hash is calculated. - updateCode(stateUpdater, worldStateUpdater); - - // next walk the account trie - final StoredMerklePatriciaTrie accountTrie = - new StoredMerklePatriciaTrie<>( - (location, hash) -> - archive - .getCachedMerkleTrieLoader() - .getAccountStateTrieNode(worldStateStorage, location, hash), - worldStateRootHash, - Function.identity(), - Function.identity()); - - // for manicured tries and composting, collect branches here (not implemented) - - addTheAccounts(stateUpdater, worldStateUpdater, accountTrie); - - // TODO write to a cache and then generate a layer update from that and the - // DB tx updates. Right now it is just DB updates. - accountTrie.commit( - (location, hash, value) -> - writeTrieNode(stateUpdater.getTrieBranchStorageTransaction(), location, value)); - final Bytes32 rootHash = accountTrie.getRootHash(); - return Hash.wrap(rootHash); - } - - private void addTheAccounts( - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, - final BonsaiWorldStateUpdater worldStateUpdater, - final StoredMerklePatriciaTrie accountTrie) { - for (final Map.Entry> accountUpdate : - worldStateUpdater.getAccountsToUpdate().entrySet()) { - final Bytes accountKey = accountUpdate.getKey(); - final BonsaiValue bonsaiValue = accountUpdate.getValue(); - final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); - try { - if (updatedAccount == null) { - final Hash addressHash = Hash.hash(accountKey); - accountTrie.remove(addressHash); - stateUpdater.removeAccountInfoState(addressHash); - } else { - final Hash addressHash = updatedAccount.getAddressHash(); - final Bytes accountValue = updatedAccount.serializeAccount(); - stateUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue); - accountTrie.put(addressHash, accountValue); - } - } catch (MerkleTrieException e) { - // need to throw to trigger the heal - throw new MerkleTrieException( - e.getMessage(), Optional.of(Address.wrap(accountKey)), e.getHash(), e.getLocation()); - } - } - } - - private void updateCode( - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, - final BonsaiWorldStateUpdater worldStateUpdater) { - for (final Map.Entry> codeUpdate : - worldStateUpdater.getCodeToUpdate().entrySet()) { - final Bytes updatedCode = codeUpdate.getValue().getUpdated(); - final Hash accountHash = Hash.hash(codeUpdate.getKey()); - if (updatedCode == null || updatedCode.size() == 0) { - stateUpdater.removeCode(accountHash); - } else { - stateUpdater.putCode(accountHash, null, updatedCode); - } - } - } - - private void updateAccountStorageState( - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, - final BonsaiWorldStateUpdater worldStateUpdater) { - for (final Map.Entry>> - storageAccountUpdate : worldStateUpdater.getStorageToUpdate().entrySet()) { - final Address updatedAddress = storageAccountUpdate.getKey(); - final Hash updatedAddressHash = Hash.hash(updatedAddress); - if (worldStateUpdater.getAccountsToUpdate().containsKey(updatedAddress)) { - final BonsaiValue accountValue = - worldStateUpdater.getAccountsToUpdate().get(updatedAddress); - final BonsaiAccount accountOriginal = accountValue.getPrior(); - final Hash storageRoot = - (accountOriginal == null) ? Hash.EMPTY_TRIE_HASH : accountOriginal.getStorageRoot(); - final StoredMerklePatriciaTrie storageTrie = - new StoredMerklePatriciaTrie<>( - (location, key) -> - archive - .getCachedMerkleTrieLoader() - .getAccountStorageTrieNode( - worldStateStorage, updatedAddressHash, location, key), - storageRoot, - Function.identity(), - Function.identity()); - - // for manicured tries and composting, collect branches here (not implemented) - - for (final Map.Entry> storageUpdate : - storageAccountUpdate.getValue().entrySet()) { - final Hash keyHash = storageUpdate.getKey(); - final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); - try { - if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) { - stateUpdater.removeStorageValueBySlotHash(updatedAddressHash, keyHash); - storageTrie.remove(keyHash); - } else { - stateUpdater.putStorageValueBySlotHash(updatedAddressHash, keyHash, updatedStorage); - storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage)); - } - } catch (MerkleTrieException e) { - // need to throw to trigger the heal - throw new MerkleTrieException( - e.getMessage(), - Optional.of(Address.wrap(updatedAddress)), - e.getHash(), - e.getLocation()); - } - } - - final BonsaiAccount accountUpdated = accountValue.getUpdated(); - if (accountUpdated != null) { - storageTrie.commit( - (location, key, value) -> - writeStorageTrieNode(stateUpdater, updatedAddressHash, location, key, value)); - final Hash newStorageRoot = Hash.wrap(storageTrie.getRootHash()); - accountUpdated.setStorageRoot(newStorageRoot); - } - } - // for manicured tries and composting, trim and compost here - } - } - - private void clearStorage( - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater, - final BonsaiWorldStateUpdater worldStateUpdater) { - for (final Address address : worldStateUpdater.getStorageToClear()) { - // because we are clearing persisted values we need the account root as persisted - final BonsaiAccount oldAccount = - worldStateStorage - .getAccount(Hash.hash(address)) - .map(bytes -> fromRLP(BonsaiPersistedWorldState.this, address, bytes, true)) - .orElse(null); - if (oldAccount == null) { - // This is when an account is both created and deleted within the scope of the same - // block. A not-uncommon DeFi bot pattern. - continue; - } - final Hash addressHash = Hash.hash(address); - final StoredMerklePatriciaTrie storageTrie = - new StoredMerklePatriciaTrie<>( - (location, key) -> getStorageTrieNode(addressHash, location, key), - oldAccount.getStorageRoot(), - Function.identity(), - Function.identity()); - try { - Map entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); - while (!entriesToDelete.isEmpty()) { - entriesToDelete - .keySet() - .forEach( - k -> stateUpdater.removeStorageValueBySlotHash(Hash.hash(address), Hash.wrap(k))); - entriesToDelete.keySet().forEach(storageTrie::remove); - if (entriesToDelete.size() == 256) { - entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); - } else { - break; - } - } - } catch (MerkleTrieException e) { - // need to throw to trigger the heal - throw new MerkleTrieException( - e.getMessage(), Optional.of(Address.wrap(address)), e.getHash(), e.getLocation()); - } - } - } - - @Override - public void persist(final BlockHeader blockHeader) { - final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); - LOG.atDebug() - .setMessage("Persist world state for block {}") - .addArgument(maybeBlockHeader) - .log(); - boolean success = false; - - final BonsaiWorldStateUpdater localUpdater = updater.copy(); - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater = worldStateStorage.updater(); - Runnable saveTrieLog = () -> {}; - - try { - final Hash newWorldStateRootHash = calculateRootHash(stateUpdater, localUpdater); - // if we are persisted with a block header, and the prior state is the parent - // then persist the TrieLog for that transition. - // If specified but not a direct descendant simply store the new block hash. - if (blockHeader != null) { - if (!newWorldStateRootHash.equals(blockHeader.getStateRoot())) { - throw new RuntimeException( - "World State Root does not match expected value, header " - + blockHeader.getStateRoot().toHexString() - + " calculated " - + newWorldStateRootHash.toHexString()); - } - saveTrieLog = - () -> - archive - .getTrieLogManager() - .saveTrieLog(archive, localUpdater, newWorldStateRootHash, blockHeader, this); - - stateUpdater - .getTrieBranchStorageTransaction() - .put(WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); - worldStateBlockHash = blockHeader.getHash(); - } else { - stateUpdater.getTrieBranchStorageTransaction().remove(WORLD_BLOCK_HASH_KEY); - worldStateBlockHash = null; - } - - stateUpdater - .getTrieBranchStorageTransaction() - .put(WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); - worldStateRootHash = newWorldStateRootHash; - success = true; - } finally { - if (success) { - stateUpdater.commit(); - updater.reset(); - saveTrieLog.run(); - } else { - stateUpdater.rollback(); - updater.reset(); - } - } - } - - @Override - public WorldUpdater updater() { - return updater; - } - - @Override - public Hash rootHash() { - return Hash.wrap(worldStateRootHash); - } - - static final KeyValueStorageTransaction noOpTx = - new KeyValueStorageTransaction() { - - @Override - public void put(final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - - @Override - public Hash frontierRootHash() { - return calculateRootHash( - new BonsaiWorldStateKeyValueStorage.Updater(noOpTx, noOpTx, noOpTx, noOpTx, noOpTx), - updater.copy()); - } - - public Hash blockHash() { - return worldStateBlockHash; - } - - @Override - public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { - throw new RuntimeException("Bonsai Tries do not provide account streaming."); - } - - @Override - public Account get(final Address address) { - return worldStateStorage - .getAccount(Hash.hash(address)) - .map(bytes -> fromRLP(updater, address, bytes, true)) - .orElse(null); - } - - protected Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { - return worldStateStorage.getAccountStateTrieNode(location, nodeHash); - } - - private void writeTrieNode( - final KeyValueStorageTransaction tx, final Bytes location, final Bytes value) { - tx.put(location.toArrayUnsafe(), value.toArrayUnsafe()); - } - - protected Optional getStorageTrieNode( - final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { - return worldStateStorage.getAccountStorageTrieNode(accountHash, location, nodeHash); - } - - private void writeStorageTrieNode( - final WorldStateStorage.Updater stateUpdater, - final Hash accountHash, - final Bytes location, - final Bytes32 nodeHash, - final Bytes value) { - stateUpdater.putAccountStorageTrieNode(accountHash, location, nodeHash, value); - } - - @Override - public Optional getStateTrieNode(final Bytes location) { - return worldStateStorage.getStateTrieNode(location); - } - - @Override - public UInt256 getStorageValue(final Address address, final UInt256 storageKey) { - return getStorageValueBySlotHash(address, Hash.hash(storageKey)).orElse(UInt256.ZERO); - } - - @Override - public Optional getStorageValueBySlotHash(final Address address, final Hash slotHash) { - return worldStateStorage - .getStorageValueBySlotHash(Hash.hash(address), slotHash) - .map(UInt256::fromBytes); - } - - public Optional getStorageValueBySlotHash( - final Supplier> storageRootSupplier, - final Address address, - final Hash slotHash) { - return worldStateStorage - .getStorageValueBySlotHash(storageRootSupplier, Hash.hash(address), slotHash) - .map(UInt256::fromBytes); - } - - @Override - public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { - return getStorageValue(address, storageKey); - } - - @Override - public Map getAllAccountStorage(final Address address, final Hash rootHash) { - final StoredMerklePatriciaTrie storageTrie = - new StoredMerklePatriciaTrie<>( - (location, key) -> getStorageTrieNode(Hash.hash(address), location, key), - rootHash, - Function.identity(), - Function.identity()); - return storageTrie.entriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldState.java deleted file mode 100644 index 0353b59c2f..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldState.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ -package org.hyperledger.besu.ethereum.bonsai; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.core.SnapshotMutableWorldState; -import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; -import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; - -/** - * This class extends BonsaiPersistedWorldstate directly such that it commits/perists directly to - * the transaction state. A SnapshotMutableWorldState is used to accumulate changes to a - * non-persisting mutable world state rather than writing worldstate changes directly. - */ -public class BonsaiSnapshotWorldState extends BonsaiPersistedWorldState - implements SnapshotMutableWorldState { - - private final SnappedKeyValueStorage accountSnap; - private final SnappedKeyValueStorage codeSnap; - private final SnappedKeyValueStorage storageSnap; - private final SnappedKeyValueStorage trieBranchSnap; - private final BonsaiWorldStateKeyValueStorage parentWorldStateStorage; - private final BonsaiSnapshotWorldStateKeyValueStorage snapshotWorldStateStorage; - - private BonsaiSnapshotWorldState( - final BonsaiWorldStateArchive archive, - final BonsaiSnapshotWorldStateKeyValueStorage snapshotWorldStateStorage, - final BonsaiWorldStateKeyValueStorage parentWorldStateStorage) { - super(archive, snapshotWorldStateStorage); - this.snapshotWorldStateStorage = snapshotWorldStateStorage; - this.accountSnap = (SnappedKeyValueStorage) snapshotWorldStateStorage.accountStorage; - this.codeSnap = (SnappedKeyValueStorage) snapshotWorldStateStorage.codeStorage; - this.storageSnap = (SnappedKeyValueStorage) snapshotWorldStateStorage.storageStorage; - this.trieBranchSnap = (SnappedKeyValueStorage) snapshotWorldStateStorage.trieBranchStorage; - this.parentWorldStateStorage = parentWorldStateStorage; - } - - public static BonsaiSnapshotWorldState create( - final BonsaiWorldStateArchive archive, - final BonsaiWorldStateKeyValueStorage parentWorldStateStorage) { - return new BonsaiSnapshotWorldState( - archive, - new BonsaiSnapshotWorldStateKeyValueStorage( - ((SnappableKeyValueStorage) parentWorldStateStorage.accountStorage).takeSnapshot(), - ((SnappableKeyValueStorage) parentWorldStateStorage.codeStorage).takeSnapshot(), - ((SnappableKeyValueStorage) parentWorldStateStorage.storageStorage).takeSnapshot(), - ((SnappableKeyValueStorage) parentWorldStateStorage.trieBranchStorage) - .takeSnapshot(), - parentWorldStateStorage.trieLogStorage), - parentWorldStateStorage) - .subscribeToParentStorage(); - } - - @Override - public Hash rootHash() { - if (updater.isDirty()) { - this.worldStateRootHash = calculateRootHash(worldStateStorage.updater(), updater); - } - return this.worldStateRootHash; - } - - @Override - public MutableWorldState copy() { - // return a clone-based copy of worldstate storage - return new BonsaiSnapshotWorldState( - archive, - new BonsaiSnapshotWorldStateKeyValueStorage( - accountSnap.cloneFromSnapshot(), - codeSnap.cloneFromSnapshot(), - storageSnap.cloneFromSnapshot(), - trieBranchSnap.cloneFromSnapshot(), - worldStateStorage.trieLogStorage), - parentWorldStateStorage) - .subscribeToParentStorage(); - } - - @Override - public void close() throws Exception { - snapshotWorldStateStorage.close(); - } - - @Override - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { - return snapshotWorldStateStorage; - } - - protected BonsaiSnapshotWorldState subscribeToParentStorage() { - snapshotWorldStateStorage.subscribeToParentStorage(parentWorldStateStorage); - return this; - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateKeyValueStorage.java deleted file mode 100644 index bfd9f312e7..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateKeyValueStorage.java +++ /dev/null @@ -1,307 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ -package org.hyperledger.besu.ethereum.bonsai; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.plugin.services.exception.StorageException; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; -import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; - -import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { - - private static final Logger LOG = - LoggerFactory.getLogger(BonsaiSnapshotWorldStateKeyValueStorage.class); - - private final AtomicReference parentStorage = - new AtomicReference<>(); - private final AtomicLong parentStorageSubscriberId = new AtomicLong(Long.MAX_VALUE); - private final AtomicBoolean shouldClose = new AtomicBoolean(false); - private final AtomicBoolean isClosed = new AtomicBoolean(false); - - public BonsaiSnapshotWorldStateKeyValueStorage( - final SnappedKeyValueStorage accountStorage, - final SnappedKeyValueStorage codeStorage, - final SnappedKeyValueStorage storageStorage, - final SnappedKeyValueStorage trieBranchStorage, - final KeyValueStorage trieLogStorage) { - super(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage); - } - - @Override - public BonsaiUpdater updater() { - return new SnapshotUpdater( - (SnappedKeyValueStorage) accountStorage, - (SnappedKeyValueStorage) codeStorage, - (SnappedKeyValueStorage) storageStorage, - (SnappedKeyValueStorage) trieBranchStorage, - trieLogStorage); - } - - @Override - public void clear() { - // snapshot storage does not implement clear - throw new StorageException("Snapshot storage does not implement clear"); - } - - @Override - public void clearFlatDatabase() { - // snapshot storage does not implement clear - throw new StorageException("Snapshot storage does not implement clear"); - } - - @Override - public void clearTrieLog() { - // snapshot storage does not implement clear - throw new StorageException("Snapshot storage does not implement clear"); - } - - @Override - public synchronized long subscribe(final BonsaiStorageSubscriber sub) { - if (isClosed.get()) { - throw new RuntimeException("Storage is marked to close or has already closed"); - } - return super.subscribe(sub); - } - - @Override - public synchronized void unSubscribe(final long id) { - super.unSubscribe(id); - try { - tryClose(); - } catch (Exception e) { - LOG.atWarn() - .setMessage("exception while trying to close : {}") - .addArgument(e::getMessage) - .log(); - } - } - - void subscribeToParentStorage(final BonsaiWorldStateKeyValueStorage parentStorage) { - this.parentStorage.set(parentStorage); - parentStorageSubscriberId.set(parentStorage.subscribe(this)); - } - - @Override - public void onClearStorage() { - try { - // when the parent storage clears, close regardless of subscribers - doClose(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void onClearFlatDatabaseStorage() { - // when the parent storage clears, close regardless of subscribers - try { - doClose(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void onClearTrieLog() { - // when the parent storage clears, close regardless of subscribers - try { - doClose(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public synchronized void close() throws Exception { - // when the parent storage clears, close - shouldClose.set(true); - tryClose(); - } - - protected synchronized void tryClose() throws Exception { - if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { - // attempting to close already closed snapshots will segfault - doClose(); - } - } - - private void doClose() throws Exception { - if (!isClosed.get()) { - // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); - - // unsubscribe from parent storage if we have subscribed - Optional.ofNullable(parentStorage.get()) - .filter(__ -> parentStorageSubscriberId.get() != Long.MAX_VALUE) - .ifPresent(parent -> parent.unSubscribe(parentStorageSubscriberId.get())); - - // close all of the SnappedKeyValueStorages: - accountStorage.close(); - codeStorage.close(); - storageStorage.close(); - trieBranchStorage.close(); - - // set storage closed - isClosed.set(true); - } - } - - public static class SnapshotUpdater implements BonsaiWorldStateKeyValueStorage.BonsaiUpdater { - - private final KeyValueStorageTransaction accountStorageTransaction; - private final KeyValueStorageTransaction codeStorageTransaction; - private final KeyValueStorageTransaction storageStorageTransaction; - private final KeyValueStorageTransaction trieBranchStorageTransaction; - private final KeyValueStorageTransaction trieLogStorageTransaction; - - public SnapshotUpdater( - final SnappedKeyValueStorage accountStorage, - final SnappedKeyValueStorage codeStorage, - final SnappedKeyValueStorage storageStorage, - final SnappedKeyValueStorage trieBranchStorage, - final KeyValueStorage trieLogStorage) { - this.accountStorageTransaction = accountStorage.getSnapshotTransaction(); - this.codeStorageTransaction = codeStorage.getSnapshotTransaction(); - this.storageStorageTransaction = storageStorage.getSnapshotTransaction(); - this.trieBranchStorageTransaction = trieBranchStorage.getSnapshotTransaction(); - this.trieLogStorageTransaction = trieLogStorage.startTransaction(); - } - - @Override - public BonsaiUpdater removeCode(final Hash accountHash) { - codeStorageTransaction.remove(accountHash.toArrayUnsafe()); - return this; - } - - @Override - public WorldStateStorage.Updater putCode( - final Hash accountHash, final Bytes32 nodeHash, final Bytes code) { - if (code.size() == 0) { - // Don't save empty values - return this; - } - codeStorageTransaction.put(accountHash.toArrayUnsafe(), code.toArrayUnsafe()); - return this; - } - - @Override - public BonsaiUpdater removeAccountInfoState(final Hash accountHash) { - accountStorageTransaction.remove(accountHash.toArrayUnsafe()); - return this; - } - - @Override - public BonsaiUpdater putAccountInfoState(final Hash accountHash, final Bytes accountValue) { - if (accountValue.size() == 0) { - // Don't save empty values - return this; - } - accountStorageTransaction.put(accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe()); - return this; - } - - @Override - public BonsaiUpdater putStorageValueBySlotHash( - final Hash accountHash, final Hash slotHash, final Bytes storage) { - storageStorageTransaction.put( - Bytes.concatenate(accountHash, slotHash).toArrayUnsafe(), storage.toArrayUnsafe()); - return this; - } - - @Override - public void removeStorageValueBySlotHash(final Hash accountHash, final Hash slotHash) { - storageStorageTransaction.remove(Bytes.concatenate(accountHash, slotHash).toArrayUnsafe()); - } - - @Override - public KeyValueStorageTransaction getTrieBranchStorageTransaction() { - return trieBranchStorageTransaction; - } - - @Override - public KeyValueStorageTransaction getTrieLogStorageTransaction() { - return trieLogStorageTransaction; - } - - @Override - public WorldStateStorage.Updater saveWorldState( - final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { - trieBranchStorageTransaction.put(Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); - trieBranchStorageTransaction.put(WORLD_ROOT_HASH_KEY, nodeHash.toArrayUnsafe()); - trieBranchStorageTransaction.put(WORLD_BLOCK_HASH_KEY, blockHash.toArrayUnsafe()); - return this; - } - - @Override - public WorldStateStorage.Updater putAccountStateTrieNode( - final Bytes location, final Bytes32 nodeHash, final Bytes node) { - if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) { - // Don't save empty nodes - return this; - } - trieBranchStorageTransaction.put(location.toArrayUnsafe(), node.toArrayUnsafe()); - return this; - } - - @Override - public WorldStateStorage.Updater removeAccountStateTrieNode( - final Bytes location, final Bytes32 nodeHash) { - trieBranchStorageTransaction.remove(location.toArrayUnsafe()); - return this; - } - - @Override - public WorldStateStorage.Updater putAccountStorageTrieNode( - final Hash accountHash, final Bytes location, final Bytes32 nodeHash, final Bytes node) { - if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) { - // Don't save empty nodes - return this; - } - trieBranchStorageTransaction.put( - Bytes.concatenate(accountHash, location).toArrayUnsafe(), node.toArrayUnsafe()); - return this; - } - - @Override - public void commit() { - accountStorageTransaction.commit(); - codeStorageTransaction.commit(); - storageStorageTransaction.commit(); - trieBranchStorageTransaction.commit(); - trieLogStorageTransaction.commit(); - } - - @Override - public void rollback() { - // no-op - } - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java index b297b8e505..4f2564be5e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java @@ -26,13 +26,13 @@ public class BonsaiValue { private T updated; private boolean cleared; - BonsaiValue(final T prior, final T updated) { + public BonsaiValue(final T prior, final T updated) { this.prior = prior; this.updated = updated; this.cleared = false; } - BonsaiValue(final T prior, final T updated, final boolean cleared) { + public BonsaiValue(final T prior, final T updated, final boolean cleared) { this.prior = prior; this.updated = updated; this.cleared = cleared; @@ -57,13 +57,13 @@ public class BonsaiValue { return this; } - void writeRlp(final RLPOutput output, final BiConsumer writer) { + public void writeRlp(final RLPOutput output, final BiConsumer writer) { output.startList(); writeInnerRlp(output, writer); output.endList(); } - void writeInnerRlp(final RLPOutput output, final BiConsumer writer) { + public void writeInnerRlp(final RLPOutput output, final BiConsumer writer) { if (prior == null) { output.writeNull(); } else { @@ -76,7 +76,7 @@ public class BonsaiValue { } } - boolean isUnchanged() { + public boolean isUnchanged() { return Objects.equals(updated, prior); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java similarity index 65% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java index 3084b18ac9..31da984c13 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java @@ -16,22 +16,25 @@ package org.hyperledger.besu.ethereum.bonsai; -import static org.hyperledger.besu.datatypes.Hash.fromPlugin; -import static org.hyperledger.besu.ethereum.bonsai.LayeredTrieLogManager.RETAINED_LAYERS; +import static org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager.RETAINED_LAYERS; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.BlockAddedEvent; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.core.SnapshotMutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; @@ -50,21 +53,19 @@ import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateArchive implements WorldStateArchive { +public class BonsaiWorldStateProvider implements WorldStateArchive { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateArchive.class); + private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); private final Blockchain blockchain; private final TrieLogManager trieLogManager; - private final BonsaiPersistedWorldState persistedState; + private final BonsaiWorldState persistedState; private final BonsaiWorldStateKeyValueStorage worldStateStorage; private final CachedMerkleTrieLoader cachedMerkleTrieLoader; - private final boolean useSnapshots; - - public BonsaiWorldStateArchive( + public BonsaiWorldStateProvider( final StorageProvider provider, final Blockchain blockchain, final CachedMerkleTrieLoader cachedMerkleTrieLoader) { @@ -73,149 +74,107 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { provider.createWorldStateStorage(DataStorageFormat.BONSAI), blockchain, Optional.empty(), - provider.isWorldStateSnappable(), cachedMerkleTrieLoader); } - public BonsaiWorldStateArchive( + public BonsaiWorldStateProvider( final BonsaiWorldStateKeyValueStorage worldStateStorage, final Blockchain blockchain, final Optional maxLayersToLoad, final CachedMerkleTrieLoader cachedMerkleTrieLoader) { - // overload while snapshots are an experimental option: - this( - worldStateStorage, - blockchain, - maxLayersToLoad, - DataStorageConfiguration.DEFAULT_BONSAI_USE_SNAPSHOTS, - cachedMerkleTrieLoader); - } - public BonsaiWorldStateArchive( - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final Blockchain blockchain, - final Optional maxLayersToLoad, - final boolean useSnapshots, - final CachedMerkleTrieLoader cachedMerkleTrieLoader) { - this( - useSnapshots - ? new SnapshotTrieLogManager( - blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS)) - : new LayeredTrieLogManager( - blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS)), - worldStateStorage, - blockchain, - useSnapshots, - cachedMerkleTrieLoader); + // TODO: de-dup constructors + this.trieLogManager = + new CachedWorldStorageManager( + this, blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS)); + this.blockchain = blockchain; + this.worldStateStorage = worldStateStorage; + this.persistedState = new BonsaiWorldState(this, worldStateStorage); + this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + blockchain + .getBlockHeader(persistedState.worldStateBlockHash) + .ifPresent( + blockHeader -> { + this.trieLogManager.addCachedLayer( + blockHeader, persistedState.worldStateRootHash, persistedState); + }); } @VisibleForTesting - BonsaiWorldStateArchive( + BonsaiWorldStateProvider( final TrieLogManager trieLogManager, final BonsaiWorldStateKeyValueStorage worldStateStorage, final Blockchain blockchain, - final boolean useSnapshots, final CachedMerkleTrieLoader cachedMerkleTrieLoader) { this.trieLogManager = trieLogManager; this.blockchain = blockchain; this.worldStateStorage = worldStateStorage; - this.persistedState = new BonsaiPersistedWorldState(this, worldStateStorage); - // TODO: https://github.com/hyperledger/besu/issues/4641 - // useSnapshots is disabled for now - this.useSnapshots = false; + this.persistedState = new BonsaiWorldState(this, worldStateStorage); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - blockchain.observeBlockAdded(this::blockAddedHandler); - } - - private void blockAddedHandler(final BlockAddedEvent event) { - LOG.debug("New block add event {}", event); - if (event.isNewCanonicalHead()) { - final BlockHeader eventBlockHeader = event.getBlock().getHeader(); - trieLogManager.updateCachedLayers( - eventBlockHeader.getParentHash(), eventBlockHeader.getHash()); - } + blockchain + .getBlockHeader(persistedState.worldStateBlockHash) + .ifPresent( + blockHeader -> { + this.trieLogManager.addCachedLayer( + blockHeader, persistedState.worldStateRootHash, persistedState); + }); } @Override public Optional get(final Hash rootHash, final Hash blockHash) { - final Optional layeredWorldState = - trieLogManager.getBonsaiCachedWorldState(blockHash); - if (layeredWorldState.isPresent()) { - return Optional.of(layeredWorldState.get()); - } else if (rootHash.equals(persistedState.blockHash())) { - return Optional.of(persistedState); - } else { - return Optional.empty(); - } + return trieLogManager + .getWorldState(blockHash) + .or( + () -> { + if (blockHash.equals(persistedState.blockHash())) { + return Optional.of(persistedState); + } else { + return Optional.empty(); + } + }) + .map(WorldState.class::cast); } @Override public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { - return trieLogManager.getBonsaiCachedWorldState(blockHash).isPresent() + return trieLogManager.containWorldStateStorage(blockHash) || persistedState.blockHash().equals(blockHash) || worldStateStorage.isWorldStateAvailable(rootHash, blockHash); } - public Optional getMutableSnapshot(final Hash blockHash) { - return rollMutableStateToBlockHash( - BonsaiSnapshotWorldState.create(this, worldStateStorage), blockHash) - .map(SnapshotMutableWorldState.class::cast); - } - @Override public Optional getMutable( - final Hash rootHash, final Hash blockHash, final boolean shouldPersistState) { + final BlockHeader blockHeader, final boolean shouldPersistState) { if (shouldPersistState) { - return getMutable(rootHash, blockHash); + return getMutable(blockHeader.getStateRoot(), blockHeader.getHash()); } else { + final BlockHeader chainHeadBlockHeader = blockchain.getChainHeadHeader(); + if (chainHeadBlockHeader.getNumber() - blockHeader.getNumber() + >= trieLogManager.getMaxLayersToLoad()) { + LOG.warn( + "Exceeded the limit of back layers that can be loaded ({})", + trieLogManager.getMaxLayersToLoad()); + return Optional.empty(); + } return trieLogManager - .getBonsaiCachedWorldState(blockHash) - .or( - () -> - blockchain - .getBlockHeader(blockHash) - .filter( - header -> { - if (blockchain.getChainHeadHeader().getNumber() - header.getNumber() - >= trieLogManager.getMaxLayersToLoad()) { - LOG.warn( - "Exceeded the limit of back layers that can be loaded ({})", - trieLogManager.getMaxLayersToLoad()); - return false; - } - return true; - }) - .flatMap(header -> snapshotOrLayeredWorldState(blockHash, header))); - } - } - - private Optional snapshotOrLayeredWorldState( - final Hash blockHash, final BlockHeader blockHeader) { - if (useSnapshots) { - // use snapshots: - return getMutableSnapshot(blockHash); - } else { - // otherwise use layered worldstate: - final Optional trieLogLayer = trieLogManager.getTrieLogLayer(blockHash); - return trieLogLayer.map( - layer -> - new BonsaiLayeredWorldState( - blockchain, - this, - Optional.empty(), - blockHeader.getNumber(), - fromPlugin(blockHeader.getStateRoot()), - layer)); + .getWorldState(blockHeader.getHash()) + .or(() -> trieLogManager.getNearestWorldState(blockHeader)) + .or(() -> trieLogManager.getHeadWorldState(blockchain::getBlockHeader)) + .flatMap( + bonsaiWorldState -> + rollMutableStateToBlockHash(bonsaiWorldState, blockHeader.getHash())) + .map(MutableWorldState::freeze); } } @Override - public Optional getMutable(final Hash rootHash, final Hash blockHash) { + public synchronized Optional getMutable( + final Hash rootHash, final Hash blockHash) { return rollMutableStateToBlockHash(persistedState, blockHash); } Optional rollMutableStateToBlockHash( - final BonsaiPersistedWorldState mutableState, final Hash blockHash) { + final BonsaiWorldState mutableState, final Hash blockHash) { if (blockHash.equals(mutableState.blockHash())) { return Optional.of(mutableState); } else { @@ -264,21 +223,26 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { } // attempt the state rolling - final BonsaiWorldStateUpdater bonsaiUpdater = getUpdaterFromPersistedState(mutableState); + final BonsaiWorldStateUpdateAccumulator bonsaiUpdater = + (BonsaiWorldStateUpdateAccumulator) mutableState.updater(); try { for (final TrieLogLayer rollBack : rollBacks) { LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); bonsaiUpdater.rollBack(rollBack); } for (int i = rollForwards.size() - 1; i >= 0; i--) { + final var forward = rollForwards.get(i); LOG.debug("Attempting Rollforward of {}", rollForwards.get(i).getBlockHash()); - bonsaiUpdater.rollForward(rollForwards.get(i)); + bonsaiUpdater.rollForward(forward); } bonsaiUpdater.commit(); mutableState.persist(blockchain.getBlockHeader(blockHash).get()); - LOG.debug("Archive rolling finished, now at {}", blockHash); + LOG.debug( + "Archive rolling finished, {} now at {}", + mutableState.getWorldStateStorage().getClass().getSimpleName(), + blockHash); return Optional.of(mutableState); } catch (final MerkleTrieException re) { // need to throw to trigger the heal @@ -286,25 +250,27 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { } catch (final Exception e) { // if we fail we must clean up the updater bonsaiUpdater.reset(); - LOG.debug("State rolling failed for block hash " + blockHash, e); + LOG.debug( + "State rolling failed on " + + mutableState.getWorldStateStorage().getClass().getSimpleName() + + " for block hash " + + blockHash, + e); + return Optional.empty(); } } catch (final RuntimeException re) { - LOG.trace("Archive rolling failed for block hash " + blockHash, re); + LOG.info("Archive rolling failed for block hash " + blockHash, re); if (re instanceof MerkleTrieException) { // need to throw to trigger the heal throw re; } - return Optional.empty(); + throw new MerkleTrieException( + "invalid", Optional.of(Address.ZERO), Hash.EMPTY, Bytes.EMPTY); } } } - BonsaiWorldStateUpdater getUpdaterFromPersistedState( - final BonsaiPersistedWorldState mutableState) { - return (BonsaiWorldStateUpdater) mutableState.updater(); - } - public CachedMerkleTrieLoader getCachedMerkleTrieLoader() { return cachedMerkleTrieLoader; } @@ -387,4 +353,13 @@ public class BonsaiWorldStateArchive implements WorldStateArchive { // FIXME we can do proofs for layered tries and the persisted trie return Optional.empty(); } + + @Override + public void close() { + try { + worldStateStorage.close(); + } catch (Exception e) { + // no op + } + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/LayeredTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/LayeredTrieLogManager.java deleted file mode 100644 index 07962dbe4e..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/LayeredTrieLogManager.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.bonsai; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import org.apache.tuweni.bytes.Bytes32; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LayeredTrieLogManager extends AbstractTrieLogManager { - private static final Logger LOG = LoggerFactory.getLogger(LayeredTrieLogManager.class); - - LayeredTrieLogManager( - final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final long maxLayersToLoad, - final Map> cachedWorldStatesByHash) { - super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash); - } - - public LayeredTrieLogManager( - final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final long maxLayersToLoad) { - this(blockchain, worldStateStorage, maxLayersToLoad, new HashMap<>()); - } - - @Override - public synchronized void addCachedLayer( - final BlockHeader blockHeader, - final Hash worldStateRootHash, - final TrieLogLayer trieLog, - final BonsaiWorldStateArchive worldStateArchive, - final BonsaiPersistedWorldState forWorldState) { - - final BonsaiLayeredWorldState bonsaiLayeredWorldState = - new BonsaiLayeredWorldState( - blockchain, - worldStateArchive, - Optional.of(forWorldState), - blockHeader.getNumber(), - worldStateRootHash, - trieLog); - LOG.atDebug() - .setMessage("adding layered world state for block {}, state root hash {}") - .addArgument(blockHeader::toLogString) - .addArgument(worldStateRootHash::toShortHexString) - .log(); - cachedWorldStatesByHash.put( - blockHeader.getHash(), new LayeredWorldStateCache(bonsaiLayeredWorldState)); - } - - @Override - public synchronized void updateCachedLayers(final Hash blockParentHash, final Hash blockHash) { - cachedWorldStatesByHash.computeIfPresent( - blockParentHash, - (parentHash, bonsaiLayeredWorldState) -> { - if (cachedWorldStatesByHash.containsKey(blockHash)) { - bonsaiLayeredWorldState - .getMutableWorldState() - .setNextWorldView( - Optional.of(cachedWorldStatesByHash.get(blockHash).getMutableWorldState())); - } - return bonsaiLayeredWorldState; - }); - } - - public static class LayeredWorldStateCache implements CachedWorldState { - - final BonsaiLayeredWorldState layeredWorldState; - - public LayeredWorldStateCache(final BonsaiLayeredWorldState layeredWorldState) { - this.layeredWorldState = layeredWorldState; - } - - @Override - public void dispose() { - // no-op - } - - @Override - public long getHeight() { - return layeredWorldState.getHeight(); - } - - @Override - public TrieLogLayer getTrieLog() { - return layeredWorldState.getTrieLog(); - } - - @Override - public BonsaiLayeredWorldState getMutableWorldState() { - return layeredWorldState; - } - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/SnapshotTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/SnapshotTrieLogManager.java deleted file mode 100644 index 5bac727c8b..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/SnapshotTrieLogManager.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ -package org.hyperledger.besu.ethereum.bonsai; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; - -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.tuweni.bytes.Bytes32; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnapshotTrieLogManager extends AbstractTrieLogManager - implements BonsaiStorageSubscriber { - private static final Logger LOG = LoggerFactory.getLogger(SnapshotTrieLogManager.class); - - public SnapshotTrieLogManager( - final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final long maxLayersToLoad) { - this(blockchain, worldStateStorage, maxLayersToLoad, new ConcurrentHashMap<>()); - } - - SnapshotTrieLogManager( - final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final long maxLayersToLoad, - final Map> cachedWorldStatesByHash) { - super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash); - worldStateStorage.subscribe(this); - } - - @Override - protected void addCachedLayer( - final BlockHeader blockHeader, - final Hash worldStateRootHash, - final TrieLogLayer trieLog, - final BonsaiWorldStateArchive worldStateArchive, - final BonsaiPersistedWorldState worldState) { - - LOG.atDebug() - .setMessage("adding snapshot world state for block {}, state root hash {}") - .addArgument(blockHeader::toLogString) - .addArgument(worldStateRootHash::toShortHexString) - .log(); - - // TODO: add a generic param so we don't have to cast: - BonsaiSnapshotWorldState snapshotWorldState; - if (worldState instanceof BonsaiSnapshotWorldState) { - snapshotWorldState = (BonsaiSnapshotWorldState) worldState; - } else { - snapshotWorldState = - BonsaiSnapshotWorldState.create(worldStateArchive, rootWorldStateStorage); - } - - cachedWorldStatesByHash.put( - blockHeader.getHash(), - new CachedSnapshotWorldState(snapshotWorldState, trieLog, blockHeader.getNumber())); - } - - @Override - public void updateCachedLayers(final Hash blockParentHash, final Hash blockHash) { - // no-op. - } - - @Override - public synchronized Optional getBonsaiCachedWorldState(final Hash blockHash) { - if (cachedWorldStatesByHash.containsKey(blockHash)) { - return Optional.ofNullable(cachedWorldStatesByHash.get(blockHash)) - .map(CachedWorldState::getMutableWorldState) - .map(MutableWorldState::copy); - } - return Optional.empty(); - } - - @Override - public synchronized void onClearStorage() { - dropArchive(); - } - - @Override - public synchronized void onClearFlatDatabaseStorage() { - dropArchive(); - } - - @Override - public void onClearTrieLog() { - dropArchive(); - } - - private void dropArchive() { - // drop all cached snapshot worldstates, they are unsafe when the db has been truncated - LOG.info("Key-value storage truncated, dropping cached worldstates"); - cachedWorldStatesByHash.clear(); - } - - public static class CachedSnapshotWorldState - implements CachedWorldState, BonsaiStorageSubscriber { - - final BonsaiSnapshotWorldState snapshot; - final Long snapshotSubscriberId; - final TrieLogLayer trieLog; - final long height; - final AtomicBoolean isClosed = new AtomicBoolean(false); - - public CachedSnapshotWorldState( - final BonsaiSnapshotWorldState snapshot, final TrieLogLayer trieLog, final long height) { - this.snapshotSubscriberId = snapshot.getWorldStateStorage().subscribe(this); - this.snapshot = snapshot; - this.trieLog = trieLog; - this.height = height; - } - - @Override - public void dispose() { - snapshot.worldStateStorage.unSubscribe(snapshotSubscriberId); - } - - @Override - public long getHeight() { - return height; - } - - @Override - public TrieLogLayer getTrieLog() { - return trieLog; - } - - @Override - public synchronized BonsaiSnapshotWorldState getMutableWorldState() { - if (isClosed.get()) { - return null; - } - return snapshot; - } - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedBonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedBonsaiWorldView.java new file mode 100644 index 0000000000..473933b395 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedBonsaiWorldView.java @@ -0,0 +1,76 @@ +/* + * Copyright Hyperledger Besu contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.bonsai.cache; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.core.BlockHeader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CachedBonsaiWorldView implements BonsaiStorageSubscriber { + private BonsaiWorldStateKeyValueStorage worldStateStorage; + private final BlockHeader blockHeader; + private long worldViewSubscriberId; + private static final Logger LOG = LoggerFactory.getLogger(CachedBonsaiWorldView.class); + + public CachedBonsaiWorldView( + final BlockHeader blockHeader, final BonsaiWorldStateKeyValueStorage worldView) { + this.blockHeader = blockHeader; + this.worldStateStorage = worldView; + this.worldViewSubscriberId = worldStateStorage.subscribe(this); + } + + public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + return worldStateStorage; + } + + public long getBlockNumber() { + return blockHeader.getNumber(); + } + + public Hash getBlockHash() { + return blockHeader.getHash(); + } + + public synchronized void close() { + worldStateStorage.unSubscribe(this.worldViewSubscriberId); + try { + worldStateStorage.close(); + } catch (final Exception e) { + LOG.warn("Failed to close worldstate storage for block " + blockHeader.toLogString(), e); + } + } + + public synchronized void updateWorldStateStorage( + final BonsaiWorldStateKeyValueStorage newWorldStateStorage) { + long newSubscriberId = newWorldStateStorage.subscribe(this); + this.worldStateStorage.unSubscribe(this.worldViewSubscriberId); + BonsaiWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateStorage; + this.worldStateStorage = newWorldStateStorage; + this.worldViewSubscriberId = newSubscriberId; + try { + oldWorldStateStorage.close(); + } catch (final Exception e) { + LOG.warn( + "During update, failed to close prior worldstate storage for block " + + blockHeader.toLogString(), + e); + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoader.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java index b2bfdd4d2c..f603f9b4af 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java @@ -13,11 +13,12 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.bonsai; +package org.hyperledger.besu.ethereum.bonsai.cache; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java new file mode 100644 index 0000000000..65d2f55bc7 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java @@ -0,0 +1,183 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.bonsai.cache; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CachedWorldStorageManager extends AbstractTrieLogManager + implements BonsaiStorageSubscriber { + private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class); + private final BonsaiWorldStateProvider archive; + + CachedWorldStorageManager( + final BonsaiWorldStateProvider archive, + final Blockchain blockchain, + final BonsaiWorldStateKeyValueStorage worldStateStorage, + final long maxLayersToLoad, + final Map cachedWorldStatesByHash) { + super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash); + worldStateStorage.subscribe(this); + this.archive = archive; + } + + public CachedWorldStorageManager( + final BonsaiWorldStateProvider archive, + final Blockchain blockchain, + final BonsaiWorldStateKeyValueStorage worldStateStorage, + final long maxLayersToLoad) { + this(archive, blockchain, worldStateStorage, maxLayersToLoad, new ConcurrentHashMap<>()); + } + + @Override + public synchronized void addCachedLayer( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final BonsaiWorldState forWorldState) { + final Optional cachedBonsaiWorldView = + Optional.ofNullable(this.cachedWorldStatesByHash.get(blockHeader.getBlockHash())); + if (cachedBonsaiWorldView.isPresent()) { + // only replace if it is a layered storage + if (forWorldState.isPersisted() + && cachedBonsaiWorldView.get().getWorldStateStorage() + instanceof BonsaiWorldStateLayerStorage) { + LOG.atDebug() + .setMessage("updating layered world state for block {}, state root hash {}") + .addArgument(blockHeader::toLogString) + .addArgument(worldStateRootHash::toShortHexString) + .log(); + cachedBonsaiWorldView + .get() + .updateWorldStateStorage( + new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.worldStateStorage)); + } + } else { + LOG.atDebug() + .setMessage("adding layered world state for block {}, state root hash {}") + .addArgument(blockHeader::toLogString) + .addArgument(worldStateRootHash::toShortHexString) + .log(); + if (forWorldState.isPersisted()) { + cachedWorldStatesByHash.put( + blockHeader.getHash(), + new CachedBonsaiWorldView( + blockHeader, + new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.worldStateStorage))); + } else { + // otherwise, add the layer to the cache + cachedWorldStatesByHash.put( + blockHeader.getHash(), + new CachedBonsaiWorldView( + blockHeader, + ((BonsaiWorldStateLayerStorage) forWorldState.getWorldStateStorage()).clone())); + } + } + scrubCachedLayers(blockHeader.getNumber()); + } + + @Override + public Optional getWorldState(final Hash blockHash) { + if (cachedWorldStatesByHash.containsKey(blockHash)) { + // return a new worldstate using worldstate storage and an isolated copy of the updater + return Optional.ofNullable(cachedWorldStatesByHash.get(blockHash)) + .map( + cached -> + new BonsaiWorldState( + archive, new BonsaiWorldStateLayerStorage(cached.getWorldStateStorage()))); + } + return Optional.empty(); + } + + @Override + public Optional getNearestWorldState(final BlockHeader blockHeader) { + return Optional.ofNullable( + cachedWorldStatesByHash.get(blockHeader.getParentHash())) // search parent block + .map(CachedBonsaiWorldView::getWorldStateStorage) + .or( + () -> { + // or else search the nearest state in the cache + final List cachedBonsaiWorldViews = + new ArrayList<>(cachedWorldStatesByHash.values()); + return cachedBonsaiWorldViews.stream() + .sorted( + Comparator.comparingLong( + view -> Math.abs(blockHeader.getNumber() - view.getBlockNumber()))) + .map(CachedBonsaiWorldView::getWorldStateStorage) + .findFirst(); + }) + .map( + storage -> + new BonsaiWorldState( // wrap the state in a layered worldstate + archive, new BonsaiWorldStateLayerStorage(storage))); + } + + @Override + public Optional getHeadWorldState( + final Function> hashBlockHeaderFunction) { + + return rootWorldStateStorage + .getWorldStateBlockHash() + .flatMap(hashBlockHeaderFunction) + .flatMap( + blockHeader -> { + // add the head to the cache + addCachedLayer( + blockHeader, + blockHeader.getStateRoot(), + new BonsaiWorldState(archive, rootWorldStateStorage)); + return getWorldState(blockHeader.getHash()); + }); + } + + @Override + public void onClearStorage() { + this.cachedWorldStatesByHash.clear(); + } + + @Override + public void onClearFlatDatabaseStorage() { + this.cachedWorldStatesByHash.clear(); + } + + @Override + public void onClearTrieLog() { + this.cachedWorldStatesByHash.clear(); + } + + @Override + public void onCloseStorage() { + this.cachedWorldStatesByHash.clear(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java new file mode 100644 index 0000000000..d373f7e9db --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -0,0 +1,217 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.bonsai.storage; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; + +import java.util.Optional; +import java.util.function.Supplier; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKeyValueStorage + implements BonsaiStorageSubscriber { + + protected final BonsaiWorldStateKeyValueStorage parentWorldStateStorage; + + private final long subscribeParentId; + + public BonsaiSnapshotWorldStateKeyValueStorage( + final BonsaiWorldStateKeyValueStorage parentWorldStateStorage, + final SnappedKeyValueStorage accountStorage, + final SnappedKeyValueStorage codeStorage, + final SnappedKeyValueStorage storageStorage, + final SnappedKeyValueStorage trieBranchStorage, + final KeyValueStorage trieLogStorage) { + super(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage); + this.parentWorldStateStorage = parentWorldStateStorage; + this.subscribeParentId = parentWorldStateStorage.subscribe(this); + } + + public BonsaiSnapshotWorldStateKeyValueStorage( + final BonsaiWorldStateKeyValueStorage worldStateStorage) { + this( + worldStateStorage, + ((SnappableKeyValueStorage) worldStateStorage.accountStorage).takeSnapshot(), + ((SnappableKeyValueStorage) worldStateStorage.codeStorage).takeSnapshot(), + ((SnappableKeyValueStorage) worldStateStorage.storageStorage).takeSnapshot(), + ((SnappableKeyValueStorage) worldStateStorage.trieBranchStorage).takeSnapshot(), + worldStateStorage.trieLogStorage); + } + + @Override + public BonsaiUpdater updater() { + return new Updater( + ((SnappedKeyValueStorage) accountStorage).getSnapshotTransaction(), + ((SnappedKeyValueStorage) codeStorage).getSnapshotTransaction(), + ((SnappedKeyValueStorage) storageStorage).getSnapshotTransaction(), + ((SnappedKeyValueStorage) trieBranchStorage).getSnapshotTransaction(), + trieLogStorage.startTransaction()); + } + + @Override + public Optional getAccount(final Hash accountHash) { + return isClosed.get() ? Optional.empty() : super.getAccount(accountHash); + } + + @Override + public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { + return isClosed.get() ? Optional.empty() : super.getCode(codeHash, accountHash); + } + + @Override + public Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { + return isClosed.get() ? Optional.empty() : super.getAccountStateTrieNode(location, nodeHash); + } + + @Override + public Optional getAccountStorageTrieNode( + final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { + return isClosed.get() + ? Optional.empty() + : super.getAccountStorageTrieNode(accountHash, location, nodeHash); + } + + @Override + public Optional getTrieLog(final Hash blockHash) { + return isClosed.get() ? Optional.empty() : super.getTrieLog(blockHash); + } + + @Override + public Optional getStateTrieNode(final Bytes location) { + return isClosed.get() ? Optional.empty() : super.getStateTrieNode(location); + } + + @Override + public Optional getWorldStateRootHash() { + return isClosed.get() ? Optional.empty() : super.getWorldStateRootHash(); + } + + @Override + public Optional getWorldStateBlockHash() { + return isClosed.get() ? Optional.empty() : super.getWorldStateBlockHash(); + } + + @Override + public Optional getStorageValueBySlotHash(final Hash accountHash, final Hash slotHash) { + return isClosed.get() + ? Optional.empty() + : super.getStorageValueBySlotHash(accountHash, slotHash); + } + + @Override + public Optional getStorageValueBySlotHash( + final Supplier> storageRootSupplier, + final Hash accountHash, + final Hash slotHash) { + return isClosed.get() + ? Optional.empty() + : super.getStorageValueBySlotHash(storageRootSupplier, accountHash, slotHash); + } + + @Override + public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { + return !isClosed.get() && super.isWorldStateAvailable(rootHash, blockHash); + } + + @Override + public void clear() { + // snapshot storage does not implement clear + throw new StorageException("Snapshot storage does not implement clear"); + } + + @Override + public void clearFlatDatabase() { + // snapshot storage does not implement clear + throw new StorageException("Snapshot storage does not implement clear"); + } + + @Override + public void clearTrieLog() { + // snapshot storage does not implement clear + throw new StorageException("Snapshot storage does not implement clear"); + } + + @Override + public void onCloseStorage() { + try { + // when the parent storage clears, close regardless of subscribers + doClose(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onClearStorage() { + try { + // when the parent storage clears, close regardless of subscribers + doClose(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onClearFlatDatabaseStorage() { + // when the parent storage clears, close regardless of subscribers + try { + doClose(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onClearTrieLog() { + // when the parent storage clears, close regardless of subscribers + try { + doClose(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected synchronized void doClose() throws Exception { + if (!isClosed.get()) { + // alert any subscribers we are closing: + subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); + + // close all of the SnappedKeyValueStorages: + accountStorage.close(); + codeStorage.close(); + storageStorage.close(); + trieBranchStorage.close(); + + // unsubscribe the parent worldstate + parentWorldStateStorage.unSubscribe(subscribeParentId); + + // set storage closed + isClosed.set(true); + } + } + + public BonsaiWorldStateKeyValueStorage getParentWorldStateStorage() { + return parentWorldStateStorage; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java similarity index 90% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 0494832740..ce43ff762b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.bonsai; +package org.hyperledger.besu.ethereum.bonsai.storage; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.storage.StorageProvider; @@ -28,6 +28,7 @@ import org.hyperledger.besu.util.Subscribers; import java.nio.charset.StandardCharsets; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -35,8 +36,13 @@ import java.util.function.Supplier; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.rlp.RLP; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); + // 0x776f726c64526f6f74 public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); // 0x776f726c64426c6f636b48617368 @@ -48,6 +54,11 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC protected final KeyValueStorage storageStorage; protected final KeyValueStorage trieBranchStorage; protected final KeyValueStorage trieLogStorage; + + private final AtomicBoolean shouldClose = new AtomicBoolean(false); + + protected final AtomicBoolean isClosed = new AtomicBoolean(false); + protected final Subscribers subscribers = Subscribers.create(); public BonsaiWorldStateKeyValueStorage(final StorageProvider provider) { @@ -144,8 +155,8 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC return trieBranchStorage.get(WORLD_ROOT_HASH_KEY).map(Bytes::wrap); } - public Optional getWorldStateBlockHash() { - return trieBranchStorage.get(WORLD_BLOCK_HASH_KEY).map(Bytes::wrap); + public Optional getWorldStateBlockHash() { + return trieBranchStorage.get(WORLD_BLOCK_HASH_KEY).map(Bytes32::wrap).map(Hash::wrap); } public Optional getStorageValueBySlotHash(final Hash accountHash, final Hash slotHash) { @@ -249,19 +260,6 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC throw new RuntimeException("removeNodeAddedListener not available"); } - public synchronized long subscribe(final BonsaiStorageSubscriber sub) { - return subscribers.subscribe(sub); - } - - public synchronized void unSubscribe(final long id) { - subscribers.unsubscribe(id); - } - - @Override - public void close() throws Exception { - // No need to close or notify because BonsaiWorldStateKeyValueStorage is persistent - } - public interface BonsaiUpdater extends WorldStateStorage.Updater { BonsaiUpdater removeCode(final Hash accountHash); @@ -360,7 +358,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC } @Override - public BonsaiUpdater putAccountStorageTrieNode( + public synchronized BonsaiUpdater putAccountStorageTrieNode( final Hash accountHash, final Bytes location, final Bytes32 nodeHash, final Bytes node) { if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) { // Don't save empty nodes @@ -414,7 +412,56 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC } } - interface BonsaiStorageSubscriber { + @Override + public synchronized void close() throws Exception { + // when the storage clears, close + shouldClose.set(true); + tryClose(); + } + + public synchronized long subscribe(final BonsaiStorageSubscriber sub) { + if (isClosed.get()) { + throw new RuntimeException("Storage is marked to close or has already closed"); + } + return subscribers.subscribe(sub); + } + + public synchronized void unSubscribe(final long id) { + subscribers.unsubscribe(id); + try { + tryClose(); + } catch (Exception e) { + LOG.atWarn() + .setMessage("exception while trying to close : {}") + .addArgument(e::getMessage) + .log(); + } + } + + protected synchronized void tryClose() throws Exception { + if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { + doClose(); + } + } + + protected synchronized void doClose() throws Exception { + if (!isClosed.get()) { + // alert any subscribers we are closing: + subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); + + // close all of the KeyValueStorages: + accountStorage.close(); + codeStorage.close(); + storageStorage.close(); + trieBranchStorage.close(); + trieLogStorage.close(); + + // set storage closed + isClosed.set(true); + } + } + + public interface BonsaiStorageSubscriber { default void onClearStorage() {} default void onClearFlatDatabaseStorage() {} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java new file mode 100644 index 0000000000..ba8fb6df11 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -0,0 +1,56 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.bonsai.storage; + +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; +import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage; + +public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyValueStorage + implements BonsaiStorageSubscriber { + + public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent) { + this( + new LayeredKeyValueStorage(parent.accountStorage), + new LayeredKeyValueStorage(parent.codeStorage), + new LayeredKeyValueStorage(parent.storageStorage), + new LayeredKeyValueStorage(parent.trieBranchStorage), + parent.trieLogStorage, + parent); + } + + public BonsaiWorldStateLayerStorage( + final SnappedKeyValueStorage accountStorage, + final SnappedKeyValueStorage codeStorage, + final SnappedKeyValueStorage storageStorage, + final SnappedKeyValueStorage trieBranchStorage, + final KeyValueStorage trieLogStorage, + final BonsaiWorldStateKeyValueStorage parent) { + super(parent, accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage); + } + + @Override + public BonsaiWorldStateLayerStorage clone() { + return new BonsaiWorldStateLayerStorage( + ((LayeredKeyValueStorage) accountStorage).clone(), + ((LayeredKeyValueStorage) codeStorage).clone(), + ((LayeredKeyValueStorage) storageStorage).clone(), + ((LayeredKeyValueStorage) trieBranchStorage).clone(), + trieLogStorage, + parentWorldStateStorage); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/AbstractTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java similarity index 56% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/AbstractTrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java index f77179acbd..c6d1960638 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/AbstractTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java @@ -13,40 +13,41 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.bonsai; +package org.hyperledger.besu.ethereum.bonsai.trielog; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.BonsaiUpdater; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedBonsaiWorldView; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiUpdater; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractTrieLogManager - implements TrieLogManager { +public abstract class AbstractTrieLogManager implements TrieLogManager { private static final Logger LOG = LoggerFactory.getLogger(AbstractTrieLogManager.class); public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks protected final Blockchain blockchain; protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; - protected final Map> cachedWorldStatesByHash; + protected final Map cachedWorldStatesByHash; protected final long maxLayersToLoad; - AbstractTrieLogManager( + protected AbstractTrieLogManager( final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateStorage, final long maxLayersToLoad, - final Map> cachedWorldStatesByHash) { + final Map cachedWorldStatesByHash) { this.blockchain = blockchain; this.rootWorldStateStorage = worldStateStorage; this.cachedWorldStatesByHash = cachedWorldStatesByHash; @@ -55,11 +56,10 @@ public abstract class AbstractTrieLogManager @Override public synchronized void saveTrieLog( - final BonsaiWorldStateArchive worldStateArchive, - final BonsaiWorldStateUpdater localUpdater, + final BonsaiWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiPersistedWorldState forWorldState) { + final BonsaiWorldState forWorldState) { // do not overwrite a trielog layer that already exists in the database. // if it's only in memory we need to save it // for example, in case of reorg we don't replace a trielog layer @@ -67,13 +67,7 @@ public abstract class AbstractTrieLogManager final BonsaiUpdater stateUpdater = forWorldState.getWorldStateStorage().updater(); boolean success = false; try { - final TrieLogLayer trieLog = - prepareTrieLog( - forBlockHeader, - forWorldStateRootHash, - localUpdater, - worldStateArchive, - forWorldState); + final TrieLogLayer trieLog = prepareTrieLog(forBlockHeader, localUpdater); persistTrieLog(forBlockHeader, forWorldStateRootHash, trieLog, stateUpdater); success = true; } finally { @@ -86,52 +80,28 @@ public abstract class AbstractTrieLogManager } } - protected abstract void addCachedLayer( - final BlockHeader blockHeader, - final Hash worldStateRootHash, - final TrieLogLayer trieLog, - final BonsaiWorldStateArchive worldStateArchive, - final BonsaiPersistedWorldState forWorldState); - @VisibleForTesting TrieLogLayer prepareTrieLog( - final BlockHeader forBlockHeader, - final Hash forWorldStateRootHash, - final BonsaiWorldStateUpdater localUpdater, - final BonsaiWorldStateArchive worldStateArchive, - final BonsaiPersistedWorldState forWorldState) { + final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) { LOG.atDebug() .setMessage("Adding layered world state for {}") - .addArgument(forBlockHeader::toLogString) + .addArgument(blockHeader::toLogString) .log(); - final TrieLogLayer trieLog = localUpdater.generateTrieLog(forBlockHeader.getBlockHash()); + final TrieLogLayer trieLog = localUpdater.generateTrieLog(blockHeader.getBlockHash()); trieLog.freeze(); - addCachedLayer( - forBlockHeader, forWorldStateRootHash, trieLog, worldStateArchive, forWorldState); - scrubCachedLayers(forBlockHeader.getNumber()); return trieLog; } - synchronized void scrubCachedLayers(final long newMaxHeight) { + public synchronized void scrubCachedLayers(final long newMaxHeight) { if (cachedWorldStatesByHash.size() > RETAINED_LAYERS) { final long waterline = newMaxHeight - RETAINED_LAYERS; cachedWorldStatesByHash.values().stream() - .filter(layer -> layer.getHeight() < waterline) - .collect(Collectors.toList()) - .stream() + .filter(layer -> layer.getBlockNumber() < waterline) + .toList() .forEach( layer -> { - cachedWorldStatesByHash.remove(layer.getTrieLog().getBlockHash()); - layer.dispose(); - Optional.ofNullable(layer.getMutableWorldState()) - .ifPresent( - ws -> { - try { - ws.close(); - } catch (Exception e) { - LOG.warn("Error closing bonsai worldstate layer", e); - } - }); + cachedWorldStatesByHash.remove(layer.getBlockHash()); + layer.close(); }); } } @@ -154,12 +124,8 @@ public abstract class AbstractTrieLogManager } @Override - public Optional getBonsaiCachedWorldState(final Hash blockHash) { - if (cachedWorldStatesByHash.containsKey(blockHash)) { - return Optional.ofNullable(cachedWorldStatesByHash.get(blockHash)) - .map(CachedWorldState::getMutableWorldState); - } - return Optional.empty(); + public boolean containWorldStateStorage(final Hash blockHash) { + return cachedWorldStatesByHash.containsKey(blockHash); } @Override @@ -169,10 +135,6 @@ public abstract class AbstractTrieLogManager @Override public Optional getTrieLogLayer(final Hash blockHash) { - if (cachedWorldStatesByHash.containsKey(blockHash)) { - return Optional.of(cachedWorldStatesByHash.get(blockHash).getTrieLog()); - } else { - return rootWorldStateStorage.getTrieLog(blockHash).map(TrieLogLayer::fromBytes); - } + return rootWorldStateStorage.getTrieLog(blockHash).map(TrieLogLayer::fromBytes); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogLayer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayer.java similarity index 98% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogLayer.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayer.java index f37e7e2cb0..344ab1ba7a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogLayer.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogLayer.java @@ -14,12 +14,13 @@ * */ -package org.hyperledger.besu.ethereum.bonsai; +package org.hyperledger.besu.ethereum.bonsai.trielog; import static com.google.common.base.Preconditions.checkState; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; @@ -53,7 +54,7 @@ public class TrieLogLayer { private final Map>> storage; private boolean frozen = false; - TrieLogLayer() { + public TrieLogLayer() { // TODO when tuweni fixes zero length byte comparison consider TreeMap this.accounts = new HashMap<>(); this.code = new HashMap<>(); @@ -74,7 +75,7 @@ public class TrieLogLayer { this.blockHash = blockHash; } - void addAccountChange( + public void addAccountChange( final Address address, final StateTrieAccountValue oldValue, final StateTrieAccountValue newValue) { @@ -82,7 +83,7 @@ public class TrieLogLayer { accounts.put(address, new BonsaiValue<>(oldValue, newValue)); } - void addCodeChange( + public void addCodeChange( final Address address, final Bytes oldValue, final Bytes newValue, final Hash blockHash) { checkState(!frozen, "Layer is Frozen"); code.put( @@ -91,7 +92,7 @@ public class TrieLogLayer { oldValue == null ? Bytes.EMPTY : oldValue, newValue == null ? Bytes.EMPTY : newValue)); } - void addStorageChange( + public void addStorageChange( final Address address, final Hash slotHash, final UInt256 oldValue, final UInt256 newValue) { checkState(!frozen, "Layer is Frozen"); storage diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java similarity index 55% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java index 31720cc73b..e63f437dec 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java @@ -13,38 +13,37 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.bonsai; +package org.hyperledger.besu.ethereum.bonsai.trielog; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import java.util.Optional; +import java.util.function.Function; public interface TrieLogManager { void saveTrieLog( - final BonsaiWorldStateArchive worldStateArchive, - final BonsaiWorldStateUpdater localUpdater, + final BonsaiWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiPersistedWorldState forWorldState); + final BonsaiWorldState forWorldState); - Optional getBonsaiCachedWorldState(final Hash blockHash); + void addCachedLayer( + BlockHeader blockHeader, Hash worldStateRootHash, BonsaiWorldState forWorldState); - long getMaxLayersToLoad(); - - void updateCachedLayers(final Hash blockParentHash, final Hash blockHash); + boolean containWorldStateStorage(final Hash blockHash); - Optional getTrieLogLayer(final Hash blockHash); + Optional getWorldState(final Hash blockHash); - interface CachedWorldState { - void dispose(); + Optional getNearestWorldState(final BlockHeader blockHeader); - long getHeight(); + Optional getHeadWorldState( + final Function> hashBlockHeaderFunction); - TrieLogLayer getTrieLog(); + long getMaxLayersToLoad(); - Z getMutableWorldState(); - } + Optional getTrieLogLayer(final Hash blockHash); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java new file mode 100644 index 0000000000..9a900a6491 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -0,0 +1,570 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai.worldview; + +import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator.StorageConsumingMap; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; +import org.hyperledger.besu.ethereum.trie.MerkleTrieException; +import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BonsaiWorldState + implements MutableWorldState, BonsaiWorldView, BonsaiStorageSubscriber { + + private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class); + + public BonsaiWorldStateKeyValueStorage worldStateStorage; + + private final BonsaiWorldStateProvider archive; + private final BonsaiWorldStateUpdateAccumulator accumulator; + + public Hash worldStateRootHash; + public Hash worldStateBlockHash; + + private boolean isFrozen; + + public BonsaiWorldState( + final BonsaiWorldStateProvider archive, + final BonsaiWorldStateKeyValueStorage worldStateStorage) { + this.archive = archive; + this.worldStateStorage = worldStateStorage; + worldStateRootHash = + Hash.wrap( + Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); + worldStateBlockHash = + Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); + accumulator = + new BonsaiWorldStateUpdateAccumulator( + this, + (addr, value) -> + archive + .getCachedMerkleTrieLoader() + .preLoadAccount(getWorldStateStorage(), worldStateRootHash, addr), + (addr, value) -> + archive + .getCachedMerkleTrieLoader() + .preLoadStorageSlot(getWorldStateStorage(), addr, value)); + } + + public BonsaiWorldState( + final BonsaiWorldStateProvider archive, + final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateUpdateAccumulator updater) { + this.archive = archive; + this.worldStateStorage = worldStateStorage; + this.worldStateRootHash = + Hash.wrap( + Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); + this.worldStateBlockHash = + Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); + this.accumulator = updater; + } + + public BonsaiWorldStateProvider getArchive() { + return archive; + } + + @Override + public boolean isPersisted() { + return isPersisted(worldStateStorage); + } + + private boolean isPersisted(final WorldStateStorage worldStateStorage) { + return !(worldStateStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); + } + + @Override + public Optional getCode(@Nonnull final Address address, final Hash codeHash) { + return worldStateStorage.getCode(codeHash, Hash.hash(address)); + } + + public void setArchiveStateUnSafe(final BlockHeader blockHeader) { + worldStateBlockHash = Hash.fromPlugin(blockHeader.getBlockHash()); + worldStateRootHash = Hash.fromPlugin(blockHeader.getStateRoot()); + } + + @Override + public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + return worldStateStorage; + } + + private Hash calculateRootHash( + final Optional maybeStateUpdater, + final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { + + clearStorage(maybeStateUpdater, worldStateUpdater); + + // This must be done before updating the accounts so + // that we can get the storage state hash + Stream>>> storageStream = + worldStateUpdater.getStorageToUpdate().entrySet().stream(); + if (maybeStateUpdater.isEmpty()) { + storageStream = + storageStream + .parallel(); // if we are not updating the state updater we can use parallel stream + } + storageStream.forEach( + addressMapEntry -> + updateAccountStorageState(maybeStateUpdater, worldStateUpdater, addressMapEntry)); + + // Third update the code. This has the side effect of ensuring a code hash is calculated. + updateCode(maybeStateUpdater, worldStateUpdater); + + // next walk the account trie + final StoredMerklePatriciaTrie accountTrie = + createTrie( + (location, hash) -> + archive + .getCachedMerkleTrieLoader() + .getAccountStateTrieNode(worldStateStorage, location, hash), + worldStateRootHash); + + // for manicured tries and composting, collect branches here (not implemented) + updateTheAccounts(maybeStateUpdater, worldStateUpdater, accountTrie); + + // TODO write to a cache and then generate a layer update from that and the + // DB tx updates. Right now it is just DB updates. + maybeStateUpdater.ifPresent( + bonsaiUpdater -> { + accountTrie.commit( + (location, hash, value) -> + writeTrieNode(bonsaiUpdater.getTrieBranchStorageTransaction(), location, value)); + }); + final Bytes32 rootHash = accountTrie.getRootHash(); + return Hash.wrap(rootHash); + } + + private void updateTheAccounts( + final Optional maybeStateUpdater, + final BonsaiWorldStateUpdateAccumulator worldStateUpdater, + final StoredMerklePatriciaTrie accountTrie) { + for (final Map.Entry> accountUpdate : + worldStateUpdater.getAccountsToUpdate().entrySet()) { + final Bytes accountKey = accountUpdate.getKey(); + final BonsaiValue bonsaiValue = accountUpdate.getValue(); + final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); + try { + if (updatedAccount == null) { + final Hash addressHash = Hash.hash(accountKey); + accountTrie.remove(addressHash); + maybeStateUpdater.ifPresent( + bonsaiUpdater -> bonsaiUpdater.removeAccountInfoState(addressHash)); + } else { + final Hash addressHash = updatedAccount.getAddressHash(); + final Bytes accountValue = updatedAccount.serializeAccount(); + maybeStateUpdater.ifPresent( + bonsaiUpdater -> + bonsaiUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue)); + accountTrie.put(addressHash, accountValue); + } + } catch (MerkleTrieException e) { + // need to throw to trigger the heal + throw new MerkleTrieException( + e.getMessage(), Optional.of(Address.wrap(accountKey)), e.getHash(), e.getLocation()); + } + } + } + + private void updateCode( + final Optional maybeStateUpdater, + final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { + maybeStateUpdater.ifPresent( + bonsaiUpdater -> { + for (final Map.Entry> codeUpdate : + worldStateUpdater.getCodeToUpdate().entrySet()) { + final Bytes updatedCode = codeUpdate.getValue().getUpdated(); + final Hash accountHash = Hash.hash(codeUpdate.getKey()); + if (updatedCode == null || updatedCode.size() == 0) { + bonsaiUpdater.removeCode(accountHash); + } else { + bonsaiUpdater.putCode(accountHash, null, updatedCode); + } + } + }); + } + + private void updateAccountStorageState( + final Optional maybeStateUpdater, + final BonsaiWorldStateUpdateAccumulator worldStateUpdater, + final Map.Entry>> storageAccountUpdate) { + final Address updatedAddress = storageAccountUpdate.getKey(); + final Hash updatedAddressHash = Hash.hash(updatedAddress); + if (worldStateUpdater.getAccountsToUpdate().containsKey(updatedAddress)) { + final BonsaiValue accountValue = + worldStateUpdater.getAccountsToUpdate().get(updatedAddress); + final BonsaiAccount accountOriginal = accountValue.getPrior(); + final Hash storageRoot = + (accountOriginal == null) ? Hash.EMPTY_TRIE_HASH : accountOriginal.getStorageRoot(); + final StoredMerklePatriciaTrie storageTrie = + createTrie( + (location, key) -> + archive + .getCachedMerkleTrieLoader() + .getAccountStorageTrieNode( + worldStateStorage, updatedAddressHash, location, key), + storageRoot); + + // for manicured tries and composting, collect branches here (not implemented) + for (final Map.Entry> storageUpdate : + storageAccountUpdate.getValue().entrySet()) { + final Hash keyHash = storageUpdate.getKey(); + final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); + try { + if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) { + maybeStateUpdater.ifPresent( + bonsaiUpdater -> + bonsaiUpdater.removeStorageValueBySlotHash(updatedAddressHash, keyHash)); + storageTrie.remove(keyHash); + } else { + maybeStateUpdater.ifPresent( + bonsaiUpdater -> + bonsaiUpdater.putStorageValueBySlotHash( + updatedAddressHash, keyHash, updatedStorage)); + storageTrie.put(keyHash, BonsaiWorldView.encodeTrieValue(updatedStorage)); + } + } catch (MerkleTrieException e) { + // need to throw to trigger the heal + throw new MerkleTrieException( + e.getMessage(), + Optional.of(Address.wrap(updatedAddress)), + e.getHash(), + e.getLocation()); + } + } + + final BonsaiAccount accountUpdated = accountValue.getUpdated(); + if (accountUpdated != null) { + maybeStateUpdater.ifPresent( + bonsaiUpdater -> { + storageTrie.commit( + (location, key, value) -> + writeStorageTrieNode( + bonsaiUpdater, updatedAddressHash, location, key, value)); + }); + final Hash newStorageRoot = Hash.wrap(storageTrie.getRootHash()); + accountUpdated.setStorageRoot(newStorageRoot); + } + } + // for manicured tries and composting, trim and compost here + } + + private void clearStorage( + final Optional maybeStateUpdater, + final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { + + maybeStateUpdater.ifPresent( + bonsaiUpdater -> { + for (final Address address : worldStateUpdater.getStorageToClear()) { + // because we are clearing persisted values we need the account root as persisted + final BonsaiAccount oldAccount = + worldStateStorage + .getAccount(Hash.hash(address)) + .map(bytes -> fromRLP(BonsaiWorldState.this, address, bytes, true)) + .orElse(null); + if (oldAccount == null) { + // This is when an account is both created and deleted within the scope of the same + // block. A not-uncommon DeFi bot pattern. + continue; + } + final Hash addressHash = Hash.hash(address); + final MerklePatriciaTrie storageTrie = + createTrie( + (location, key) -> getStorageTrieNode(addressHash, location, key), + oldAccount.getStorageRoot()); + try { + Map entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); + while (!entriesToDelete.isEmpty()) { + entriesToDelete + .keySet() + .forEach( + k -> + bonsaiUpdater.removeStorageValueBySlotHash( + Hash.hash(address), Hash.wrap(k))); + entriesToDelete.keySet().forEach(storageTrie::remove); + if (entriesToDelete.size() == 256) { + entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); + } else { + break; + } + } + } catch (MerkleTrieException e) { + // need to throw to trigger the heal + throw new MerkleTrieException( + e.getMessage(), Optional.of(Address.wrap(address)), e.getHash(), e.getLocation()); + } + } + }); + } + + @Override + public void persist(final BlockHeader blockHeader) { + final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); + LOG.atDebug() + .setMessage("Persist world state for block {}") + .addArgument(maybeBlockHeader) + .log(); + + final BonsaiWorldStateUpdateAccumulator localCopy = accumulator.copy(); + + boolean success = false; + + final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater = worldStateStorage.updater(); + Runnable saveTrieLog = () -> {}; + + try { + final Hash newWorldStateRootHash = + calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); + // if we are persisted with a block header, and the prior state is the parent + // then persist the TrieLog for that transition. + // If specified but not a direct descendant simply store the new block hash. + if (blockHeader != null) { + if (!newWorldStateRootHash.equals(blockHeader.getStateRoot())) { + throw new RuntimeException( + "World State Root does not match expected value, header " + + blockHeader.getStateRoot().toHexString() + + " calculated " + + newWorldStateRootHash.toHexString()); + } + saveTrieLog = + () -> { + final TrieLogManager trieLogManager = archive.getTrieLogManager(); + trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); + // not save a frozen state in the cache + if (!isFrozen) { + trieLogManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); + } + }; + + stateUpdater + .getTrieBranchStorageTransaction() + .put(WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); + worldStateBlockHash = blockHeader.getHash(); + } else { + stateUpdater.getTrieBranchStorageTransaction().remove(WORLD_BLOCK_HASH_KEY); + worldStateBlockHash = null; + } + + stateUpdater + .getTrieBranchStorageTransaction() + .put(WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); + worldStateRootHash = newWorldStateRootHash; + success = true; + } finally { + if (success) { + stateUpdater.commit(); + accumulator.reset(); + saveTrieLog.run(); + } else { + stateUpdater.rollback(); + accumulator.reset(); + } + } + } + + @Override + public WorldUpdater updater() { + return accumulator; + } + + @Override + public Hash rootHash() { + if (isFrozen && accumulator.isAccumulatorStateChanged()) { + worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); + accumulator.resetAccumulatorStateChanged(); + } + return Hash.wrap(worldStateRootHash); + } + + static final KeyValueStorageTransaction noOpTx = + new KeyValueStorageTransaction() { + + @Override + public void put(final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + @Override + public Hash frontierRootHash() { + return calculateRootHash( + Optional.of( + new BonsaiWorldStateKeyValueStorage.Updater(noOpTx, noOpTx, noOpTx, noOpTx, noOpTx)), + accumulator.copy()); + } + + public Hash blockHash() { + return worldStateBlockHash; + } + + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + throw new RuntimeException("Bonsai Tries do not provide account streaming."); + } + + @Override + public Account get(final Address address) { + return worldStateStorage + .getAccount(Hash.hash(address)) + .map(bytes -> fromRLP(accumulator, address, bytes, true)) + .orElse(null); + } + + protected Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { + return worldStateStorage.getAccountStateTrieNode(location, nodeHash); + } + + private void writeTrieNode( + final KeyValueStorageTransaction tx, final Bytes location, final Bytes value) { + tx.put(location.toArrayUnsafe(), value.toArrayUnsafe()); + } + + protected Optional getStorageTrieNode( + final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { + return worldStateStorage.getAccountStorageTrieNode(accountHash, location, nodeHash); + } + + private void writeStorageTrieNode( + final WorldStateStorage.Updater stateUpdater, + final Hash accountHash, + final Bytes location, + final Bytes32 nodeHash, + final Bytes value) { + stateUpdater.putAccountStorageTrieNode(accountHash, location, nodeHash, value); + } + + @Override + public UInt256 getStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValueBySlotHash(address, Hash.hash(storageKey)).orElse(UInt256.ZERO); + } + + @Override + public Optional getStorageValueBySlotHash(final Address address, final Hash slotHash) { + return worldStateStorage + .getStorageValueBySlotHash(Hash.hash(address), slotHash) + .map(UInt256::fromBytes); + } + + public Optional getStorageValueBySlotHash( + final Supplier> storageRootSupplier, + final Address address, + final Hash slotHash) { + return worldStateStorage + .getStorageValueBySlotHash(storageRootSupplier, Hash.hash(address), slotHash) + .map(UInt256::fromBytes); + } + + @Override + public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValue(address, storageKey); + } + + @Override + public Map getAllAccountStorage(final Address address, final Hash rootHash) { + final StoredMerklePatriciaTrie storageTrie = + createTrie( + (location, key) -> getStorageTrieNode(Hash.hash(address), location, key), rootHash); + return storageTrie.entriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); + } + + @Override + public MutableWorldState freeze() { + this.isFrozen = true; + this.worldStateStorage = new BonsaiWorldStateLayerStorage(worldStateStorage); + return this; + } + + private StoredMerklePatriciaTrie createTrie( + final NodeLoader nodeLoader, final Bytes32 rootHash) { + return new StoredMerklePatriciaTrie<>( + nodeLoader, rootHash, Function.identity(), Function.identity()); + } + + @Override + public void close() { + try { + if (!isPersisted()) { + this.worldStateStorage.close(); + if (isFrozen) { + closeFrozenStorage(); + } + } + } catch (Exception e) { + // no op + } + } + + private void closeFrozenStorage() { + try { + final BonsaiWorldStateLayerStorage worldStateLayerStorage = + (BonsaiWorldStateLayerStorage) worldStateStorage; + if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { + worldStateLayerStorage.getParentWorldStateStorage().close(); + } + } catch (Exception e) { + // no op + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java similarity index 91% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 2078f6e83f..d6fb5359ad 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -14,11 +14,15 @@ * */ -package org.hyperledger.besu.ethereum.bonsai; +package org.hyperledger.besu.ethereum.bonsai.worldview; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -48,13 +52,14 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateUpdater extends AbstractWorldUpdater - implements BonsaiWorldView { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateUpdater.class); - - private final AccountConsumingMap> accountsToUpdate; +public class BonsaiWorldStateUpdateAccumulator + extends AbstractWorldUpdater implements BonsaiWorldView { + private static final Logger LOG = + LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class); private final Consumer> accountPreloader; private final Consumer storagePreloader; + + private final AccountConsumingMap> accountsToUpdate; private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set
storageToClear = Collections.synchronizedSet(new HashSet<>()); private final Set emptySlot = Collections.synchronizedSet(new HashSet<>()); @@ -65,11 +70,9 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater>> storageToUpdate = new ConcurrentHashMap<>(); - BonsaiWorldStateUpdater(final BonsaiWorldView world) { - this(world, (__, ___) -> {}, (__, ___) -> {}); - } + private boolean isAccumulatorStateChanged; - BonsaiWorldStateUpdater( + public BonsaiWorldStateUpdateAccumulator( final BonsaiWorldView world, final Consumer> accountPreloader, final Consumer storagePreloader) { @@ -77,16 +80,18 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater(new ConcurrentHashMap<>(), accountPreloader); this.accountPreloader = accountPreloader; this.storagePreloader = storagePreloader; + this.isAccumulatorStateChanged = false; } - public BonsaiWorldStateUpdater copy() { - final BonsaiWorldStateUpdater copy = - new BonsaiWorldStateUpdater(wrappedWorldView(), accountPreloader, storagePreloader); + public BonsaiWorldStateUpdateAccumulator copy() { + final BonsaiWorldStateUpdateAccumulator copy = + new BonsaiWorldStateUpdateAccumulator( + wrappedWorldView(), accountPreloader, storagePreloader); copy.cloneFromUpdater(this); return copy; } - void cloneFromUpdater(final BonsaiWorldStateUpdater source) { + void cloneFromUpdater(final BonsaiWorldStateUpdateAccumulator source) { accountsToUpdate.putAll(source.getAccountsToUpdate()); codeToUpdate.putAll(source.codeToUpdate); storageToClear.addAll(source.storageToClear); @@ -94,6 +99,7 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater bonsaiValue = accountsToUpdate.get(address); + if (bonsaiValue == null) { bonsaiValue = new BonsaiValue<>(null, null); accountsToUpdate.put(address, bonsaiValue); } else if (bonsaiValue.getUpdated() != null) { throw new IllegalStateException("Cannot create an account when one already exists"); } + final BonsaiAccount newAccount = new BonsaiAccount( this, @@ -135,11 +143,11 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater(newAccount))); } - Map> getAccountsToUpdate() { + public Map> getAccountsToUpdate() { return accountsToUpdate; } - Map> getCodeToUpdate() { + public Map> getCodeToUpdate() { return codeToUpdate; } @@ -147,7 +155,7 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater>> getStorageToUpdate() { + public Map>> getStorageToUpdate() { return storageToUpdate; } @@ -162,7 +170,14 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater bonsaiValue = accountsToUpdate.get(address); if (bonsaiValue == null) { - final Account account = wrappedWorldView().get(address); + final Account account; + if (wrappedWorldView() instanceof BonsaiWorldStateUpdateAccumulator) { + account = + ((BonsaiWorldStateUpdateAccumulator) wrappedWorldView()) + .loadAccount(address, bonsaiAccountFunction); + } else { + account = wrappedWorldView().get(address); + } if (account instanceof BonsaiAccount) { final BonsaiAccount mutableAccount = new BonsaiAccount((BonsaiAccount) account, this, true); @@ -198,6 +213,7 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater accountValue = accountsToUpdate.computeIfAbsent( @@ -367,12 +383,6 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater getStateTrieNode(final Bytes location) { - // updater doesn't track trie nodes. Always a miss. - return Optional.empty(); - } - @Override public UInt256 getStorageValue(final Address address, final UInt256 storageKey) { // TODO maybe log the read into the trie layer? @@ -395,8 +405,8 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater valueUInt = - (wrappedWorldView() instanceof BonsaiPersistedWorldState) - ? ((BonsaiPersistedWorldState) wrappedWorldView()) + (wrappedWorldView() instanceof BonsaiWorldState) + ? ((BonsaiWorldState) wrappedWorldView()) .getStorageValueBySlotHash( () -> Optional.ofNullable(loadAccount(address, BonsaiValue::getPrior)) @@ -455,10 +465,24 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater getAllAccountStorage(final Address address, final Hash rootHash) { final Map results = wrappedWorldView().getAllAccountStorage(address, rootHash); - storageToUpdate.get(address).forEach((key, value) -> results.put(key, value.getUpdated())); + final StorageConsumingMap> bonsaiValueStorage = + storageToUpdate.get(address); + if (bonsaiValueStorage != null) { + bonsaiValueStorage.forEach((key, value) -> results.put(key, value.getUpdated())); + } return results; } + @Override + public boolean isPersisted() { + return true; + } + + @Override + public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + return wrappedWorldView().getWorldStateStorage(); + } + public TrieLogLayer generateTrieLog(final Hash blockHash) { final TrieLogLayer layer = new TrieLogLayer(); importIntoTrieLog(layer, blockHash); @@ -600,6 +624,8 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater> getAccountPreloader() { + return accountPreloader; + } + + public Consumer getStoragePreloader() { + return storagePreloader; + } + @Override public void reset() { storageToClear.clear(); @@ -776,18 +818,10 @@ public class BonsaiWorldStateUpdater extends AbstractWorldUpdater extends ForwardingMap { private final ConcurrentHashMap accounts; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldView.java similarity index 80% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldView.java index 8847dd9547..41123405f3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldView.java @@ -14,11 +14,13 @@ * */ -package org.hyperledger.besu.ethereum.bonsai; +package org.hyperledger.besu.ethereum.bonsai.worldview; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evm.worldstate.WorldView; import java.util.Map; @@ -32,8 +34,6 @@ public interface BonsaiWorldView extends WorldView { Optional getCode(Address address, final Hash codeHash); - Optional getStateTrieNode(Bytes location); - UInt256 getStorageValue(Address address, UInt256 key); Optional getStorageValueBySlotHash(Address address, Hash slotHash); @@ -41,18 +41,24 @@ public interface BonsaiWorldView extends WorldView { UInt256 getPriorStorageValue(Address address, UInt256 key); /** - * Retrieve all the storage values of a account. + * Retrieve all the storage values of an account. * * @param address the account to stream * @param rootHash the root hash of the account storage trie * @return A map that is a copy of the entries. The key is the hashed slot number, and the value * is the Bytes representation of the storage value. */ - Map getAllAccountStorage(Address address, Hash rootHash); + Map getAllAccountStorage(final Address address, final Hash rootHash); static Bytes encodeTrieValue(final Bytes bytes) { final BytesValueRLPOutput out = new BytesValueRLPOutput(); out.writeBytes(bytes.trimLeadingZeros()); return out.encoded(); } + + boolean isPersisted(); + + BonsaiWorldStateKeyValueStorage getWorldStateStorage(); + + WorldUpdater updater(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java index 4d4e7e078e..0e016a8db8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java @@ -19,13 +19,6 @@ import org.hyperledger.besu.evm.worldstate.WorldState; public interface MutableWorldState extends WorldState, MutableWorldView { - /** - * Creates an independent copy of this world state initially equivalent to this world state. - * - * @return a copy of this world state. - */ - MutableWorldState copy(); - /** * Persist accumulated changes to underlying storage. * @@ -35,13 +28,8 @@ public interface MutableWorldState extends WorldState, MutableWorldView { */ void persist(BlockHeader blockHeader); - /** - * Returns whether this mutable worldstate can be persisted to storage. - * - * @return boolean - */ - default boolean isPersistable() { - // default to true - return true; + default MutableWorldState freeze() { + // no op + throw new UnsupportedOperationException("cannot freeze"); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/SnapshotMutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/SnapshotMutableWorldState.java deleted file mode 100644 index 941373f440..0000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/SnapshotMutableWorldState.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ -package org.hyperledger.besu.ethereum.core; - -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; - -public interface SnapshotMutableWorldState extends MutableWorldState, AutoCloseable { - BonsaiWorldStateKeyValueStorage getWorldStateStorage(); -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 1fa628d1fd..886df1b0a3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -19,8 +19,8 @@ import org.hyperledger.besu.datatypes.DataGas; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.BlockProcessingOutputs; import org.hyperledger.besu.ethereum.BlockProcessingResult; -import org.hyperledger.besu.ethereum.bonsai.BonsaiPersistedWorldState; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateUpdater; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; @@ -140,8 +140,8 @@ public abstract class AbstractBlockProcessor implements BlockProcessor { blockHeader.getHash().toHexString(), transaction.getHash().toHexString()); LOG.info(errorMessage); - if (worldState instanceof BonsaiPersistedWorldState) { - ((BonsaiWorldStateUpdater) worldStateUpdater).reset(); + if (worldState instanceof BonsaiWorldState) { + ((BonsaiWorldStateUpdateAccumulator) worldStateUpdater).reset(); } return new BlockProcessingResult(Optional.empty(), errorMessage); } @@ -169,8 +169,8 @@ public abstract class AbstractBlockProcessor implements BlockProcessor { if (!rewardCoinbase(worldState, blockHeader, ommers, skipZeroBlockRewards)) { // no need to log, rewardCoinbase logs the error. - if (worldState instanceof BonsaiPersistedWorldState) { - ((BonsaiWorldStateUpdater) worldState.updater()).reset(); + if (worldState instanceof BonsaiWorldState) { + ((BonsaiWorldStateUpdateAccumulator) worldState.updater()).reset(); } return new BlockProcessingResult(Optional.empty(), "ommer too old"); } @@ -179,8 +179,8 @@ public abstract class AbstractBlockProcessor implements BlockProcessor { worldState.persist(blockHeader); } catch (MerkleTrieException e) { LOG.trace("Merkle trie exception during Transaction processing ", e); - if (worldState instanceof BonsaiPersistedWorldState) { - ((BonsaiWorldStateUpdater) worldState.updater()).reset(); + if (worldState instanceof BonsaiWorldState) { + ((BonsaiWorldStateUpdateAccumulator) worldState.updater()).reset(); } throw e; } catch (Exception e) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractProtocolScheduleBuilder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractProtocolScheduleBuilder.java index f7e650f9cc..15da3bd436 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractProtocolScheduleBuilder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractProtocolScheduleBuilder.java @@ -99,7 +99,7 @@ public abstract class AbstractProtocolScheduleBuilder { addProtocolSpec( protocolSchedule, e.getBlockIdentifier(), e.getBuilder(), e.modifier)); - postBuildStep(specFactory); + postBuildStep(specFactory, builders); LOG.info("Protocol schedule created with milestones: {}", protocolSchedule.listMilestones()); } @@ -112,18 +112,12 @@ public abstract class AbstractProtocolScheduleBuilder { if (lastForkBlock > referenceForkBlock) { throw new RuntimeException( String.format( - "Genesis Config Error: '%s' is scheduled for %s %d but it must be on or after %s %d.", - forkName, - getBlockIdentifierName(), - thisForkBlock.getAsLong(), - getBlockIdentifierName(), - lastForkBlock)); + "Genesis Config Error: '%s' is scheduled for milestone %d but it must be on or after milestone %d.", + forkName, thisForkBlock.getAsLong(), lastForkBlock)); } return referenceForkBlock; } - abstract String getBlockIdentifierName(); - private TreeMap buildMilestoneMap( final MainnetProtocolSpecFactory specFactory) { return createMilestones(specFactory) @@ -149,9 +143,8 @@ public abstract class AbstractProtocolScheduleBuilder { new BuilderMapEntry(blockVal, builder, protocolSpecAdapters.getModifierForBlock(blockVal))); } - protected void addProtocolSpec( + protected ProtocolSpec getProtocolSpec( final HeaderBasedProtocolSchedule protocolSchedule, - final long blockNumberOrTimestamp, final ProtocolSpecBuilder definition, final Function modifier) { definition @@ -160,11 +153,21 @@ public abstract class AbstractProtocolScheduleBuilder { .privateTransactionValidatorBuilder( () -> new PrivateTransactionValidator(protocolSchedule.getChainId())); + return modifier.apply(definition).build(protocolSchedule); + } + + protected void addProtocolSpec( + final HeaderBasedProtocolSchedule protocolSchedule, + final long blockNumberOrTimestamp, + final ProtocolSpecBuilder definition, + final Function modifier) { + protocolSchedule.putMilestone( - blockNumberOrTimestamp, modifier.apply(definition).build(protocolSchedule)); + blockNumberOrTimestamp, getProtocolSpec(protocolSchedule, definition, modifier)); } - abstract void postBuildStep(final MainnetProtocolSpecFactory specFactory); + abstract void postBuildStep( + final MainnetProtocolSpecFactory specFactory, final TreeMap builders); protected static class BuilderMapEntry { @@ -188,5 +191,9 @@ public abstract class AbstractProtocolScheduleBuilder { public ProtocolSpecBuilder getBuilder() { return builder; } + + public Function getModifier() { + return modifier; + } } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/DefaultTimestampSchedule.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/DefaultTimestampSchedule.java index 307339abd7..c5e210f541 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/DefaultTimestampSchedule.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/DefaultTimestampSchedule.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.mainnet; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.TransactionFilter; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; @@ -25,12 +26,13 @@ import java.util.Comparator; import java.util.NavigableSet; import java.util.Optional; import java.util.TreeSet; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; public class DefaultTimestampSchedule implements TimestampSchedule { - private final NavigableSet protocolSpecs = - new TreeSet<>(Comparator.comparing(TimedProtocolSpec::getTimestamp).reversed()); + private final NavigableSet protocolSpecs = + new TreeSet<>(Comparator.comparing(ScheduledProtocolSpec::milestone).reversed()); private final Optional chainId; DefaultTimestampSchedule(final Optional chainId) { @@ -39,9 +41,9 @@ public class DefaultTimestampSchedule implements TimestampSchedule { @Override public Optional getByTimestamp(final long timestamp) { - for (final TimedProtocolSpec protocolSpec : protocolSpecs) { - if (protocolSpec.getTimestamp() <= timestamp) { - return Optional.of(protocolSpec.getSpec()); + for (final ScheduledProtocolSpec protocolSpec : protocolSpecs) { + if (protocolSpec.milestone() <= timestamp) { + return Optional.of(protocolSpec.spec()); } } return Optional.empty(); @@ -49,7 +51,17 @@ public class DefaultTimestampSchedule implements TimestampSchedule { @Override public Stream streamMilestoneBlocks() { - return protocolSpecs.stream().map(TimedProtocolSpec::getTimestamp).sorted(); + return protocolSpecs.stream().map(ScheduledProtocolSpec::milestone).sorted(); + } + + @Override + public boolean anyMatch(final Predicate predicate) { + return this.protocolSpecs.stream().anyMatch(predicate); + } + + @Override + public boolean isOnMilestoneBoundary(final BlockHeader blockHeader) { + return this.protocolSpecs.stream().anyMatch(s -> blockHeader.getTimestamp() == s.milestone()); } @Override @@ -59,7 +71,8 @@ public class DefaultTimestampSchedule implements TimestampSchedule { @Override public void putMilestone(final long timestamp, final ProtocolSpec protocolSpec) { - final TimedProtocolSpec scheduledProtocolSpec = new TimedProtocolSpec(timestamp, protocolSpec); + final ScheduledProtocolSpec scheduledProtocolSpec = + new ScheduledProtocolSpec(timestamp, protocolSpec); // Ensure this replaces any existing spec at the same block number. protocolSpecs.remove(scheduledProtocolSpec); protocolSpecs.add(scheduledProtocolSpec); @@ -68,15 +81,15 @@ public class DefaultTimestampSchedule implements TimestampSchedule { @Override public String listMilestones() { return protocolSpecs.stream() - .sorted(Comparator.comparing(TimedProtocolSpec::getTimestamp)) - .map(spec -> spec.getSpec().getName() + ": " + spec.getTimestamp()) + .sorted(Comparator.comparing(ScheduledProtocolSpec::milestone)) + .map(scheduledSpec -> scheduledSpec.spec().getName() + ": " + scheduledSpec.milestone()) .collect(Collectors.joining(", ", "[", "]")); } @Override public void setTransactionFilter(final TransactionFilter transactionFilter) { protocolSpecs.forEach( - spec -> spec.getSpec().getTransactionValidator().setTransactionFilter(transactionFilter)); + spec -> spec.spec().getTransactionValidator().setTransactionFilter(transactionFilter)); } @Override @@ -84,28 +97,10 @@ public class DefaultTimestampSchedule implements TimestampSchedule { final WorldStateArchive publicWorldStateArchive) { protocolSpecs.forEach( spec -> { - final BlockProcessor blockProcessor = spec.getSpec().getBlockProcessor(); + final BlockProcessor blockProcessor = spec.spec().getBlockProcessor(); if (PrivacyBlockProcessor.class.isAssignableFrom(blockProcessor.getClass())) ((PrivacyBlockProcessor) blockProcessor) .setPublicWorldStateArchive(publicWorldStateArchive); }); } - - private static class TimedProtocolSpec { - private final long timestamp; - private final ProtocolSpec spec; - - public TimedProtocolSpec(final long timestamp, final ProtocolSpec spec) { - this.timestamp = timestamp; - this.spec = spec; - } - - public long getTimestamp() { - return timestamp; - } - - public ProtocolSpec getSpec() { - return spec; - } - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/HeaderBasedProtocolSchedule.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/HeaderBasedProtocolSchedule.java index 05f72726fe..dcbf7cc515 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/HeaderBasedProtocolSchedule.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/HeaderBasedProtocolSchedule.java @@ -17,10 +17,12 @@ package org.hyperledger.besu.ethereum.mainnet; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; import java.math.BigInteger; import java.util.Optional; +import java.util.function.Predicate; import java.util.stream.Stream; public interface HeaderBasedProtocolSchedule { @@ -34,4 +36,8 @@ public interface HeaderBasedProtocolSchedule { String listMilestones(); Stream streamMilestoneBlocks(); + + boolean anyMatch(Predicate predicate); + + boolean isOnMilestoneBoundary(final BlockHeader blockHeader); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MutableProtocolSchedule.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MutableProtocolSchedule.java index 8145af43ac..2d8ac70fe4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MutableProtocolSchedule.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MutableProtocolSchedule.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.ethereum.mainnet; import static com.google.common.base.Preconditions.checkArgument; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.TransactionFilter; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; @@ -25,15 +26,14 @@ import java.util.List; import java.util.NavigableSet; import java.util.Optional; import java.util.TreeSet; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; public class MutableProtocolSchedule implements ProtocolSchedule { private final NavigableSet protocolSpecs = - new TreeSet<>( - Comparator.comparing(ScheduledProtocolSpec::getBlock) - .reversed()); + new TreeSet<>(Comparator.comparing(ScheduledProtocolSpec::milestone).reversed()); private final Optional chainId; public MutableProtocolSchedule(final Optional chainId) { @@ -60,12 +60,12 @@ public class MutableProtocolSchedule implements ProtocolSchedule { checkArgument( !protocolSpecs.isEmpty(), "At least 1 milestone must be provided to the protocol schedule"); checkArgument( - protocolSpecs.last().getBlock() == 0, "There must be a milestone starting from block 0"); + protocolSpecs.last().milestone() == 0, "There must be a milestone starting from block 0"); // protocolSpecs is sorted in descending block order, so the first one we find that's lower than // the requested level will be the most appropriate spec for (final ScheduledProtocolSpec s : protocolSpecs) { - if (number >= s.getBlock()) { - return s.getSpec(); + if (number >= s.milestone()) { + return s.spec(); } } return null; @@ -74,22 +74,32 @@ public class MutableProtocolSchedule implements ProtocolSchedule { @Override public String listMilestones() { return protocolSpecs.stream() - .sorted(Comparator.comparing(ScheduledProtocolSpec::getBlock)) - .map(spec -> spec.getSpec().getName() + ": " + spec.getBlock()) + .sorted(Comparator.comparing(ScheduledProtocolSpec::milestone)) + .map(scheduledSpec -> scheduledSpec.spec().getName() + ": " + scheduledSpec.milestone()) .collect(Collectors.joining(", ", "[", "]")); } @Override public Stream streamMilestoneBlocks() { return protocolSpecs.stream() - .sorted(Comparator.comparing(ScheduledProtocolSpec::getBlock)) - .map(ScheduledProtocolSpec::getBlock); + .sorted(Comparator.comparing(ScheduledProtocolSpec::milestone)) + .map(ScheduledProtocolSpec::milestone); + } + + @Override + public boolean anyMatch(final Predicate predicate) { + return this.protocolSpecs.stream().anyMatch(predicate); + } + + @Override + public boolean isOnMilestoneBoundary(final BlockHeader blockHeader) { + return this.protocolSpecs.stream().anyMatch(s -> blockHeader.getNumber() == s.milestone()); } @Override public void setTransactionFilter(final TransactionFilter transactionFilter) { protocolSpecs.forEach( - spec -> spec.getSpec().getTransactionValidator().setTransactionFilter(transactionFilter)); + spec -> spec.spec().getTransactionValidator().setTransactionFilter(transactionFilter)); } @Override @@ -97,7 +107,7 @@ public class MutableProtocolSchedule implements ProtocolSchedule { final WorldStateArchive publicWorldStateArchive) { protocolSpecs.forEach( spec -> { - final BlockProcessor blockProcessor = spec.getSpec().getBlockProcessor(); + final BlockProcessor blockProcessor = spec.spec().getBlockProcessor(); if (PrivacyBlockProcessor.class.isAssignableFrom(blockProcessor.getClass())) ((PrivacyBlockProcessor) blockProcessor) .setPublicWorldStateArchive(publicWorldStateArchive); @@ -105,6 +115,6 @@ public class MutableProtocolSchedule implements ProtocolSchedule { } public List getScheduledProtocolSpecs() { - return protocolSpecs.stream().collect(Collectors.toUnmodifiableList()); + return protocolSpecs.stream().toList(); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java index 0c7d17e24f..d0c4847bba 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.math.BigInteger; import java.util.Optional; import java.util.OptionalLong; +import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Stream; @@ -144,11 +145,6 @@ public class ProtocolScheduleBuilder extends AbstractProtocolScheduleBuilder { assert (lastForkBlock >= 0); } - @Override - protected String getBlockIdentifierName() { - return "block"; - } - @Override protected Stream> createMilestones( final MainnetProtocolSpecFactory specFactory) { @@ -184,15 +180,21 @@ public class ProtocolScheduleBuilder extends AbstractProtocolScheduleBuilder { } @Override - protected void postBuildStep(final MainnetProtocolSpecFactory specFactory) { + protected void postBuildStep( + final MainnetProtocolSpecFactory specFactory, final TreeMap builders) { // NOTE: It is assumed that Daofork blocks will not be used for private networks // as too many risks exist around inserting a protocol-spec between daoBlock and daoBlock+10. config .getDaoForkBlock() .ifPresent( daoBlockNumber -> { + final BuilderMapEntry previousSpecBuilder = + builders.floorEntry(daoBlockNumber).getValue(); final ProtocolSpec originalProtocolSpec = - protocolSchedule.getByBlockNumber(daoBlockNumber); + getProtocolSpec( + protocolSchedule, + previousSpecBuilder.getBuilder(), + previousSpecBuilder.getModifier()); addProtocolSpec( protocolSchedule, daoBlockNumber, @@ -212,8 +214,13 @@ public class ProtocolScheduleBuilder extends AbstractProtocolScheduleBuilder { .getClassicForkBlock() .ifPresent( classicBlockNumber -> { + final BuilderMapEntry previousSpecBuilder = + builders.floorEntry(classicBlockNumber).getValue(); final ProtocolSpec originalProtocolSpec = - protocolSchedule.getByBlockNumber(classicBlockNumber); + getProtocolSpec( + protocolSchedule, + previousSpecBuilder.getBuilder(), + previousSpecBuilder.getModifier()); addProtocolSpec( protocolSchedule, classicBlockNumber, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ScheduleBasedBlockHeaderFunctions.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ScheduleBasedBlockHeaderFunctions.java index 651a93dc98..b6c0de70a4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ScheduleBasedBlockHeaderFunctions.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ScheduleBasedBlockHeaderFunctions.java @@ -47,12 +47,6 @@ public class ScheduleBasedBlockHeaderFunctions implements BlockHeaderFunctions { } private BlockHeaderFunctions getBlockHeaderFunctions(final SealableBlockHeader header) { - if (protocolSchedule instanceof ProtocolSchedule) { - return ((ProtocolSchedule) protocolSchedule) - .getByBlockNumber(header.getNumber()) - .getBlockHeaderFunctions(); - } else { - return protocolSchedule.getByBlockHeader(header).getBlockHeaderFunctions(); - } + return protocolSchedule.getByBlockHeader(header).getBlockHeaderFunctions(); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ScheduledProtocolSpec.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ScheduledProtocolSpec.java index c08c139e74..e70df151b8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ScheduledProtocolSpec.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ScheduledProtocolSpec.java @@ -14,21 +14,8 @@ */ package org.hyperledger.besu.ethereum.mainnet; -/** Tuple that associates a {@link ProtocolSpec} with a given block level starting point */ -public class ScheduledProtocolSpec { - private final long block; - private final ProtocolSpec spec; - - public ScheduledProtocolSpec(final long block, final ProtocolSpec spec) { - this.block = block; - this.spec = spec; - } - - public long getBlock() { - return block; - } - - public ProtocolSpec getSpec() { - return spec; - } -} +/** + * Tuple that associates a {@link ProtocolSpec} with a given block number or timestamp level + * starting point + */ +public record ScheduledProtocolSpec(long milestone, ProtocolSpec spec) {} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/TimestampScheduleBuilder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/TimestampScheduleBuilder.java index fb78241b40..894ab894c3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/TimestampScheduleBuilder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/TimestampScheduleBuilder.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.math.BigInteger; import java.util.Optional; +import java.util.TreeMap; import java.util.stream.Stream; public class TimestampScheduleBuilder extends AbstractProtocolScheduleBuilder { @@ -64,18 +65,13 @@ public class TimestampScheduleBuilder extends AbstractProtocolScheduleBuilder { assert (lastForkTimestamp >= 0); } - @Override - protected String getBlockIdentifierName() { - return "timestamp"; - } - @Override protected Stream> createMilestones( final MainnetProtocolSpecFactory specFactory) { return Stream.of( // generally this TimestampSchedule will not have an entry for 0 instead it is relying // on defaulting to a MergeProtocolSchedule in - // TransitionProtocolSchedule.getByBlockHeader if the given timestamp is before the + // TransitionProtocolSchedule.getByBlockHeader if the given milestone is before the // first entry in TimestampSchedule create(config.getShanghaiTime(), specFactory.shanghaiDefinition(config)), create(config.getCancunTime(), specFactory.cancunDefinition(config)), @@ -84,5 +80,7 @@ public class TimestampScheduleBuilder extends AbstractProtocolScheduleBuilder { } @Override - protected void postBuildStep(final MainnetProtocolSpecFactory specFactory) {} + protected void postBuildStep( + final MainnetProtocolSpecFactory specFactory, + final TreeMap builders) {} } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 0d86533091..f93dda4e02 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.storage.keyvalue; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.BlockchainStorage; import org.hyperledger.besu.ethereum.goquorum.GoQuorumPrivateKeyValueStorage; import org.hyperledger.besu.ethereum.goquorum.GoQuorumPrivateStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java index 653a5b56d7..a6addd7ad6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java @@ -165,7 +165,7 @@ public class TransactionSimulator { private MutableWorldState getWorldState(final BlockHeader header) { return worldStateArchive - .getMutable(header.getStateRoot(), header.getHash(), false) + .getMutable(header, false) .orElseThrow( () -> new IllegalArgumentException( @@ -179,7 +179,7 @@ public class TransactionSimulator { final OperationTracer operationTracer, final BlockHeader header, final WorldUpdater updater) { - final ProtocolSpec protocolSpec = protocolSchedule.getByBlockNumber(header.getNumber()); + final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(header); final Address senderAddress = callParams.getFrom() != null ? callParams.getFrom() : DEFAULT_FROM; @@ -205,9 +205,7 @@ public class TransactionSimulator { final Bytes payload = callParams.getPayload() != null ? callParams.getPayload() : Bytes.EMPTY; final MainnetTransactionProcessor transactionProcessor = - protocolSchedule - .getByBlockNumber(blockHeaderToProcess.getNumber()) - .getTransactionProcessor(); + protocolSchedule.getByBlockHeader(blockHeaderToProcess).getTransactionProcessor(); final Optional maybeTransaction = buildTransaction( @@ -347,9 +345,7 @@ public class TransactionSimulator { public Optional doesAddressExistAtHead(final Address address) { final BlockHeader header = blockchain.getChainHeadHeader(); try (final MutableWorldState worldState = - worldStateArchive - .getMutable(header.getStateRoot(), header.getHash(), false) - .orElseThrow()) { + worldStateArchive.getMutable(header, false).orElseThrow()) { return doesAddressExist(worldState, address, header); } catch (final Exception ex) { return Optional.empty(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java index 87e2a82aa6..542605f20b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java @@ -111,11 +111,6 @@ public class DefaultMutableWorldState implements MutableWorldState { return rootHash(); } - @Override - public MutableWorldState copy() { - return new DefaultMutableWorldState(rootHash(), worldStateStorage, preimageStorage); - } - @Override public Account get(final Address address) { final Hash addressHash = Hash.hash(address); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java index cf4173c638..c2174f8107 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java @@ -56,8 +56,8 @@ public class DefaultWorldStateArchive implements WorldStateArchive { @Override public Optional getMutable( - final Hash rootHash, final Hash blockHash, final boolean isPersistingState) { - return getMutable(rootHash, blockHash); + final BlockHeader blockHeader, final boolean isPersistingState) { + return getMutable(blockHeader.getStateRoot(), blockHeader.getHash()); } @Override @@ -95,4 +95,9 @@ public class DefaultWorldStateArchive implements WorldStateArchive { final List accountStorageKeys) { return worldStateProof.getAccountProof(worldStateRoot, accountAddress, accountStorageKeys); } + + @Override + public void close() { + // no op + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java index 677560a4e1..74b5560574 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java @@ -22,20 +22,21 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; import org.hyperledger.besu.evm.worldstate.WorldState; +import java.io.Closeable; import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; -public interface WorldStateArchive { +public interface WorldStateArchive extends Closeable { Hash EMPTY_ROOT_HASH = Hash.wrap(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); Optional get(Hash rootHash, Hash blockHash); boolean isWorldStateAvailable(Hash rootHash, Hash blockHash); - Optional getMutable(Hash rootHash, Hash blockHash, boolean isPersistingState); + Optional getMutable(BlockHeader blockHeader, boolean isPersistingState); Optional getMutable(Hash rootHash, Hash blockHash); diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 35621b71c4..468adabbc8 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -14,8 +14,8 @@ */ package org.hyperledger.besu.ethereum.core; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive; -import org.hyperledger.besu.ethereum.bonsai.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -63,13 +63,13 @@ public class InMemoryKeyValueStorageProvider extends KeyValueStorageProvider { new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage())); } - public static BonsaiWorldStateArchive createBonsaiInMemoryWorldStateArchive( + public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( final Blockchain blockchain) { final InMemoryKeyValueStorageProvider inMemoryKeyValueStorageProvider = new InMemoryKeyValueStorageProvider(); final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); - return new BonsaiWorldStateArchive( + return new BonsaiWorldStateProvider( inMemoryKeyValueStorageProvider, blockchain, cachedMerkleTrieLoader); } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java index 28f47a992f..75c9347b59 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java @@ -16,7 +16,7 @@ package org.hyperledger.besu.ethereum.core; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index dfccd24a0d..6c0a0e23e5 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -23,10 +23,10 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.bonsai.BonsaiPersistedWorldState; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.bonsai.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.chain.BadBlockManager; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; @@ -87,14 +87,14 @@ public class BlockImportExceptionHandlingTest { private CachedMerkleTrieLoader cachedMerkleTrieLoader; private final WorldStateArchive worldStateArchive = - // contains a BonsaiPersistedWorldState which we need to spy on. + // contains a BonsaiWorldState which we need to spy on. // do we need to also test with a DefaultWorldStateArchive? - spy(new BonsaiWorldStateArchive(storageProvider, blockchain, cachedMerkleTrieLoader)); + spy(new BonsaiWorldStateProvider(storageProvider, blockchain, cachedMerkleTrieLoader)); - private final BonsaiPersistedWorldState persisted = + private final BonsaiWorldState persisted = spy( - new BonsaiPersistedWorldState( - (BonsaiWorldStateArchive) worldStateArchive, + new BonsaiWorldState( + (BonsaiWorldStateProvider) worldStateArchive, (BonsaiWorldStateKeyValueStorage) worldStateStorage)); private final BadBlockManager badBlockManager = new BadBlockManager(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/MainnetBlockValidatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/MainnetBlockValidatorTest.java index 636bc73230..dbee1a7cea 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/MainnetBlockValidatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/MainnetBlockValidatorTest.java @@ -105,16 +105,16 @@ public class MainnetBlockValidatorTest { @Test public void shouldDetectAndCacheInvalidBlocksWhenParentWorldStateNotAvailable() { - when(blockchain.getBlockHeader(any(Hash.class))) - .thenReturn(Optional.of(new BlockHeaderTestFixture().buildHeader())); + + final BlockHeader blockHeader = new BlockHeaderTestFixture().buildHeader(); + when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(blockHeader)); when(blockHeaderValidator.validateHeader( any(BlockHeader.class), any(BlockHeader.class), eq(protocolContext), eq(HeaderValidationMode.DETACHED_ONLY))) .thenReturn(true); - when(worldStateArchive.getMutable(any(Hash.class), any(Hash.class), anyBoolean())) - .thenReturn(Optional.empty()); + when(worldStateArchive.getMutable(eq(blockHeader), anyBoolean())).thenReturn(Optional.empty()); assertThat(badBlockManager.getBadBlocks().size()).isEqualTo(0); mainnetBlockValidator.validateAndProcessBlock( @@ -127,15 +127,15 @@ public class MainnetBlockValidatorTest { @Test public void shouldDetectAndCacheInvalidBlocksWhenProcessBlockFailed() { - when(blockchain.getBlockHeader(any(Hash.class))) - .thenReturn(Optional.of(new BlockHeaderTestFixture().buildHeader())); + final BlockHeader blockHeader = new BlockHeaderTestFixture().buildHeader(); + when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(blockHeader)); when(blockHeaderValidator.validateHeader( any(BlockHeader.class), any(BlockHeader.class), eq(protocolContext), eq(HeaderValidationMode.DETACHED_ONLY))) .thenReturn(true); - when(worldStateArchive.getMutable(any(Hash.class), any(Hash.class), anyBoolean())) + when(worldStateArchive.getMutable(eq(blockHeader), anyBoolean())) .thenReturn(Optional.of(mock(MutableWorldState.class))); when(blockProcessor.processBlock(eq(blockchain), any(MutableWorldState.class), eq(badBlock))) .thenReturn(new BlockProcessingResult(Optional.empty())); @@ -150,15 +150,15 @@ public class MainnetBlockValidatorTest { @Test public void shouldDetectAndCacheInvalidBlocksWhenBodyInvalid() { - when(blockchain.getBlockHeader(any(Hash.class))) - .thenReturn(Optional.of(new BlockHeaderTestFixture().buildHeader())); + final BlockHeader blockHeader = new BlockHeaderTestFixture().buildHeader(); + when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(blockHeader)); when(blockHeaderValidator.validateHeader( any(BlockHeader.class), any(BlockHeader.class), eq(protocolContext), eq(HeaderValidationMode.DETACHED_ONLY))) .thenReturn(true); - when(worldStateArchive.getMutable(any(Hash.class), any(Hash.class), anyBoolean())) + when(worldStateArchive.getMutable(eq(blockHeader), anyBoolean())) .thenReturn(Optional.of(mock(MutableWorldState.class))); when(blockProcessor.processBlock(eq(blockchain), any(MutableWorldState.class), eq(badBlock))) .thenReturn(new BlockProcessingResult(Optional.empty())); @@ -173,18 +173,18 @@ public class MainnetBlockValidatorTest { @Test public void shouldNotCacheWhenValidBlocks() { - MutableWorldState mockWorldState = - when(mock(MutableWorldState.class).isPersistable()).thenReturn(true).getMock(); - when(blockchain.getBlockHeader(any(Hash.class))) - .thenReturn(Optional.of(new BlockHeaderTestFixture().buildHeader())); + MutableWorldState mockWorldState = mock(MutableWorldState.class); + + final BlockHeader blockHeader = new BlockHeaderTestFixture().buildHeader(); + when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(blockHeader)); when(blockHeaderValidator.validateHeader( any(BlockHeader.class), any(BlockHeader.class), eq(protocolContext), eq(HeaderValidationMode.DETACHED_ONLY))) .thenReturn(true); - when(worldStateArchive.getMutable(any(Hash.class), any(Hash.class), anyBoolean())) + when(worldStateArchive.getMutable(eq(blockHeader), anyBoolean())) .thenReturn(Optional.of(mockWorldState)); when(worldStateArchive.getMutable(any(Hash.class), any(Hash.class))) .thenReturn(Optional.of(mockWorldState)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java index edd7d9c1a8..7daf5a3f43 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java @@ -28,6 +28,8 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.BlockProcessingResult; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.AbstractBlockCreator; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; @@ -72,7 +74,7 @@ import org.junit.Rule; import org.junit.rules.TemporaryFolder; public abstract class AbstractIsolationTests { - protected BonsaiWorldStateArchive archive; + protected BonsaiWorldStateProvider archive; protected BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorage; protected ProtocolContext protocolContext; final Function asKeyPair = @@ -101,26 +103,19 @@ public abstract class AbstractIsolationTests { @Rule public final TemporaryFolder tempData = new TemporaryFolder(); - protected boolean shouldUseSnapshots() { - // override for layered worldstate - return true; - } - @Before public void createStorage() { bonsaiWorldStateStorage = (BonsaiWorldStateKeyValueStorage) createKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.BONSAI); archive = - new BonsaiWorldStateArchive( + new BonsaiWorldStateProvider( bonsaiWorldStateStorage, blockchain, Optional.of(16L), - shouldUseSnapshots(), new CachedMerkleTrieLoader(new NoOpMetricsSystem())); var ws = archive.getMutable(); genesisState.writeStateTo(ws); - ws.persist(blockchain.getChainHeadHeader()); protocolContext = new ProtocolContext(blockchain, archive, null); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryIsolationTests.java deleted file mode 100644 index a889789549..0000000000 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiInMemoryIsolationTests.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ -package org.hyperledger.besu.ethereum.bonsai; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.Wei; - -import java.util.List; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public class BonsaiInMemoryIsolationTests extends AbstractIsolationTests { - - @Override - protected boolean shouldUseSnapshots() { - // override for layered worldstate - return false; - } - - @Test - public void testInMemoryWorldStateConsistentAfterMutatingPersistedState() { - Address testAddress = Address.fromHexString("0xdeadbeef"); - var persisted = archive.getMutable(); - - var genesisBlock = genesisState.getBlock(); - - var layered = - archive - .getMutable(genesisBlock.getHeader().getStateRoot(), genesisBlock.getHash(), false) - .orElse(null); - var inMemoryBeforeMutation = layered.copy(); - - var firstBlock = forTransactions(List.of(burnTransaction(sender1, 0L, testAddress))); - var res = executeBlock(persisted, firstBlock); - assertThat(res.isSuccessful()).isTrue(); - - assertThat(archive.getTrieLogManager().getBonsaiCachedWorldState(firstBlock.getHash())) - .isNotEmpty(); - - assertThat(res.isSuccessful()).isTrue(); - assertThat(persisted.get(testAddress)).isNotNull(); - assertThat(persisted.get(testAddress).getBalance()) - .isEqualTo(Wei.of(1_000_000_000_000_000_000L)); - assertThat(persisted.rootHash()).isEqualTo(firstBlock.getHeader().getStateRoot()); - - var layered2 = - archive - .getMutable(genesisBlock.getHeader().getStateRoot(), genesisBlock.getHash(), false) - .orElse(null); - - // assert layered before and after the block is what and where we expect it to be: - assertThat(layered2).isInstanceOf(BonsaiLayeredWorldState.class); - assertThat(layered2.rootHash()).isEqualTo(genesisBlock.getHeader().getStateRoot()); - assertThat(layered2.get(testAddress)).isNull(); - assertThat(layered).isInstanceOf(BonsaiLayeredWorldState.class); - assertThat(layered.rootHash()).isEqualTo(genesisBlock.getHeader().getStateRoot()); - assertThat(layered.get(testAddress)).isNull(); - - var inMemoryAfterMutation = layered2.copy(); - - // assert we have not modified the head worldstate: - assertThat(persisted.rootHash()).isEqualTo(firstBlock.getHeader().getStateRoot()); - - // assert the inMemory copy of worldstate 0 from [before mutating persisted state to 1] is - // consistent - assertThat(inMemoryBeforeMutation).isInstanceOf(BonsaiInMemoryWorldState.class); - assertThat(inMemoryBeforeMutation.rootHash()) - .isEqualTo(genesisBlock.getHeader().getStateRoot()); - assertThat(inMemoryBeforeMutation.get(testAddress)).isNull(); - - // assert the inMemory copy of worldstate 0 from [after mutating persisted state to 1] is - // consistent - assertThat(inMemoryAfterMutation).isInstanceOf(BonsaiInMemoryWorldState.class); - assertThat(inMemoryAfterMutation.rootHash()).isEqualTo(genesisBlock.getHeader().getStateRoot()); - assertThat(inMemoryAfterMutation.get(testAddress)).isNull(); - - try { - layered.close(); - layered2.close(); - inMemoryBeforeMutation.close(); - inMemoryAfterMutation.close(); - } catch (Exception ex) { - throw new RuntimeException("failed to close worldstates"); - } - } -} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java index ee2c34e81b..384e667bda 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java @@ -21,10 +21,8 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.Block; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import java.util.List; -import java.util.function.Consumer; import org.junit.Test; import org.junit.runner.RunWith; @@ -36,12 +34,11 @@ public class BonsaiSnapshotIsolationTests extends AbstractIsolationTests { @Test public void ensureTruncateDoesNotCauseSegfault() { - var preTruncatedWorldState = archive.getMutable(null, genesisState.getBlock().getHash(), false); + var preTruncatedWorldState = archive.getMutable(genesisState.getBlock().getHeader(), false); assertThat(preTruncatedWorldState) .isPresent(); // really just assert that we have not segfaulted after truncating bonsaiWorldStateStorage.clear(); - var postTruncatedWorldState = - archive.getMutable(null, genesisState.getBlock().getHash(), false); + var postTruncatedWorldState = archive.getMutable(genesisState.getBlock().getHeader(), false); assertThat(postTruncatedWorldState).isEmpty(); // assert that trying to access pre-worldstate does not segfault after truncating preTruncatedWorldState.get().get(Address.fromHexString(accounts.get(0).getAddress())); @@ -52,22 +49,21 @@ public class BonsaiSnapshotIsolationTests extends AbstractIsolationTests { public void testIsolatedFromHead_behindHead() { Address testAddress = Address.fromHexString("0xdeadbeef"); // assert we can mutate head without mutating the isolated snapshot - var isolated = archive.getMutableSnapshot(genesisState.getBlock().getHash()); + var isolated = archive.getMutable(genesisState.getBlock().getHeader(), false); var firstBlock = forTransactions(List.of(burnTransaction(sender1, 0L, testAddress))); var res = executeBlock(archive.getMutable(), firstBlock); - var isolated2 = archive.getMutableSnapshot(firstBlock.getHash()); + var isolated2 = archive.getMutable(firstBlock.getHeader(), false); var secondBlock = forTransactions(List.of(burnTransaction(sender1, 1L, testAddress))); var res2 = executeBlock(archive.getMutable(), secondBlock); assertThat(res.isSuccessful()).isTrue(); assertThat(res2.isSuccessful()).isTrue(); - assertThat(archive.getTrieLogManager().getBonsaiCachedWorldState(firstBlock.getHash())) - .isNotEmpty(); - assertThat(archive.getTrieLogManager().getBonsaiCachedWorldState(secondBlock.getHash())) - .isNotEmpty(); + assertThat(archive.getTrieLogManager().containWorldStateStorage(firstBlock.getHash())).isTrue(); + assertThat(archive.getTrieLogManager().containWorldStateStorage(secondBlock.getHash())) + .isTrue(); assertThat(archive.getMutable().get(testAddress)).isNotNull(); assertThat(archive.getMutable().get(testAddress).getBalance()) @@ -90,162 +86,6 @@ public class BonsaiSnapshotIsolationTests extends AbstractIsolationTests { } } - @Test - public void testIsolatedSnapshotMutation() { - Address testAddress = Address.fromHexString("0xdeadbeef"); - // assert we can correctly execute a block on a mutable snapshot without mutating head - var isolated = archive.getMutableSnapshot(genesisState.getBlock().getHash()); - - var firstBlock = forTransactions(List.of(burnTransaction(sender1, 0L, testAddress))); - var res = executeBlock(isolated.get(), firstBlock); - - assertThat(archive.getTrieLogManager().getBonsaiCachedWorldState(firstBlock.getHash())) - .isNotEmpty(); - - assertThat(res.isSuccessful()).isTrue(); - assertThat(isolated.get().get(testAddress)).isNotNull(); - assertThat(isolated.get().get(testAddress).getBalance()) - .isEqualTo(Wei.of(1_000_000_000_000_000_000L)); - assertThat(isolated.get().rootHash()).isEqualTo(firstBlock.getHeader().getStateRoot()); - - // persist the isolated worldstate as trielog only: - isolated.get().persist(firstBlock.getHeader()); - - // assert we have not modified the head worldstate: - assertThat(archive.getMutable().get(testAddress)).isNull(); - - // roll the persisted world state to the new trie log from the persisted snapshot - var ws = archive.getMutable(null, firstBlock.getHash()); - assertThat(ws).isPresent(); - assertThat(ws.get().get(testAddress)).isNotNull(); - assertThat(ws.get().get(testAddress).getBalance()) - .isEqualTo(Wei.of(1_000_000_000_000_000_000L)); - assertThat(ws.get().rootHash()).isEqualTo(firstBlock.getHeader().getStateRoot()); - try { - isolated.get().close(); - } catch (Exception ex) { - throw new RuntimeException("failed to close isolated worldstates"); - } - } - - @Test - public void testSnapshotCloneIsolation() { - Address testAddress = Address.fromHexString("0xdeadbeef"); - Address altTestAddress = Address.fromHexString("0xd1ffbeef"); - - // create a snapshot worldstate, and then clone it: - var isolated = archive.getMutableSnapshot(genesisState.getBlock().getHash()).get(); - var isolatedClone = isolated.copy(); - - // execute a block with a single transaction on the first snapshot: - var firstBlock = forTransactions(List.of(burnTransaction(sender1, 0L, testAddress))); - var res = executeBlock(isolated, firstBlock); - - assertThat(res.isSuccessful()).isTrue(); - Runnable checkIsolatedState = - () -> { - assertThat(isolated.rootHash()).isEqualTo(firstBlock.getHeader().getStateRoot()); - assertThat(isolated.get(testAddress)).isNotNull(); - assertThat(isolated.get(altTestAddress)).isNull(); - assertThat(isolated.get(testAddress).getBalance()) - .isEqualTo(Wei.of(1_000_000_000_000_000_000L)); - }; - checkIsolatedState.run(); - - // assert clone is isolated and unmodified: - assertThat(isolatedClone.get(testAddress)).isNull(); - assertThat(isolatedClone.rootHash()) - .isEqualTo(genesisState.getBlock().getHeader().getStateRoot()); - - // assert clone isolated block execution - var cloneForkBlock = - forTransactions( - List.of(burnTransaction(sender1, 0L, altTestAddress)), - genesisState.getBlock().getHeader()); - var altRes = executeBlock(isolatedClone, cloneForkBlock); - - assertThat(altRes.isSuccessful()).isTrue(); - assertThat(isolatedClone.rootHash()).isEqualTo(cloneForkBlock.getHeader().getStateRoot()); - assertThat(isolatedClone.get(altTestAddress)).isNotNull(); - assertThat(isolatedClone.get(testAddress)).isNull(); - assertThat(isolatedClone.get(altTestAddress).getBalance()) - .isEqualTo(Wei.of(1_000_000_000_000_000_000L)); - assertThat(isolatedClone.rootHash()).isEqualTo(cloneForkBlock.getHeader().getStateRoot()); - - // re-check isolated state remains unchanged: - checkIsolatedState.run(); - - // assert that the actual persisted worldstate remains unchanged: - var persistedWorldState = archive.getMutable(); - assertThat(persistedWorldState.rootHash()) - .isEqualTo(genesisState.getBlock().getHeader().getStateRoot()); - assertThat(persistedWorldState.get(testAddress)).isNull(); - assertThat(persistedWorldState.get(altTestAddress)).isNull(); - - // assert that trieloglayers exist for both of the isolated states: - var firstBlockTrieLog = archive.getTrieLogManager().getTrieLogLayer(firstBlock.getHash()); - assertThat(firstBlockTrieLog).isNotEmpty(); - assertThat(firstBlockTrieLog.get().getAccount(testAddress)).isNotEmpty(); - assertThat(firstBlockTrieLog.get().getAccount(altTestAddress)).isEmpty(); - assertThat(archive.getTrieLogManager().getBonsaiCachedWorldState(firstBlock.getHash())) - .isNotEmpty(); - - var cloneForkTrieLog = archive.getTrieLogManager().getTrieLogLayer(cloneForkBlock.getHash()); - assertThat(cloneForkTrieLog.get().getAccount(testAddress)).isEmpty(); - assertThat(cloneForkTrieLog.get().getAccount(altTestAddress)).isNotEmpty(); - - try { - isolated.close(); - isolatedClone.close(); - } catch (Exception ex) { - throw new RuntimeException("failed to close isolated worldstates"); - } - } - - @Test - public void assertSnapshotDoesNotClose() { - Address testAddress = Address.fromHexString("0xdeadbeef"); - - // create a snapshot worldstate, and then clone it: - var isolated = archive.getMutableSnapshot(genesisState.getBlock().getHash()).get(); - - // execute a block with a single transaction on the first snapshot: - var firstBlock = forTransactions(List.of(burnTransaction(sender1, 0L, testAddress))); - var res = executeBlock(isolated, firstBlock); - assertThat(archive.getTrieLogManager().getBonsaiCachedWorldState(firstBlock.getHash())) - .isNotEmpty(); - - assertThat(res.isSuccessful()).isTrue(); - Consumer checkIsolatedState = - (ws) -> { - assertThat(ws.rootHash()).isEqualTo(firstBlock.getHeader().getStateRoot()); - assertThat(ws.get(testAddress)).isNotNull(); - assertThat(ws.get(testAddress).getBalance()) - .isEqualTo(Wei.of(1_000_000_000_000_000_000L)); - }; - checkIsolatedState.accept(isolated); - - var isolatedClone = isolated.copy(); - checkIsolatedState.accept(isolatedClone); - - try { - // close the first snapshot worldstate. The second worldstate should still be able to read - // through its snapshot - isolated.close(); - } catch (Exception ex) { - // meh - } - - // copy of closed isolated worldstate should still pass check - checkIsolatedState.accept(isolatedClone); - - try { - isolatedClone.close(); - } catch (Exception ex) { - throw new RuntimeException("failed to close isolated worldstates"); - } - } - @Test public void testSnapshotRollToTrieLogBlockHash() { // assert we can roll a snapshot to a specific worldstate without mutating head @@ -268,13 +108,13 @@ public class BonsaiSnapshotIsolationTests extends AbstractIsolationTests { blockchain.rewindToBlock(2L); var block1State = archive.getMutable(null, block2.getHash()); - // BonsaiPersistedWorldState should be at block 2 + // BonsaiWorldState should be at block 2 assertThat(block1State.get().get(testAddress)).isNotNull(); assertThat(block1State.get().get(testAddress).getBalance()) .isEqualTo(Wei.of(2_000_000_000_000_000_000L)); assertThat(block1State.get().rootHash()).isEqualTo(block2.getHeader().getStateRoot()); - var isolatedRollForward = archive.getMutableSnapshot(block3.getHash()); + var isolatedRollForward = archive.getMutable(block3.getHeader(), false); // we should be at block 3, one block ahead of BonsaiPersistatedWorldState assertThat(isolatedRollForward.get().get(testAddress)).isNotNull(); @@ -283,7 +123,7 @@ public class BonsaiSnapshotIsolationTests extends AbstractIsolationTests { assertThat(isolatedRollForward.get().rootHash()).isEqualTo(block3.getHeader().getStateRoot()); // we should be at block 1, one block behind BonsaiPersistatedWorldState - var isolatedRollBack = archive.getMutableSnapshot(block1.getHash()); + var isolatedRollBack = archive.getMutable(block1.getHeader(), false); assertThat(isolatedRollBack.get().get(testAddress)).isNotNull(); assertThat(isolatedRollBack.get().get(testAddress).getBalance()) .isEqualTo(Wei.of(1_000_000_000_000_000_000L)); @@ -304,7 +144,7 @@ public class BonsaiSnapshotIsolationTests extends AbstractIsolationTests { var head = archive.getMutable(); try (var shouldCloseSnapshot = - archive.getMutableSnapshot(genesisState.getBlock().getHash()).get()) { + archive.getMutable(genesisState.getBlock().getHeader(), false).get()) { var tx1 = burnTransaction(sender1, 0L, testAddress); Block oneTx = forTransactions(List.of(tx1)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateArchiveTest.java deleted file mode 100644 index a72ba7f4f2..0000000000 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotWorldStateArchiveTest.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ -package org.hyperledger.besu.ethereum.bonsai; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.SnapshotTrieLogManager.CachedSnapshotWorldState; -import org.hyperledger.besu.ethereum.bonsai.TrieLogManager.CachedWorldState; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; -import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; -import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import org.apache.tuweni.bytes.Bytes32; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Answers; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public class BonsaiSnapshotWorldStateArchiveTest { - - final BlockHeaderTestFixture blockBuilder = new BlockHeaderTestFixture(); - - @Mock Blockchain blockchain; - - @Mock StorageProvider storageProvider; - - @Mock SnappableKeyValueStorage keyValueStorage; - - BonsaiWorldStateArchive bonsaiWorldStateArchive; - - CachedMerkleTrieLoader cachedMerkleTrieLoader; - - @Before - public void setUp() { - when(storageProvider.getStorageBySegmentIdentifier(any(KeyValueSegmentIdentifier.class))) - .thenReturn(keyValueStorage); - cachedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); - } - - @Test - public void testGetMutableReturnPersistedStateWhenNeeded() { - final BlockHeader chainHead = blockBuilder.number(0).buildHeader(); - - when(keyValueStorage.get(WORLD_ROOT_HASH_KEY)) - .thenReturn(Optional.of(chainHead.getStateRoot().toArrayUnsafe())); - when(keyValueStorage.get(WORLD_BLOCK_HASH_KEY)) - .thenReturn(Optional.of(chainHead.getHash().toArrayUnsafe())); - when(keyValueStorage.get(WORLD_ROOT_HASH_KEY)) - .thenReturn(Optional.of(chainHead.getStateRoot().toArrayUnsafe())); - when(keyValueStorage.get(WORLD_BLOCK_HASH_KEY)) - .thenReturn(Optional.of(chainHead.getHash().toArrayUnsafe())); - bonsaiWorldStateArchive = - new BonsaiWorldStateArchive( - new BonsaiWorldStateKeyValueStorage(storageProvider), - blockchain, - Optional.of(1L), - true, - cachedMerkleTrieLoader); - - assertThat(bonsaiWorldStateArchive.getMutable(null, chainHead.getHash(), true)) - .containsInstanceOf(BonsaiPersistedWorldState.class); - } - - @Test - public void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { - bonsaiWorldStateArchive = - new BonsaiWorldStateArchive( - new BonsaiWorldStateKeyValueStorage(storageProvider), - blockchain, - Optional.of(512L), - cachedMerkleTrieLoader); - final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); - final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); - when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); - when(blockchain.getChainHeadHeader()).thenReturn(chainHead); - assertThat(bonsaiWorldStateArchive.getMutable(null, blockHeader.getHash(), false)).isEmpty(); - } - - @SuppressWarnings({"unchecked"}) - @Test - public void testGetMutableWithRollbackNotOverrideTrieLogLayer() { - final KeyValueStorageTransaction keyValueStorageTransaction = - mock(KeyValueStorageTransaction.class); - when(keyValueStorage.startTransaction()).thenReturn(keyValueStorageTransaction); - final BlockHeader genesis = blockBuilder.number(0).buildHeader(); - final BlockHeader blockHeaderChainA = - blockBuilder.number(1).timestamp(1).parentHash(genesis.getHash()).buildHeader(); - final BlockHeader blockHeaderChainB = - blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader(); - - final Map> worldStatesByHash = - new HashMap<>(); - var mockCachedState = - new CachedSnapshotWorldState( - mock(BonsaiSnapshotWorldState.class, Answers.RETURNS_MOCKS), - mock(TrieLogLayer.class, Answers.RETURNS_MOCKS), - 2); - worldStatesByHash.put(blockHeaderChainA.getHash(), mockCachedState); - worldStatesByHash.put(blockHeaderChainB.getHash(), mockCachedState); - var worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider); - bonsaiWorldStateArchive = - spy( - new BonsaiWorldStateArchive( - new SnapshotTrieLogManager(blockchain, worldStateStorage, 12L, worldStatesByHash), - worldStateStorage, - blockchain, - true, - cachedMerkleTrieLoader)); - var worldState = (BonsaiPersistedWorldState) bonsaiWorldStateArchive.getMutable(); - var updater = spy(bonsaiWorldStateArchive.getUpdaterFromPersistedState(worldState)); - when(bonsaiWorldStateArchive.getUpdaterFromPersistedState(worldState)).thenReturn(updater); - - // initial persisted state hash key - when(blockchain.getBlockHeader(eq(Hash.ZERO))).thenReturn(Optional.of(blockHeaderChainA)); - // fake trie log layer - final BytesValueRLPOutput rlpLogBlockB = new BytesValueRLPOutput(); - final TrieLogLayer trieLogLayerBlockB = new TrieLogLayer(); - trieLogLayerBlockB.setBlockHash(blockHeaderChainB.getHash()); - trieLogLayerBlockB.writeTo(rlpLogBlockB); - when(keyValueStorage.get(blockHeaderChainB.getHash().toArrayUnsafe())) - .thenReturn(Optional.of(rlpLogBlockB.encoded().toArrayUnsafe())); - - when(blockchain.getBlockHeader(eq(blockHeaderChainB.getHash()))) - .thenReturn(Optional.of(blockHeaderChainB)); - when(blockchain.getBlockHeader(eq(genesis.getHash()))).thenReturn(Optional.of(genesis)); - - assertThat(bonsaiWorldStateArchive.getMutable(null, blockHeaderChainB.getHash())) - .containsInstanceOf(BonsaiPersistedWorldState.class); - - // verify is not persisting if already present - verify(keyValueStorageTransaction, never()) - .put(eq(blockHeaderChainA.getHash().toArrayUnsafe()), any()); - verify(keyValueStorageTransaction, never()) - .put(eq(blockHeaderChainB.getHash().toArrayUnsafe()), any()); - } -} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java index ac70222af6..df61066c14 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java @@ -16,20 +16,22 @@ package org.hyperledger.besu.ethereum.bonsai; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.LayeredTrieLogManager.LayeredWorldStateCache; -import org.hyperledger.besu.ethereum.bonsai.TrieLogManager.CachedWorldState; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; @@ -40,16 +42,14 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; -import java.util.HashMap; -import java.util.Map; import java.util.Optional; -import org.apache.tuweni.bytes.Bytes32; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Answers; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) @@ -63,7 +63,9 @@ public class BonsaiWorldStateArchiveTest { @Mock SnappableKeyValueStorage keyValueStorage; - BonsaiWorldStateArchive bonsaiWorldStateArchive; + BonsaiWorldStateProvider bonsaiWorldStateArchive; + + @Mock TrieLogManager trieLogManager; @Before public void setUp() { @@ -84,56 +86,52 @@ public class BonsaiWorldStateArchiveTest { when(keyValueStorage.get(WORLD_BLOCK_HASH_KEY)) .thenReturn(Optional.of(chainHead.getHash().toArrayUnsafe())); bonsaiWorldStateArchive = - new BonsaiWorldStateArchive( + new BonsaiWorldStateProvider( + trieLogManager, new BonsaiWorldStateKeyValueStorage(storageProvider), blockchain, - Optional.of(1L), - false, new CachedMerkleTrieLoader(new NoOpMetricsSystem())); - assertThat(bonsaiWorldStateArchive.getMutable(null, chainHead.getHash(), true)) - .containsInstanceOf(BonsaiPersistedWorldState.class); + assertThat(bonsaiWorldStateArchive.getMutable(chainHead, true)) + .containsInstanceOf(BonsaiWorldState.class); } @Test public void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { bonsaiWorldStateArchive = - new BonsaiWorldStateArchive( + new BonsaiWorldStateProvider( new BonsaiWorldStateKeyValueStorage(storageProvider), blockchain, Optional.of(512L), - false, new CachedMerkleTrieLoader(new NoOpMetricsSystem())); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); - when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); when(blockchain.getChainHeadHeader()).thenReturn(chainHead); - assertThat(bonsaiWorldStateArchive.getMutable(null, blockHeader.getHash(), false)).isEmpty(); + assertThat(bonsaiWorldStateArchive.getMutable(blockHeader, false)).isEmpty(); + verify(trieLogManager, Mockito.never()).getWorldState(any(Hash.class)); } @Test public void testGetMutableWhenLoadLessThanLimitLayersBack() { + bonsaiWorldStateArchive = - new BonsaiWorldStateArchive( + new BonsaiWorldStateProvider( + trieLogManager, new BonsaiWorldStateKeyValueStorage(storageProvider), blockchain, - Optional.of(512L), - false, new CachedMerkleTrieLoader(new NoOpMetricsSystem())); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(511).buildHeader(); + final BonsaiWorldState mockWorldState = mock(BonsaiWorldState.class); + when(mockWorldState.blockHash()).thenReturn(blockHeader.getHash()); + when(mockWorldState.freeze()).thenReturn(mockWorldState); - final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); - final TrieLogLayer trieLogLayer = new TrieLogLayer(); - trieLogLayer.setBlockHash(blockHeader.getHash()); - trieLogLayer.writeTo(rlpLog); - when(keyValueStorage.get(blockHeader.getHash().toArrayUnsafe())) - .thenReturn(Optional.of(rlpLog.encoded().toArrayUnsafe())); - - when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); + when(trieLogManager.getMaxLayersToLoad()).thenReturn(Long.valueOf(512)); + when(trieLogManager.getWorldState(blockHeader.getHash())) + .thenReturn(Optional.of(mockWorldState)); when(blockchain.getChainHeadHeader()).thenReturn(chainHead); - assertThat(bonsaiWorldStateArchive.getMutable(null, blockHeader.getHash(), false)) - .containsInstanceOf(BonsaiLayeredWorldState.class); + assertThat(bonsaiWorldStateArchive.getMutable(blockHeader, false)) + .containsInstanceOf(BonsaiWorldState.class); } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -141,27 +139,25 @@ public class BonsaiWorldStateArchiveTest { public void testGetMutableWithStorageInconsistencyRollbackTheState() { when(keyValueStorage.startTransaction()).thenReturn(mock(KeyValueStorageTransaction.class)); - final Map layeredWorldStatesByHash = mock(HashMap.class); + when(trieLogManager.getTrieLogLayer(any())).thenReturn(Optional.of(mock(TrieLogLayer.class))); var worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider); bonsaiWorldStateArchive = spy( - new BonsaiWorldStateArchive( - new LayeredTrieLogManager( - blockchain, worldStateStorage, 12L, layeredWorldStatesByHash), + new BonsaiWorldStateProvider( + trieLogManager, worldStateStorage, blockchain, - false, new CachedMerkleTrieLoader(new NoOpMetricsSystem()))); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); assertThat(bonsaiWorldStateArchive.getMutable(null, blockHeader.getHash())) - .containsInstanceOf(BonsaiPersistedWorldState.class); + .containsInstanceOf(BonsaiWorldState.class); // verify is trying to get the trie log layer to rollback - verify(layeredWorldStatesByHash).containsKey(Hash.ZERO); + verify(trieLogManager).getTrieLogLayer(Hash.ZERO); } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -169,34 +165,26 @@ public class BonsaiWorldStateArchiveTest { public void testGetMutableWithStorageConsistencyNotRollbackTheState() { when(keyValueStorage.startTransaction()).thenReturn(mock(KeyValueStorageTransaction.class)); - final Map layeredWorldStatesByHash = mock(HashMap.class); var worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider); bonsaiWorldStateArchive = spy( - new BonsaiWorldStateArchive( - new LayeredTrieLogManager( - blockchain, worldStateStorage, 12L, layeredWorldStatesByHash), + new BonsaiWorldStateProvider( + trieLogManager, worldStateStorage, blockchain, - false, new CachedMerkleTrieLoader(new NoOpMetricsSystem()))); - var worldState = (BonsaiPersistedWorldState) bonsaiWorldStateArchive.getMutable(); - var updater = spy(bonsaiWorldStateArchive.getUpdaterFromPersistedState(worldState)); - when(bonsaiWorldStateArchive.getUpdaterFromPersistedState(worldState)).thenReturn(updater); - final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); when(blockchain.getBlockHeader(eq(blockHeader.getHash()))).thenReturn(Optional.of(blockHeader)); when(blockchain.getBlockHeader(eq(Hash.ZERO))).thenReturn(Optional.of(blockHeader)); assertThat(bonsaiWorldStateArchive.getMutable(null, blockHeader.getHash())) - .containsInstanceOf(BonsaiPersistedWorldState.class); + .containsInstanceOf(BonsaiWorldState.class); // verify is not trying to get the trie log layer to rollback when block is present - verify(updater, times(0)).rollBack(any()); - verify(updater, times(0)).rollForward(any()); + verify(trieLogManager, Mockito.never()).getTrieLogLayer(any()); } @SuppressWarnings({"unchecked"}) @@ -209,30 +197,17 @@ public class BonsaiWorldStateArchiveTest { final BlockHeader blockHeaderChainB = blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader(); - final Map> layeredWorldStatesByHash = - mock(HashMap.class); - when(layeredWorldStatesByHash.containsKey(any(Bytes32.class))).thenReturn(true); - when(layeredWorldStatesByHash.get(eq(blockHeaderChainA.getHash()))) - .thenReturn( - new LayeredWorldStateCache(mock(BonsaiLayeredWorldState.class, Answers.RETURNS_MOCKS))); - when(layeredWorldStatesByHash.get(eq(blockHeaderChainB.getHash()))) - .thenReturn( - new LayeredWorldStateCache(mock(BonsaiLayeredWorldState.class, Answers.RETURNS_MOCKS))); + when(trieLogManager.getTrieLogLayer(any())).thenReturn(Optional.of(mock(TrieLogLayer.class))); var worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider); bonsaiWorldStateArchive = spy( - new BonsaiWorldStateArchive( - new LayeredTrieLogManager( - blockchain, worldStateStorage, 12L, layeredWorldStatesByHash), + new BonsaiWorldStateProvider( + trieLogManager, worldStateStorage, blockchain, - false, new CachedMerkleTrieLoader(new NoOpMetricsSystem()))); - var worldState = (BonsaiPersistedWorldState) bonsaiWorldStateArchive.getMutable(); - var updater = spy(bonsaiWorldStateArchive.getUpdaterFromPersistedState(worldState)); - when(bonsaiWorldStateArchive.getUpdaterFromPersistedState(worldState)).thenReturn(updater); // initial persisted state hash key when(blockchain.getBlockHeader(eq(Hash.ZERO))).thenReturn(Optional.of(blockHeaderChainA)); @@ -241,19 +216,17 @@ public class BonsaiWorldStateArchiveTest { when(blockchain.getBlockHeader(eq(genesis.getHash()))).thenReturn(Optional.of(genesis)); assertThat(bonsaiWorldStateArchive.getMutable(null, blockHeaderChainB.getHash())) - .containsInstanceOf(BonsaiPersistedWorldState.class); + .containsInstanceOf(BonsaiWorldState.class); // verify is trying to get the trie log layers to rollback and roll forward - verify(layeredWorldStatesByHash).containsKey(eq(blockHeaderChainA.getHash())); - verify(layeredWorldStatesByHash).get(eq(blockHeaderChainA.getHash())); - verify(layeredWorldStatesByHash).containsKey(eq(blockHeaderChainB.getHash())); - verify(layeredWorldStatesByHash).get(eq(blockHeaderChainB.getHash())); - verify(updater, times(1)).rollBack(any()); - verify(updater, times(1)).rollForward(any()); + verify(trieLogManager).getTrieLogLayer(eq(blockHeaderChainA.getHash())); + verify(trieLogManager).getTrieLogLayer(eq(blockHeaderChainB.getHash())); } @SuppressWarnings({"unchecked"}) @Test + // TODO: refactor to test original intent + @Ignore("needs refactor, getMutable(hash, hash) cannot trigger saveTrieLog") public void testGetMutableWithRollbackNotOverrideTrieLogLayer() { final KeyValueStorageTransaction keyValueStorageTransaction = mock(KeyValueStorageTransaction.class); @@ -264,28 +237,16 @@ public class BonsaiWorldStateArchiveTest { final BlockHeader blockHeaderChainB = blockBuilder.number(1).timestamp(2).parentHash(genesis.getHash()).buildHeader(); - final Map> layeredWorldStatesByHash = - mock(HashMap.class); - when(layeredWorldStatesByHash.containsKey(any(Bytes32.class))).thenReturn(true); - when(layeredWorldStatesByHash.get(eq(blockHeaderChainA.getHash()))) - .thenReturn( - new LayeredWorldStateCache(mock(BonsaiLayeredWorldState.class, Answers.RETURNS_MOCKS))); - when(layeredWorldStatesByHash.get(eq(blockHeaderChainB.getHash()))) - .thenReturn( - new LayeredWorldStateCache(mock(BonsaiLayeredWorldState.class, Answers.RETURNS_MOCKS))); - var worldStateStorage = new BonsaiWorldStateKeyValueStorage(storageProvider); + when(trieLogManager.getTrieLogLayer(any(Hash.class))) + .thenReturn(Optional.of(mock(TrieLogLayer.class))); + bonsaiWorldStateArchive = spy( - new BonsaiWorldStateArchive( - new LayeredTrieLogManager( - blockchain, worldStateStorage, 12L, layeredWorldStatesByHash), - worldStateStorage, + new BonsaiWorldStateProvider( + trieLogManager, + new BonsaiWorldStateKeyValueStorage(storageProvider), blockchain, - false, new CachedMerkleTrieLoader(new NoOpMetricsSystem()))); - var worldState = (BonsaiPersistedWorldState) bonsaiWorldStateArchive.getMutable(); - var updater = spy(bonsaiWorldStateArchive.getUpdaterFromPersistedState(worldState)); - when(bonsaiWorldStateArchive.getUpdaterFromPersistedState(worldState)).thenReturn(updater); // initial persisted state hash key when(blockchain.getBlockHeader(eq(Hash.ZERO))).thenReturn(Optional.of(blockHeaderChainA)); @@ -302,7 +263,7 @@ public class BonsaiWorldStateArchiveTest { when(blockchain.getBlockHeader(eq(genesis.getHash()))).thenReturn(Optional.of(genesis)); assertThat(bonsaiWorldStateArchive.getMutable(null, blockHeaderChainB.getHash())) - .containsInstanceOf(BonsaiPersistedWorldState.class); + .containsInstanceOf(BonsaiWorldState.class); // verify is not persisting if already present verify(keyValueStorageTransaction, never()) diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java index 77f3e11a15..d905178d03 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.bonsai; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.spy; @@ -24,6 +24,7 @@ import static org.mockito.Mockito.verify; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.rlp.RLP; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java index 3c92e949ae..7a3508b704 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java @@ -19,6 +19,8 @@ import static org.assertj.core.api.Assertions.assertThat; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.rlp.RLP; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LayeredWorldStateTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LayeredWorldStateTests.java deleted file mode 100644 index 2bdbeb5465..0000000000 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LayeredWorldStateTests.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ -package org.hyperledger.besu.ethereum.bonsai; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.ethereum.core.SnapshotMutableWorldState; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.tuweni.bytes.Bytes; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Answers; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public class LayeredWorldStateTests { - - @Mock BonsaiWorldStateArchive archive; - @Mock Blockchain blockchain; - - @Test - public void layeredWorldStateUsesCorrectPersistedWorldStateOnCopy() { - // when copying a layered worldstate we return mutable copy, - // ensure it is for the correct/corresponding worldstate: - - Hash state1Hash = Hash.hash(Bytes.of("first_state".getBytes(StandardCharsets.UTF_8))); - Hash block1Hash = Hash.hash(Bytes.of("first_block".getBytes(StandardCharsets.UTF_8))); - var mockStorage = mock(BonsaiWorldStateKeyValueStorage.class); - when(mockStorage.getWorldStateBlockHash()).thenReturn(Optional.of(block1Hash)); - when(mockStorage.getWorldStateRootHash()).thenReturn(Optional.of(state1Hash)); - SnapshotMutableWorldState mockState = - when(mock(SnapshotMutableWorldState.class).getWorldStateStorage()) - .thenReturn(mockStorage) - .getMock(); - - TrieLogLayer mockLayer = - when(mock(TrieLogLayer.class).getBlockHash()).thenReturn(Hash.ZERO).getMock(); - BonsaiLayeredWorldState mockLayerWs = - new BonsaiLayeredWorldState( - blockchain, - archive, - Optional.of(mock(BonsaiLayeredWorldState.class)), - 1L, - state1Hash, - mockLayer); - - // mimic persisted state being at a different state: - when(archive.getMutableSnapshot(mockLayer.getBlockHash())).thenReturn(Optional.of(mockState)); - - try (var copyOfLayer1 = mockLayerWs.copy()) { - assertThat(copyOfLayer1.rootHash()).isEqualTo(state1Hash); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - @Test - public void saveTrieLogShouldUseCorrectPersistedWorldStateOnCopy() { - // when we save a snapshot based worldstate, ensure - // we used the passed in worldstate and roothash for calculating the trielog diff - Hash testStateRoot = Hash.fromHexStringLenient("0xdeadbeef"); - BlockHeader testHeader = new BlockHeaderTestFixture().stateRoot(testStateRoot).buildHeader(); - - BonsaiWorldStateKeyValueStorage testStorage = - mock(BonsaiWorldStateKeyValueStorage.class, Answers.RETURNS_DEEP_STUBS); - - BonsaiSnapshotWorldState testState = mock(BonsaiSnapshotWorldState.class); - when(testState.getWorldStateStorage()).thenReturn(testStorage); - when(testState.rootHash()).thenReturn(testStateRoot); - when(testState.blockHash()).thenReturn(testHeader.getBlockHash()); - - BonsaiWorldStateUpdater testUpdater = new BonsaiWorldStateUpdater(testState); - // mock kvstorage to mimic head being in a different state than testState - LayeredTrieLogManager manager = - spy( - new LayeredTrieLogManager( - blockchain, mock(BonsaiWorldStateKeyValueStorage.class), 10L, new HashMap<>())); - - // assert we are using the target worldstate storage: - final AtomicBoolean calledPrepareTrieLog = new AtomicBoolean(false); - doAnswer( - prepareCallSpec -> { - Hash blockHash = prepareCallSpec.getArgument(0, BlockHeader.class).getHash(); - Hash rootHash = prepareCallSpec.getArgument(1, Hash.class); - BonsaiPersistedWorldState ws = - prepareCallSpec.getArgument(4, BonsaiPersistedWorldState.class); - assertThat(ws.rootHash()).isEqualTo(rootHash); - assertThat(ws.blockHash()).isEqualTo(blockHash); - calledPrepareTrieLog.set(true); - return mock(TrieLogLayer.class); - }) - .when(manager) - .prepareTrieLog( - any(BlockHeader.class), - any(Hash.class), - any(BonsaiWorldStateUpdater.class), - any(BonsaiWorldStateArchive.class), - any(BonsaiPersistedWorldState.class)); - - manager.saveTrieLog(archive, testUpdater, testStateRoot, testHeader, testState); - assertThat(calledPrepareTrieLog.get()).isTrue(); - } -} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java index 60a1da60c3..5217a367f1 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java @@ -22,6 +22,11 @@ import static org.mockito.Mockito.mock; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Difficulty; @@ -50,14 +55,14 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public class LogRollingTests { - private BonsaiWorldStateArchive archive; + private BonsaiWorldStateProvider archive; private InMemoryKeyValueStorage accountStorage; private InMemoryKeyValueStorage codeStorage; private InMemoryKeyValueStorage storageStorage; private InMemoryKeyValueStorage trieBranchStorage; private InMemoryKeyValueStorage trieLogStorage; - private BonsaiWorldStateArchive secondArchive; + private BonsaiWorldStateProvider secondArchive; private InMemoryKeyValueStorage secondAccountStorage; private InMemoryKeyValueStorage secondCodeStorage; private InMemoryKeyValueStorage secondStorageStorage; @@ -116,7 +121,7 @@ public class LogRollingTests { final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); - archive = new BonsaiWorldStateArchive(provider, blockchain, cachedMerkleTrieLoader); + archive = new BonsaiWorldStateProvider(provider, blockchain, cachedMerkleTrieLoader); accountStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); @@ -138,7 +143,7 @@ public class LogRollingTests { final CachedMerkleTrieLoader secondOptimizedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); secondArchive = - new BonsaiWorldStateArchive(secondProvider, blockchain, secondOptimizedMerkleTrieLoader); + new BonsaiWorldStateProvider(secondProvider, blockchain, secondOptimizedMerkleTrieLoader); secondAccountStorage = (InMemoryKeyValueStorage) secondProvider.getStorageBySegmentIdentifier( @@ -163,8 +168,8 @@ public class LogRollingTests { @Test public void simpleRollForwardTest() { - final BonsaiPersistedWorldState worldState = - new BonsaiPersistedWorldState( + final BonsaiWorldState worldState = + new BonsaiWorldState( archive, new BonsaiWorldStateKeyValueStorage( accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage)); @@ -177,8 +182,8 @@ public class LogRollingTests { updater.commit(); worldState.persist(headerOne); - final BonsaiPersistedWorldState secondWorldState = - new BonsaiPersistedWorldState( + final BonsaiWorldState secondWorldState = + new BonsaiWorldState( secondArchive, new BonsaiWorldStateKeyValueStorage( secondAccountStorage, @@ -186,8 +191,8 @@ public class LogRollingTests { secondStorageStorage, secondTrieBranchStorage, secondTrieLogStorage)); - final BonsaiWorldStateUpdater secondUpdater = - (BonsaiWorldStateUpdater) secondWorldState.updater(); + final BonsaiWorldStateUpdateAccumulator secondUpdater = + (BonsaiWorldStateUpdateAccumulator) secondWorldState.updater(); final Optional value = trieLogStorage.get(headerOne.getHash().toArrayUnsafe()); @@ -212,8 +217,8 @@ public class LogRollingTests { @Test public void rollForwardTwice() { - final BonsaiPersistedWorldState worldState = - new BonsaiPersistedWorldState( + final BonsaiWorldState worldState = + new BonsaiWorldState( archive, new BonsaiWorldStateKeyValueStorage( accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage)); @@ -234,8 +239,8 @@ public class LogRollingTests { worldState.persist(headerTwo); - final BonsaiPersistedWorldState secondWorldState = - new BonsaiPersistedWorldState( + final BonsaiWorldState secondWorldState = + new BonsaiWorldState( secondArchive, new BonsaiWorldStateKeyValueStorage( secondAccountStorage, @@ -243,8 +248,8 @@ public class LogRollingTests { secondStorageStorage, secondTrieBranchStorage, secondTrieLogStorage)); - final BonsaiWorldStateUpdater secondUpdater = - (BonsaiWorldStateUpdater) secondWorldState.updater(); + final BonsaiWorldStateUpdateAccumulator secondUpdater = + (BonsaiWorldStateUpdateAccumulator) secondWorldState.updater(); final TrieLogLayer layerOne = getTrieLogLayer(trieLogStorage, headerOne.getHash()); secondUpdater.rollForward(layerOne); @@ -270,8 +275,8 @@ public class LogRollingTests { @Test public void rollBackOnce() { - final BonsaiPersistedWorldState worldState = - new BonsaiPersistedWorldState( + final BonsaiWorldState worldState = + new BonsaiWorldState( archive, new BonsaiWorldStateKeyValueStorage( accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage)); @@ -291,16 +296,16 @@ public class LogRollingTests { updater2.commit(); worldState.persist(headerTwo); - final BonsaiWorldStateUpdater firstRollbackUpdater = - (BonsaiWorldStateUpdater) worldState.updater(); + final BonsaiWorldStateUpdateAccumulator firstRollbackUpdater = + (BonsaiWorldStateUpdateAccumulator) worldState.updater(); final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, headerTwo.getHash()); firstRollbackUpdater.rollBack(layerTwo); worldState.persist(headerOne); - final BonsaiPersistedWorldState secondWorldState = - new BonsaiPersistedWorldState( + final BonsaiWorldState secondWorldState = + new BonsaiWorldState( secondArchive, new BonsaiWorldStateKeyValueStorage( secondAccountStorage, diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java index 5f37ea292e..f00b88cc12 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java @@ -18,6 +18,11 @@ package org.hyperledger.besu.ethereum.bonsai; import static com.google.common.base.Preconditions.checkArgument; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; @@ -41,8 +46,8 @@ public class RollingImport { final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); - final BonsaiWorldStateArchive archive = - new BonsaiWorldStateArchive(provider, null, cachedMerkleTrieLoader); + final BonsaiWorldStateProvider archive = + new BonsaiWorldStateProvider(provider, null, cachedMerkleTrieLoader); final InMemoryKeyValueStorage accountStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); @@ -59,8 +64,8 @@ public class RollingImport { final InMemoryKeyValueStorage trieLogStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); - final BonsaiPersistedWorldState bonsaiState = - new BonsaiPersistedWorldState( + final BonsaiWorldState bonsaiState = + new BonsaiWorldState( archive, new BonsaiWorldStateKeyValueStorage( accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage)); @@ -74,7 +79,8 @@ public class RollingImport { } final TrieLogLayer layer = TrieLogLayer.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false)); - final BonsaiWorldStateUpdater updater = (BonsaiWorldStateUpdater) bonsaiState.updater(); + final BonsaiWorldStateUpdateAccumulator updater = + (BonsaiWorldStateUpdateAccumulator) bonsaiState.updater(); updater.rollForward(layer); updater.commit(); bonsaiState.persist(null); @@ -102,7 +108,8 @@ public class RollingImport { final byte[] bytes = reader.readBytes(); final TrieLogLayer layer = TrieLogLayer.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false)); - final BonsaiWorldStateUpdater updater = (BonsaiWorldStateUpdater) bonsaiState.updater(); + final BonsaiWorldStateUpdateAccumulator updater = + (BonsaiWorldStateUpdateAccumulator) bonsaiState.updater(); updater.rollBack(layer); updater.commit(); bonsaiState.persist(null); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilderTest.java index 28c01281be..c223bbbe11 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilderTest.java @@ -22,6 +22,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.config.GenesisConfigOptions; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -97,7 +99,7 @@ public class ProtocolScheduleBuilderTest { assertThatThrownBy(() -> builder.createProtocolSchedule()) .isInstanceOf(RuntimeException.class) .hasMessage( - "Genesis Config Error: 'GrayGlacier' is scheduled for block 11 but it must be on or after block 12."); + "Genesis Config Error: 'GrayGlacier' is scheduled for milestone 11 but it must be on or after milestone 12."); } @Test @@ -187,4 +189,22 @@ public class ProtocolScheduleBuilderTest { verify(modifier, times(1)).apply(any()); } + + @Test + public void isOnMilestoneBoundary() { + when(configOptions.getBerlinBlockNumber()).thenReturn(OptionalLong.of(1L)); + when(configOptions.getLondonBlockNumber()).thenReturn(OptionalLong.of(2L)); + when(configOptions.getMergeNetSplitBlockNumber()).thenReturn(OptionalLong.of(4L)); + final HeaderBasedProtocolSchedule protocolSchedule = builder.createProtocolSchedule(); + + assertThat(protocolSchedule.isOnMilestoneBoundary(header(0))).isEqualTo(true); + assertThat(protocolSchedule.isOnMilestoneBoundary(header(1))).isEqualTo(true); + assertThat(protocolSchedule.isOnMilestoneBoundary(header(2))).isEqualTo(true); + assertThat(protocolSchedule.isOnMilestoneBoundary(header(3))).isEqualTo(false); + assertThat(protocolSchedule.isOnMilestoneBoundary(header(4))).isEqualTo(true); + } + + private BlockHeader header(final long blockNumber) { + return new BlockHeaderTestFixture().number(blockNumber).buildHeader(); + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TimestampScheduleBuilderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TimestampScheduleBuilderTest.java index ab220f3ecb..b1489e88ff 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TimestampScheduleBuilderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TimestampScheduleBuilderTest.java @@ -111,7 +111,7 @@ public class TimestampScheduleBuilderTest { assertThatThrownBy(() -> builder.createTimestampSchedule()) .isInstanceOf(RuntimeException.class) .hasMessage( - "Genesis Config Error: 'Cancun' is scheduled for timestamp 2 but it must be on or after timestamp 3."); + "Genesis Config Error: 'Cancun' is scheduled for milestone 2 but it must be on or after milestone 3."); } @Test @@ -149,4 +149,23 @@ public class TimestampScheduleBuilderTest { assertThat(schedule.streamMilestoneBlocks()).containsExactly(FIRST_TIMESTAMP_FORK, 2L, 3L, 5L); } + + @Test + public void isOnMilestoneBoundary() { + config.shanghaiTime(FIRST_TIMESTAMP_FORK); + config.cancunTime(2L); + config.experimentalEipsTime(4L); + final HeaderBasedProtocolSchedule protocolSchedule = builder.createTimestampSchedule(); + + assertThat(protocolSchedule.isOnMilestoneBoundary(header(0))).isEqualTo(false); + assertThat(protocolSchedule.isOnMilestoneBoundary(header(FIRST_TIMESTAMP_FORK))) + .isEqualTo(true); + assertThat(protocolSchedule.isOnMilestoneBoundary(header(2))).isEqualTo(true); + assertThat(protocolSchedule.isOnMilestoneBoundary(header(3))).isEqualTo(false); + assertThat(protocolSchedule.isOnMilestoneBoundary(header(4))).isEqualTo(true); + } + + private BlockHeader header(final long timestamp) { + return new BlockHeaderTestFixture().timestamp(timestamp).buildHeader(); + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalsProcessorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalsProcessorTest.java index d244782461..db82d33e33 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalsProcessorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalsProcessorTest.java @@ -38,7 +38,8 @@ class WithdrawalsProcessorTest { void shouldProcessEmptyWithdrawalsWithoutChangingWorldState() { final MutableWorldState worldState = createWorldStateWithAccounts(List.of(entry("0x1", 1), entry("0x2", 2), entry("0x3", 3))); - final MutableWorldState originalState = worldState.copy(); + final MutableWorldState originalWorldState = + createWorldStateWithAccounts(List.of(entry("0x1", 1), entry("0x2", 2), entry("0x3", 3))); final WorldUpdater updater = worldState.updater(); final WithdrawalsProcessor withdrawalsProcessor = new WithdrawalsProcessor(); @@ -47,7 +48,7 @@ class WithdrawalsProcessorTest { assertThat(worldState.get(Address.fromHexString("0x1")).getBalance()).isEqualTo(Wei.of(1)); assertThat(worldState.get(Address.fromHexString("0x2")).getBalance()).isEqualTo(Wei.of(2)); assertThat(worldState.get(Address.fromHexString("0x3")).getBalance()).isEqualTo(Wei.of(3)); - assertThat(originalState).isEqualTo(worldState); + assertThat(originalWorldState).isEqualTo(worldState); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java index e8b07d5d92..54184b4014 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; @@ -57,7 +58,6 @@ import org.apache.tuweni.bytes.Bytes; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Answers; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; @@ -74,9 +74,6 @@ public class TransactionSimulatorTest { private static final Address DEFAULT_FROM = Address.fromHexString("0x0000000000000000000000000000000000000000"); - private static final Hash DEFAULT_BLOCK_HEADER_HASH = - Hash.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000001"); - private TransactionSimulator transactionSimulator; @Mock private Blockchain blockchain; @@ -87,6 +84,8 @@ public class TransactionSimulatorTest { @Mock private MainnetTransactionProcessor transactionProcessor; @Mock private MainnetTransactionValidator transactionValidator; + private final BlockHeaderTestFixture blockHeaderTestFixture = new BlockHeaderTestFixture(); + @Before public void setUp() { this.transactionSimulator = @@ -110,8 +109,11 @@ public class TransactionSimulatorTest { public void shouldReturnSuccessfulResultWhenProcessingIsSuccessful() { final CallParameter callParameter = legacyTransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L); - mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, callParameter.getFrom(), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -125,7 +127,7 @@ public class TransactionSimulatorTest { .payload(callParameter.getPayload()) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); final Optional result = transactionSimulator.process(callParameter, 1L); @@ -138,8 +140,11 @@ public class TransactionSimulatorTest { public void shouldSetGasPriceToZeroWhenExceedingBalanceAllowed() { final CallParameter callParameter = legacyTransactionCallParameter(Wei.ONE); - mockBlockchainForBlockHeader(Hash.ZERO, 1L); - mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, callParameter.getFrom(), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -154,7 +159,7 @@ public class TransactionSimulatorTest { .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); transactionSimulator.process( callParameter, @@ -169,8 +174,10 @@ public class TransactionSimulatorTest { public void shouldSetFeePerGasToZeroWhenExceedingBalanceAllowed() { final CallParameter callParameter = eip1559TransactionCallParameter(Wei.ONE, Wei.ONE); - mockBlockchainForBlockHeader(Hash.ZERO, 1L, Wei.ONE); - mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + final BlockHeader blockHeader = mockBlockHeader(Hash.ZERO, 1L, Wei.ONE); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, callParameter.getFrom(), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -187,7 +194,7 @@ public class TransactionSimulatorTest { .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); transactionSimulator.process( callParameter, @@ -202,8 +209,11 @@ public class TransactionSimulatorTest { public void shouldNotSetGasPriceToZeroWhenExceedingBalanceIsNotAllowed() { final CallParameter callParameter = legacyTransactionCallParameter(Wei.ONE); - mockBlockchainForBlockHeader(Hash.ZERO, 1L); - mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, callParameter.getFrom(), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -218,7 +228,7 @@ public class TransactionSimulatorTest { .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); transactionSimulator.process( callParameter, @@ -233,8 +243,10 @@ public class TransactionSimulatorTest { public void shouldNotSetFeePerGasToZeroWhenExceedingBalanceIsNotAllowed() { final CallParameter callParameter = eip1559TransactionCallParameter(Wei.ONE, Wei.ONE); - mockBlockchainForBlockHeader(Hash.ZERO, 1L, Wei.ONE); - mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + final BlockHeader blockHeader = mockBlockHeader(Hash.ZERO, 1L, Wei.ONE); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, callParameter.getFrom(), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -251,7 +263,7 @@ public class TransactionSimulatorTest { .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); transactionSimulator.process( callParameter, @@ -266,8 +278,11 @@ public class TransactionSimulatorTest { public void shouldUseDefaultValuesWhenMissingOptionalFields() { final CallParameter callParameter = legacyTransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L); - mockWorldStateForAccount(Hash.ZERO, Address.fromHexString("0x0"), 1L); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, Address.fromHexString("0x0"), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -281,7 +296,7 @@ public class TransactionSimulatorTest { .payload(Bytes.EMPTY) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); transactionSimulator.process(callParameter, 1L); @@ -292,8 +307,11 @@ public class TransactionSimulatorTest { public void shouldUseZeroNonceWhenAccountDoesNotExist() { final CallParameter callParameter = legacyTransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L); - mockWorldStateForAbsentAccount(Hash.ZERO); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAbsentAccount(blockHeader); final Transaction expectedTransaction = Transaction.builder() @@ -307,7 +325,7 @@ public class TransactionSimulatorTest { .payload(Bytes.EMPTY) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); transactionSimulator.process(callParameter, 1L); @@ -318,8 +336,11 @@ public class TransactionSimulatorTest { public void shouldReturnFailureResultWhenProcessingFails() { final CallParameter callParameter = legacyTransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L); - mockWorldStateForAccount(Hash.ZERO, Address.fromHexString("0x0"), 1L); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, Address.fromHexString("0x0"), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -333,7 +354,7 @@ public class TransactionSimulatorTest { .payload(callParameter.getPayload()) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.FAILED); + mockProcessorStatusForTransaction(expectedTransaction, Status.FAILED); final Optional result = transactionSimulator.process(callParameter, 1L); @@ -356,8 +377,11 @@ public class TransactionSimulatorTest { public void shouldReturnSuccessfulResultWhenProcessingIsSuccessfulByHash() { final CallParameter callParameter = legacyTransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L, DEFAULT_BLOCK_HEADER_HASH); - mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, callParameter.getFrom(), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -371,10 +395,10 @@ public class TransactionSimulatorTest { .payload(callParameter.getPayload()) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); final Optional result = - transactionSimulator.process(callParameter, DEFAULT_BLOCK_HEADER_HASH); + transactionSimulator.process(callParameter, blockHeader.getBlockHash()); assertThat(result.get().isSuccessful()).isTrue(); verifyTransactionWasProcessed(expectedTransaction); @@ -384,8 +408,11 @@ public class TransactionSimulatorTest { public void shouldUseDefaultValuesWhenMissingOptionalFieldsByHash() { final CallParameter callParameter = legacyTransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L, DEFAULT_BLOCK_HEADER_HASH); - mockWorldStateForAccount(Hash.ZERO, Address.fromHexString("0x0"), 1L); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, Address.fromHexString("0x0"), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -399,9 +426,9 @@ public class TransactionSimulatorTest { .payload(Bytes.EMPTY) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); - transactionSimulator.process(callParameter, DEFAULT_BLOCK_HEADER_HASH); + transactionSimulator.process(callParameter, blockHeader.getBlockHash()); verifyTransactionWasProcessed(expectedTransaction); } @@ -410,8 +437,11 @@ public class TransactionSimulatorTest { public void shouldUseZeroNonceWhenAccountDoesNotExistByHash() { final CallParameter callParameter = legacyTransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L, DEFAULT_BLOCK_HEADER_HASH); - mockWorldStateForAbsentAccount(Hash.ZERO); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAbsentAccount(blockHeader); final Transaction expectedTransaction = Transaction.builder() @@ -425,9 +455,9 @@ public class TransactionSimulatorTest { .payload(Bytes.EMPTY) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); - transactionSimulator.process(callParameter, DEFAULT_BLOCK_HEADER_HASH); + transactionSimulator.process(callParameter, blockHeader.getBlockHash()); verifyTransactionWasProcessed(expectedTransaction); } @@ -436,8 +466,11 @@ public class TransactionSimulatorTest { public void shouldReturnFailureResultWhenProcessingFailsByHash() { final CallParameter callParameter = legacyTransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L, DEFAULT_BLOCK_HEADER_HASH); - mockWorldStateForAccount(Hash.ZERO, Address.fromHexString("0x0"), 1L); + final BlockHeader blockHeader = + blockHeaderTestFixture.number(1L).stateRoot(Hash.ZERO).buildHeader(); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, Address.fromHexString("0x0"), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -451,10 +484,10 @@ public class TransactionSimulatorTest { .payload(callParameter.getPayload()) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.FAILED); + mockProcessorStatusForTransaction(expectedTransaction, Status.FAILED); final Optional result = - transactionSimulator.process(callParameter, DEFAULT_BLOCK_HEADER_HASH); + transactionSimulator.process(callParameter, blockHeader.getBlockHash()); assertThat(result.get().isSuccessful()).isFalse(); verifyTransactionWasProcessed(expectedTransaction); @@ -464,8 +497,10 @@ public class TransactionSimulatorTest { public void shouldReturnSuccessfulResultWhenEip1559TransactionProcessingIsSuccessful() { final CallParameter callParameter = eip1559TransactionCallParameter(); - mockBlockchainForBlockHeader(Hash.ZERO, 1L, Wei.ONE); - mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + final BlockHeader blockHeader = mockBlockHeader(Hash.ZERO, 1L, Wei.ONE); + + mockBlockchainForBlockHeader(blockHeader); + mockWorldStateForAccount(blockHeader, callParameter.getFrom(), 1L); final Transaction expectedTransaction = Transaction.builder() @@ -481,7 +516,7 @@ public class TransactionSimulatorTest { .payload(callParameter.getPayload()) .signature(FAKE_SIGNATURE) .build(); - mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + mockProcessorStatusForTransaction(expectedTransaction, Status.SUCCESSFUL); final Optional result = transactionSimulator.process(callParameter, 1L); @@ -491,52 +526,45 @@ public class TransactionSimulatorTest { } private void mockWorldStateForAccount( - final Hash stateRoot, final Address address, final long nonce) { + final BlockHeader blockHeader, final Address address, final long nonce) { final Account account = mock(Account.class); when(account.getNonce()).thenReturn(nonce); - when(worldStateArchive.getMutable(eq(stateRoot), any(), anyBoolean())) + when(worldStateArchive.getMutable(eq(blockHeader), anyBoolean())) .thenReturn(Optional.of(worldState)); final WorldUpdater updater = mock(WorldUpdater.class); when(updater.get(address)).thenReturn(account); when(worldState.updater()).thenReturn(updater); } - private void mockWorldStateForAbsentAccount(final Hash stateRoot) { - when(worldStateArchive.getMutable(eq(stateRoot), any(), anyBoolean())) + private void mockWorldStateForAbsentAccount(final BlockHeader blockHeader) { + when(worldStateArchive.getMutable(eq(blockHeader), anyBoolean())) .thenReturn(Optional.of(worldState)); final WorldUpdater updater = mock(WorldUpdater.class); when(updater.get(any())).thenReturn(null); when(worldState.updater()).thenReturn(updater); } - private void mockBlockchainForBlockHeader(final Hash stateRoot, final long blockNumber) { - mockBlockchainForBlockHeader(stateRoot, blockNumber, Hash.ZERO); + private BlockHeader mockBlockHeader( + final Hash stateRoot, final long blockNumber, final Wei baseFee) { + return blockHeaderTestFixture + .stateRoot(stateRoot) + .number(blockNumber) + .baseFeePerGas(baseFee) + .difficulty(Difficulty.ONE) + .buildHeader(); } - private void mockBlockchainForBlockHeader( - final Hash stateRoot, final long blockNumber, final Hash headerHash) { - final BlockHeader blockHeader = mock(BlockHeader.class); - when(blockHeader.getStateRoot()).thenReturn(stateRoot); - when(blockHeader.getNumber()).thenReturn(blockNumber); - when(blockchain.getBlockHeader(blockNumber)).thenReturn(Optional.of(blockHeader)); - when(blockchain.getBlockHeader(headerHash)).thenReturn(Optional.of(blockHeader)); - } - - private void mockBlockchainForBlockHeader( - final Hash stateRoot, final long blockNumber, final Wei baseFee) { - final BlockHeader blockHeader = mock(BlockHeader.class, Answers.RETURNS_MOCKS); - when(blockHeader.getStateRoot()).thenReturn(stateRoot); - when(blockHeader.getNumber()).thenReturn(blockNumber); - when(blockHeader.getBaseFee()).thenReturn(Optional.of(baseFee)); - when(blockHeader.getDifficulty()).thenReturn(Difficulty.ONE); - when(blockchain.getBlockHeader(blockNumber)).thenReturn(Optional.of(blockHeader)); + private void mockBlockchainForBlockHeader(final BlockHeader blockHeader) { + when(blockchain.getBlockHeader(blockHeader.getNumber())).thenReturn(Optional.of(blockHeader)); + when(blockchain.getBlockHeader(blockHeader.getBlockHash())) + .thenReturn(Optional.of(blockHeader)); } private void mockProcessorStatusForTransaction( - final long blockNumber, final Transaction transaction, final Status status) { + final Transaction transaction, final Status status) { final BlockHeaderFunctions blockHeaderFunctions = mock(BlockHeaderFunctions.class); when(protocolSchedule.getChainId()).thenReturn(Optional.of(BigInteger.ONE)); - when(protocolSchedule.getByBlockNumber(eq(blockNumber))).thenReturn(protocolSpec); + when(protocolSchedule.getByBlockHeader(any())).thenReturn(protocolSpec); when(protocolSpec.getTransactionProcessor()).thenReturn(transactionProcessor); when(protocolSpec.getMiningBeneficiaryCalculator()).thenReturn(BlockHeader::getCoinbase); when(protocolSpec.getBlockHeaderFunctions()).thenReturn(blockHeaderFunctions); @@ -552,6 +580,7 @@ public class TransactionSimulatorTest { when(result.isSuccessful()).thenReturn(false); break; } + when(transactionProcessor.processTransaction( any(), any(), diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/AbstractRetryingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/AbstractRetryingTest.java index 98243952a9..bbb7eb4c59 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/AbstractRetryingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/AbstractRetryingTest.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.vm; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import org.junit.Before; import org.junit.Test; @@ -61,7 +61,7 @@ public abstract class AbstractRetryingTest { } catch (final RuntimeException | AssertionError e) { if (!"trace".equalsIgnoreCase(originalRootLogLevel) || !"trace".equalsIgnoreCase(originalEvmLogLevel)) { - // try again, this time with more logging so we can capture more information. + // try again, this time with more logging, so we can capture more information. System.setProperty("root.log.level", "trace"); System.setProperty("evm.log.level", "trace"); resetLogging(); @@ -73,7 +73,7 @@ public abstract class AbstractRetryingTest { } private void resetLogging() { - Log4j2ConfiguratorUtil.reconfigure(); + LogConfigurator.reconfigure(); } /** Subclasses should implement this method to run the actual JUnit test. */ diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java index 2de84eb660..c8cdafc684 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java @@ -42,6 +42,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -58,7 +59,7 @@ public class MarkSweepPrunerTest { private final BlockDataGenerator gen = new BlockDataGenerator(); private final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); - private final Map hashValueStore = spy(new HashMap<>()); + private final Map> hashValueStore = spy(new HashMap<>()); private final InMemoryKeyValueStorage stateStorage = new TestInMemoryStorage(hashValueStore); private final WorldStateStorage worldStateStorage = spy(new WorldStateKeyValueStorage(stateStorage)); @@ -113,7 +114,7 @@ public class MarkSweepPrunerTest { // Check that storage contains only the values we expect assertThat(hashValueStore.size()).isEqualTo(expectedNodes.size()); - assertThat(hashValueStore.values()) + assertThat(hashValueStore.values().stream().map(Optional::get)) .containsExactlyInAnyOrderElementsOf( expectedNodes.stream().map(Bytes::toArrayUnsafe).collect(Collectors.toSet())); } @@ -275,7 +276,7 @@ public class MarkSweepPrunerTest { // Proxy class so that we have access to the constructor that takes our own map private static class TestInMemoryStorage extends InMemoryKeyValueStorage { - public TestInMemoryStorage(final Map hashValueStore) { + public TestInMemoryStorage(final Map> hashValueStore) { super(hashValueStore); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index cb774655c2..ac008a8201 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -19,7 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import org.hyperledger.besu.consensus.merge.ForkchoiceEvent; import org.hyperledger.besu.consensus.merge.UnverifiedForkchoiceListener; import org.hyperledger.besu.ethereum.ProtocolContext; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.core.Synchronizer; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.sync.checkpointsync.CheckpointDownloaderFactory; @@ -43,6 +43,8 @@ import org.hyperledger.besu.plugin.services.BesuEvents.SyncStatusListener; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.util.log.FramedLogMessage; +import java.io.PrintWriter; +import java.io.StringWriter; import java.nio.file.Path; import java.time.Clock; import java.util.ArrayList; @@ -206,6 +208,7 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi manager.start(); } }); + CompletableFuture future; if (fastSyncDownloader.isPresent()) { future = fastSyncDownloader.get().start().thenCompose(this::handleSyncResult); @@ -312,6 +315,16 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi fastSyncDownloader.get().deleteFastSyncState(); } + LOG.atDebug() + .setMessage("heal stacktrace: \n{}") + .addArgument( + () -> { + var sw = new StringWriter(); + new Exception().printStackTrace(new PrintWriter(sw, true)); + return sw.toString(); + }) + .log(); + final List lines = new ArrayList<>(); lines.add("Besu has identified a problem with its worldstate database."); lines.add("Your node will fetch the correct data from peers to repair the problem."); @@ -322,8 +335,8 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi this.syncState.markResyncNeeded(); maybeAccountToRepair.ifPresent( address -> { - if (this.protocolContext.getWorldStateArchive() instanceof BonsaiWorldStateArchive) { - ((BonsaiWorldStateArchive) this.protocolContext.getWorldStateArchive()) + if (this.protocolContext.getWorldStateArchive() instanceof BonsaiWorldStateProvider) { + ((BonsaiWorldStateProvider) this.protocolContext.getWorldStateArchive()) .prepareStateHealing( org.hyperledger.besu.datatypes.Address.wrap(address), location); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java index 7e2588ad65..ee4718b061 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; @@ -35,20 +36,48 @@ import org.slf4j.Logger; public class BackwardChain { private static final Logger LOG = getLogger(BackwardChain.class); + private static final String FIRST_STORED_ANCESTOR_KEY = "firstStoredAncestor"; + private static final String LAST_STORED_PIVOT_KEY = "lastStoredPivot"; + private final GenericKeyValueStorageFacade headers; private final GenericKeyValueStorageFacade blocks; private final GenericKeyValueStorageFacade chainStorage; - private Optional firstStoredAncestor = Optional.empty(); - private Optional lastStoredPivot = Optional.empty(); + private final GenericKeyValueStorageFacade sessionDataStorage; + private Optional firstStoredAncestor; + private Optional lastStoredPivot; private final Queue hashesToAppend = new ArrayDeque<>(); public BackwardChain( final GenericKeyValueStorageFacade headersStorage, final GenericKeyValueStorageFacade blocksStorage, - final GenericKeyValueStorageFacade chainStorage) { + final GenericKeyValueStorageFacade chainStorage, + final GenericKeyValueStorageFacade sessionDataStorage) { this.headers = headersStorage; this.blocks = blocksStorage; this.chainStorage = chainStorage; + this.sessionDataStorage = sessionDataStorage; + firstStoredAncestor = + sessionDataStorage + .get(FIRST_STORED_ANCESTOR_KEY) + .map( + header -> { + LOG.atDebug() + .setMessage(FIRST_STORED_ANCESTOR_KEY + " loaded from storage with value {}") + .addArgument(header::toLogString) + .log(); + return header; + }); + lastStoredPivot = + sessionDataStorage + .get(LAST_STORED_PIVOT_KEY) + .map( + header -> { + LOG.atDebug() + .setMessage(LAST_STORED_PIVOT_KEY + " loaded from storage with value {}") + .addArgument(header::toLogString) + .log(); + return header; + }); } public static BackwardChain from( @@ -67,6 +96,14 @@ public class BackwardChain { new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), + storageProvider.getStorageBySegmentIdentifier( + KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN)), + // using BACKWARD_SYNC_CHAIN that contains the sequence of the work to do, + // to also store the session data that will be used to resume + // the backward sync from where it was left before the restart + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + BlocksHeadersConvertor.of(blockHeaderFunctions), storageProvider.getStorageBySegmentIdentifier( KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN))); } @@ -86,22 +123,43 @@ public class BackwardChain { } public synchronized void prependAncestorsHeader(final BlockHeader blockHeader) { - if (firstStoredAncestor.isEmpty()) { - firstStoredAncestor = Optional.of(blockHeader); - lastStoredPivot = Optional.of(blockHeader); + prependAncestorsHeader(blockHeader, false); + } + + public synchronized void prependAncestorsHeader( + final BlockHeader blockHeader, final boolean alreadyStored) { + if (!alreadyStored) { headers.put(blockHeader.getHash(), blockHeader); - return; } - final BlockHeader firstHeader = firstStoredAncestor.get(); - headers.put(blockHeader.getHash(), blockHeader); - chainStorage.put(blockHeader.getHash(), firstHeader.getHash()); - firstStoredAncestor = Optional.of(blockHeader); - LOG.atDebug() - .setMessage("Added header {} to backward chain led by pivot {} on height {}") - .addArgument(blockHeader::toLogString) - .addArgument(() -> lastStoredPivot.orElseThrow().toLogString()) - .addArgument(firstHeader::getNumber) - .log(); + + if (firstStoredAncestor.isEmpty()) { + updateLastStoredPivot(Optional.of(blockHeader)); + } else { + final BlockHeader firstHeader = firstStoredAncestor.get(); + chainStorage.put(blockHeader.getHash(), firstHeader.getHash()); + LOG.atDebug() + .setMessage("Added header {} to backward chain led by pivot {} on height {}") + .addArgument(blockHeader::toLogString) + .addArgument(() -> lastStoredPivot.orElseThrow().toLogString()) + .addArgument(firstHeader::getNumber) + .log(); + } + + updateFirstStoredAncestor(Optional.of(blockHeader)); + } + + private void updateFirstStoredAncestor(final Optional maybeHeader) { + maybeHeader.ifPresentOrElse( + header -> sessionDataStorage.put(FIRST_STORED_ANCESTOR_KEY, header), + () -> sessionDataStorage.drop(FIRST_STORED_ANCESTOR_KEY)); + firstStoredAncestor = maybeHeader; + } + + private void updateLastStoredPivot(final Optional maybeHeader) { + maybeHeader.ifPresentOrElse( + header -> sessionDataStorage.put(LAST_STORED_PIVOT_KEY, header), + () -> sessionDataStorage.drop(LAST_STORED_PIVOT_KEY)); + lastStoredPivot = maybeHeader; } public synchronized Optional getPivot() { @@ -118,9 +176,9 @@ public class BackwardChain { headers.drop(firstStoredAncestor.get().getHash()); final Optional hash = chainStorage.get(firstStoredAncestor.get().getHash()); chainStorage.drop(firstStoredAncestor.get().getHash()); - firstStoredAncestor = hash.flatMap(headers::get); + updateFirstStoredAncestor(hash.flatMap(headers::get)); if (firstStoredAncestor.isEmpty()) { - lastStoredPivot = Optional.empty(); + updateLastStoredPivot(Optional.empty()); } } @@ -129,7 +187,7 @@ public class BackwardChain { headers.put(newPivot.getHash(), newPivot.getHeader()); blocks.put(newPivot.getHash(), newPivot); if (lastStoredPivot.isEmpty()) { - firstStoredAncestor = Optional.of(newPivot.getHeader()); + updateFirstStoredAncestor(Optional.of(newPivot.getHeader())); } else { if (newPivot.getHeader().getParentHash().equals(lastStoredPivot.get().getHash())) { LOG.atDebug() @@ -140,14 +198,14 @@ public class BackwardChain { .log(); chainStorage.put(lastStoredPivot.get().getHash(), newPivot.getHash()); } else { - firstStoredAncestor = Optional.of(newPivot.getHeader()); + updateFirstStoredAncestor(Optional.of(newPivot.getHeader())); LOG.atDebug() .setMessage("Re-pivoting to new target block {}") .addArgument(newPivot::toLogString) .log(); } } - lastStoredPivot = Optional.of(newPivot.getHeader()); + updateLastStoredPivot(Optional.of(newPivot.getHeader())); } public synchronized boolean isTrusted(final Hash hash) { @@ -162,6 +220,7 @@ public class BackwardChain { blocks.clear(); headers.clear(); chainStorage.clear(); + sessionDataStorage.clear(); firstStoredAncestor = Optional.empty(); lastStoredPivot = Optional.empty(); hashesToAppend.clear(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardsSyncAlgorithm.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java similarity index 94% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardsSyncAlgorithm.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java index 3b06eb4424..731291dc1e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardsSyncAlgorithm.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java @@ -35,8 +35,8 @@ import java.util.concurrent.atomic.AtomicReference; import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; -public class BackwardsSyncAlgorithm implements BesuEvents.InitialSyncCompletionListener { - private static final Logger LOG = getLogger(BackwardsSyncAlgorithm.class); +public class BackwardSyncAlgorithm implements BesuEvents.InitialSyncCompletionListener { + private static final Logger LOG = getLogger(BackwardSyncAlgorithm.class); private final BackwardSyncContext context; private final FinalBlockConfirmation finalBlockConfirmation; @@ -44,7 +44,7 @@ public class BackwardsSyncAlgorithm implements BesuEvents.InitialSyncCompletionL new AtomicReference<>(new CountDownLatch(1)); private volatile boolean finished = false; - public BackwardsSyncAlgorithm( + public BackwardSyncAlgorithm( final BackwardSyncContext context, final FinalBlockConfirmation finalBlockConfirmation) { this.context = context; this.finalBlockConfirmation = finalBlockConfirmation; @@ -64,7 +64,10 @@ public class BackwardsSyncAlgorithm implements BesuEvents.InitialSyncCompletionL return executeSyncStep(firstHash.get()) .thenAccept( result -> { - LOG.info("Backward sync target block is {}", result.toLogString()); + LOG.atDebug() + .setMessage("Backward sync target block is {}") + .addArgument(result::toLogString) + .log(); context.getBackwardChain().removeFromHashToAppend(firstHash.get()); context.getStatus().updateTargetHeight(result.getHeader().getNumber()); }); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index 6574d48e67..bc618e7c13 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -153,7 +153,7 @@ public class BackwardSyncContext { if (backwardChain.isTrusted(hash)) { LOG.atDebug() .setMessage( - "not fetching or appending hash {} to backwards sync since it is present in successors") + "Not fetching or appending hash {} to backward sync since it is present in successors") .addArgument(hash::toHexString) .log(); return true; @@ -237,7 +237,7 @@ public class BackwardSyncContext { @VisibleForTesting CompletableFuture prepareBackwardSyncFuture() { final MutableBlockchain blockchain = getProtocolContext().getBlockchain(); - return new BackwardsSyncAlgorithm( + return new BackwardSyncAlgorithm( this, FinalBlockConfirmation.confirmationChain( FinalBlockConfirmation.genesisConfirmation(blockchain), diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java index 16478366f2..10315a6010 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java @@ -48,7 +48,7 @@ public class BackwardSyncStep { Hash lastHash = firstAncestor.getParentHash(); Optional iterator = backwardChain.getHeader(lastHash); while (iterator.isPresent()) { - backwardChain.prependAncestorsHeader(iterator.get()); + backwardChain.prependAncestorsHeader(iterator.get(), true); lastHash = iterator.get().getParentHash(); iterator = backwardChain.getHeader(lastHash); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSource.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSource.java index 9b5fb5d3ba..2f3fa2f430 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSource.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSource.java @@ -16,9 +16,7 @@ package org.hyperledger.besu.ethereum.eth.sync.checkpointsync; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; -import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import java.util.Iterator; @@ -35,12 +33,10 @@ public class CheckpointSource implements Iterator { private final AtomicBoolean isDownloading = new AtomicBoolean(false); public CheckpointSource( - final SyncState syncState, - final EthPeer ethPeer, - final BlockHeaderFunctions blockHeaderFunctions) { + final SyncState syncState, final BlockHeader blockHeader, final int nbBlocks) { this.syncState = syncState; - this.checkpoint = ethPeer.getCheckpointHeader().orElseThrow(); - this.nbBlocks = blockHeaderFunctions.getCheckPointWindowSize(checkpoint); + this.checkpoint = blockHeader; + this.nbBlocks = nbBlocks; this.lastHeaderDownloaded = Optional.empty(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java index 3dcbb96701..8c853717f5 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncDownloadPipelineFactory.java @@ -61,11 +61,15 @@ public class CheckpointSyncDownloadPipelineFactory extends FastSyncDownloadPipel final Checkpoint checkpoint = syncState.getCheckpoint().orElseThrow(); + final BlockHeader checkpointBlockHeader = target.peer().getCheckpointHeader().orElseThrow(); final CheckpointSource checkPointSource = new CheckpointSource( syncState, - target.peer(), - protocolSchedule.getByBlockNumber(checkpoint.blockNumber()).getBlockHeaderFunctions()); + checkpointBlockHeader, + protocolSchedule + .getByBlockHeader(checkpointBlockHeader) + .getBlockHeaderFunctions() + .getCheckPointWindowSize(checkpointBlockHeader)); final CheckpointBlockImportStep checkPointBlockImportStep = new CheckpointBlockImportStep( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index 6596e9a927..07215f80b9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -16,7 +16,7 @@ package org.hyperledger.besu.ethereum.eth.sync.fastsync; import static org.hyperledger.besu.util.FutureUtils.exceptionallyCompose; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.manager.exceptions.MaxRetriesReachedException; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/AccountTrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/AccountTrieNodeDataRequest.java index 5903b5e1a6..edbdb4d1cc 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/AccountTrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/AccountTrieNodeDataRequest.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.ethereum.trie.CompactEncoding; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java index 137488ebdb..467dc0a53d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; import org.hyperledger.besu.ethereum.ProtocolContext; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/StorageTrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/StorageTrieNodeDataRequest.java index 5aebaa987a..abbbce64c1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/StorageTrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/StorageTrieNodeDataRequest.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountTrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountTrieNodeDataRequest.java index dff436697e..3ca8a45068 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountTrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountTrieNodeDataRequest.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; import org.hyperledger.besu.ethereum.rlp.RLP; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageTrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageTrieNodeDataRequest.java index 8d3eedde05..6bdf1577a6 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageTrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageTrieNodeDataRequest.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; import org.hyperledger.besu.ethereum.trie.CompactEncoding; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksTask.java index 43cfeadf89..413cd1da46 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksTask.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksTask.java @@ -32,12 +32,14 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,12 +76,38 @@ public class CompleteBlocksTask extends AbstractRetryingPeerTask> { this.blocks = headers.stream() .filter(this::hasEmptyBody) - .collect(toMap(BlockHeader::getNumber, header -> new Block(header, BlockBody.empty()))); + .collect( + toMap( + BlockHeader::getNumber, + header -> + new Block( + header, + createEmptyBodyBasedOnProtocolSchedule(protocolSchedule, header)))); + } + + @NotNull + private BlockBody createEmptyBodyBasedOnProtocolSchedule( + final ProtocolSchedule protocolSchedule, final BlockHeader header) { + return new BlockBody( + Collections.emptyList(), + Collections.emptyList(), + isWithdrawalsEnabled(protocolSchedule, header) + ? Optional.of(Collections.emptyList()) + : Optional.empty()); + } + + private boolean isWithdrawalsEnabled( + final ProtocolSchedule protocolSchedule, final BlockHeader header) { + return protocolSchedule.getByBlockHeader(header).getWithdrawalsProcessor().isPresent(); } private boolean hasEmptyBody(final BlockHeader header) { return header.getOmmersHash().equals(Hash.EMPTY_LIST_HASH) - && header.getTransactionsRoot().equals(Hash.EMPTY_TRIE_HASH); + && header.getTransactionsRoot().equals(Hash.EMPTY_TRIE_HASH) + && header + .getWithdrawalsRoot() + .map(wsRoot -> wsRoot.equals(Hash.EMPTY_TRIE_HASH)) + .orElse(true); } public static CompleteBlocksTask forHeaders( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java index b3f0dda81d..b4251e058e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPool.java @@ -33,8 +33,8 @@ import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.mainnet.HeaderBasedProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionValidator; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; @@ -72,7 +72,7 @@ public class TransactionPool implements BlockAddedObserver { private static final String REMOTE = "remote"; private static final String LOCAL = "local"; private final PendingTransactions pendingTransactions; - private final ProtocolSchedule protocolSchedule; + private final HeaderBasedProtocolSchedule protocolSchedule; private final ProtocolContext protocolContext; private final TransactionBroadcaster transactionBroadcaster; private final MiningParameters miningParameters; @@ -82,7 +82,7 @@ public class TransactionPool implements BlockAddedObserver { public TransactionPool( final PendingTransactions pendingTransactions, - final ProtocolSchedule protocolSchedule, + final HeaderBasedProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final TransactionBroadcaster transactionBroadcaster, final EthContext ethContext, @@ -334,15 +334,7 @@ public class TransactionPool implements BlockAddedObserver { try (final var worldState = protocolContext .getWorldStateArchive() - .getMutable( - chainHeadBlockHeader.getStateRoot(), chainHeadBlockHeader.getBlockHash(), false) - .map( - ws -> { - if (!ws.isPersistable()) { - return ws.copy(); - } - return ws; - }) + .getMutable(chainHeadBlockHeader, false) .orElseThrow()) { final Account senderAccount = worldState.get(transaction.getSender()); return new ValidationResultAndAccount( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java index c5f6e0747c..388f0aed45 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactory.java @@ -22,9 +22,7 @@ import org.hyperledger.besu.ethereum.eth.messages.EthPV65; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter; import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; -import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.mainnet.HeaderBasedProtocolSchedule; import org.hyperledger.besu.plugin.services.BesuEvents; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -37,7 +35,7 @@ public class TransactionPoolFactory { private static final Logger LOG = LoggerFactory.getLogger(TransactionPoolFactory.class); public static TransactionPool createTransactionPool( - final ProtocolSchedule protocolSchedule, + final HeaderBasedProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, final Clock clock, @@ -72,7 +70,7 @@ public class TransactionPoolFactory { } static TransactionPool createTransactionPool( - final ProtocolSchedule protocolSchedule, + final HeaderBasedProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, final MetricsSystem metricsSystem, @@ -172,17 +170,14 @@ public class TransactionPoolFactory { } private static PendingTransactions createPendingTransactions( - final ProtocolSchedule protocolSchedule, + final HeaderBasedProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final Clock clock, final MetricsSystem metricsSystem, final TransactionPoolConfiguration transactionPoolConfiguration) { boolean isFeeMarketImplementBaseFee = - protocolSchedule - .streamMilestoneBlocks() - .map(protocolSchedule::getByBlockNumber) - .map(ProtocolSpec::getFeeMarket) - .anyMatch(FeeMarket::implementsBaseFee); + protocolSchedule.anyMatch( + scheduledSpec -> scheduledSpec.spec().getFeeMarket().implementsBaseFee()); if (isFeeMarketImplementBaseFee) { return new BaseFeePendingTransactionsSorter( transactionPoolConfiguration, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java index 803871d69d..6fc31a3396 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManagerTest.java @@ -1110,14 +1110,15 @@ public final class EthProtocolManagerTest { // Create a transaction pool. This has a side effect of registering a listener for the // transactions message. TransactionPoolFactory.createTransactionPool( - protocolSchedule, - protocolContext, - ethManager.ethContext(), - TestClock.system(ZoneId.systemDefault()), - metricsSystem, - new SyncState(blockchain, ethManager.ethContext().getEthPeers()), - new MiningParameters.Builder().minTransactionGasPrice(Wei.ZERO).build(), - TransactionPoolConfiguration.DEFAULT); + protocolSchedule, + protocolContext, + ethManager.ethContext(), + TestClock.system(ZoneId.systemDefault()), + metricsSystem, + new SyncState(blockchain, ethManager.ethContext().getEthPeers()), + new MiningParameters.Builder().minTransactionGasPrice(Wei.ZERO).build(), + TransactionPoolConfiguration.DEFAULT) + .setEnabled(); // Send just a transaction message. final PeerConnection peer = setupPeer(ethManager, (cap, msg, connection) -> {}); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java index 69ab5470f3..f53b32cd54 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java @@ -66,7 +66,7 @@ public class BackwardSyncAlgSpec { @Captor ArgumentCaptor ttdCaptor; @Captor ArgumentCaptor completionCaptor; - @InjectMocks BackwardsSyncAlgorithm algorithm; + @InjectMocks BackwardSyncAlgorithm algorithm; @Mock private Hash hash; private static final BlockDataGenerator blockDataGenerator = new BlockDataGenerator(); @@ -95,7 +95,7 @@ public class BackwardSyncAlgSpec { algorithm = Mockito.spy( - new BackwardsSyncAlgorithm( + new BackwardSyncAlgorithm( context, FinalBlockConfirmation.confirmationChain( FinalBlockConfirmation.genesisConfirmation(localBlockchain), @@ -292,7 +292,7 @@ public class BackwardSyncAlgSpec { doReturn(backwardChain).when(context).getBackwardChain(); algorithm = Mockito.spy( - new BackwardsSyncAlgorithm( + new BackwardSyncAlgorithm( context, FinalBlockConfirmation.genesisConfirmation(otherLocalBlockchain))); assertThatThrownBy(() -> algorithm.pickNextStep()) .isInstanceOf(BackwardSyncException.class) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index a85df3cdb2..ea3d872cdb 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -52,6 +52,7 @@ import org.hyperledger.besu.plugin.data.TransactionType; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -193,7 +194,12 @@ public class BackwardSyncContextTest { final GenericKeyValueStorageFacade chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); - return new BackwardChain(headersStorage, blocksStorage, chainStorage); + final GenericKeyValueStorageFacade sessionDataStorage = + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + BlocksHeadersConvertor.of(new MainnetBlockHeaderFunctions()), + new InMemoryKeyValueStorage()); + return new BackwardChain(headersStorage, blocksStorage, chainStorage, sessionDataStorage); } @Test diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java index d9df79a402..9ddcaf7b8c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; @@ -73,6 +74,7 @@ public class BackwardSyncStepTest { GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; GenericKeyValueStorageFacade chainStorage; + GenericKeyValueStorageFacade sessionDataStorage; @Before public void setup() { @@ -86,10 +88,14 @@ public class BackwardSyncStepTest { Hash::toArrayUnsafe, new BlocksConvertor(new MainnetBlockHeaderFunctions()), new InMemoryKeyValueStorage()); - chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); + sessionDataStorage = + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), + new InMemoryKeyValueStorage()); Block genesisBlock = blockDataGenerator.genesisBlock(); remoteBlockchain = createInMemoryBlockchain(genesisBlock); @@ -234,7 +240,7 @@ public class BackwardSyncStepTest { @Nonnull private BackwardChain createBackwardChain(final int number) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, sessionDataStorage); backwardChain.appendTrustedBlock(remoteBlockchain.getBlockByNumber(number).orElseThrow()); return backwardChain; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java index 2b6831d71e..cf03c676ad 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java @@ -41,6 +41,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -73,8 +74,8 @@ public class ForwardSyncStepTest { private MutableBlockchain localBlockchain; GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; - GenericKeyValueStorageFacade chainStorage; + GenericKeyValueStorageFacade sessionDataStorage; @Before public void setup() { @@ -91,6 +92,11 @@ public class ForwardSyncStepTest { chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); + sessionDataStorage = + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), + new InMemoryKeyValueStorage()); Block genesisBlock = blockDataGenerator.genesisBlock(); remoteBlockchain = createInMemoryBlockchain(genesisBlock); @@ -197,7 +203,7 @@ public class ForwardSyncStepTest { @Nonnull private BackwardChain backwardChainFromBlock(final int number) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, sessionDataStorage); backwardChain.appendTrustedBlock(remoteBlockchain.getBlockByNumber(number).orElseThrow()); return backwardChain; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java index 3fabb7bad8..405e06d1d3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; @@ -45,6 +46,7 @@ public class InMemoryBackwardChainTest { GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; GenericKeyValueStorageFacade chainStorage; + GenericKeyValueStorageFacade sessionDataStorage; @Before public void prepareData() { @@ -61,6 +63,11 @@ public class InMemoryBackwardChainTest { chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); + sessionDataStorage = + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), + new InMemoryKeyValueStorage()); blocks = prepareChain(ELEMENTS, HEIGHT); } @@ -78,7 +85,7 @@ public class InMemoryBackwardChainTest { @Nonnull private BackwardChain createChainFromBlock(final Block pivot) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, sessionDataStorage); backwardChain.appendTrustedBlock(pivot); return backwardChain; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSourceTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSourceTest.java index ae9ac3d3b1..a7d8638821 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSourceTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSourceTest.java @@ -15,14 +15,11 @@ package org.hyperledger.besu.ethereum.eth.sync.checkpointsync; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import java.util.Optional; @@ -33,15 +30,11 @@ import org.junit.Test; public class CheckPointSourceTest { private final SyncState syncState = mock(SyncState.class); - private final EthPeer peer = mock(EthPeer.class); - private final BlockHeaderFunctions blockHeaderFunctions = mock(BlockHeaderFunctions.class); private CheckpointSource checkPointSource; @Before public void setup() { - when(peer.getCheckpointHeader()).thenReturn(Optional.of(header(12))); - when(blockHeaderFunctions.getCheckPointWindowSize(any(BlockHeader.class))).thenReturn(1); - checkPointSource = new CheckpointSource(syncState, peer, blockHeaderFunctions); + checkPointSource = new CheckpointSource(syncState, header(12), 1); } @Test diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 189e4a5d58..5cc61cb2b9 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -20,7 +20,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 3c297a520e..398536863e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -25,7 +25,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.BlockAddedEvent; import org.hyperledger.besu.ethereum.chain.BlockAddedObserver; import org.hyperledger.besu.ethereum.chain.Blockchain; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksTaskTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksTaskTest.java index 5193dd9986..097a8e814d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksTaskTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/tasks/CompleteBlocksTaskTest.java @@ -17,26 +17,42 @@ package org.hyperledger.besu.ethereum.eth.sync.tasks; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.GWei; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.Withdrawal; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; import org.hyperledger.besu.ethereum.eth.manager.ethtaskutils.RetryingMessageTaskTest; import org.hyperledger.besu.ethereum.eth.manager.exceptions.MaxRetriesReachedException; import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; import org.hyperledger.besu.ethereum.eth.messages.GetBlockBodiesMessage; +import org.hyperledger.besu.ethereum.mainnet.BodyValidation; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.WithdrawalsProcessor; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import org.apache.tuweni.units.bigints.UInt64; import org.junit.Test; +import org.mockito.Mockito; public class CompleteBlocksTaskTest extends RetryingMessageTaskTest> { @@ -79,6 +95,114 @@ public class CompleteBlocksTaskTest extends RetryingMessageTaskTest> assertThat(task.run()).isCompletedWithValue(blocks); } + @Test + public void shouldCreateWithdrawalsAwareEmptyBlock_whenWithdrawalsAreEnabled() { + final ProtocolSchedule mockProtocolSchedule = Mockito.mock(ProtocolSchedule.class); + final ProtocolSpec mockParisSpec = Mockito.mock(ProtocolSpec.class); + final ProtocolSpec mockShanghaiSpec = Mockito.mock(ProtocolSpec.class); + final WithdrawalsProcessor mockWithdrawalsProcessor = Mockito.mock(WithdrawalsProcessor.class); + + final BlockHeader header1 = + new BlockHeaderTestFixture().number(1).withdrawalsRoot(null).buildHeader(); + final BlockHeader header2 = + new BlockHeaderTestFixture().number(2).withdrawalsRoot(Hash.EMPTY_TRIE_HASH).buildHeader(); + + when(mockProtocolSchedule.getByBlockHeader((eq(header1)))).thenReturn(mockParisSpec); + when(mockParisSpec.getWithdrawalsProcessor()).thenReturn(Optional.empty()); + when(mockProtocolSchedule.getByBlockHeader((eq(header2)))).thenReturn(mockShanghaiSpec); + when(mockShanghaiSpec.getWithdrawalsProcessor()) + .thenReturn(Optional.of(mockWithdrawalsProcessor)); + + final Block block1 = + new Block( + header1, + new BlockBody(Collections.emptyList(), Collections.emptyList(), Optional.empty())); + final Block block2 = + new Block( + header2, + new BlockBody( + Collections.emptyList(), + Collections.emptyList(), + Optional.of(Collections.emptyList()))); + + final List expectedBlocks = asList(block1, block2); + final EthTask> task = + CompleteBlocksTask.forHeaders( + mockProtocolSchedule, + ethContext, + List.of(header1, header2), + maxRetries, + new NoOpMetricsSystem()); + assertThat(task.run()).isCompletedWithValue(expectedBlocks); + } + + @Test + public void shouldCompleteBlockThatOnlyContainsWithdrawals_whenWithdrawalsAreEnabled() { + final ProtocolSchedule mockProtocolSchedule = Mockito.mock(ProtocolSchedule.class); + final ProtocolSpec mockParisSpec = Mockito.mock(ProtocolSpec.class); + final ProtocolSpec mockShanghaiSpec = Mockito.mock(ProtocolSpec.class); + final WithdrawalsProcessor mockWithdrawalsProcessor = Mockito.mock(WithdrawalsProcessor.class); + + final Withdrawal withdrawal = + new Withdrawal(UInt64.ONE, UInt64.ONE, Address.fromHexString("0x1"), GWei.ONE); + final List withdrawals = List.of(withdrawal); + final Hash withdrawalsRoot = BodyValidation.withdrawalsRoot(withdrawals); + + final BlockHeader header1 = new BlockHeaderTestFixture().number(1).buildHeader(); + final BlockHeader header2 = + new BlockHeaderTestFixture().number(2).withdrawalsRoot(withdrawalsRoot).buildHeader(); + final BlockHeader header3 = + new BlockHeaderTestFixture().number(3).withdrawalsRoot(Hash.EMPTY_TRIE_HASH).buildHeader(); + + final Block block1 = new Block(header1, BlockBody.empty()); + final Block block2 = + new Block( + header2, + new BlockBody( + Collections.emptyList(), Collections.emptyList(), Optional.of(withdrawals))); + final Block block3 = + new Block( + header3, + new BlockBody( + Collections.emptyList(), + Collections.emptyList(), + Optional.of(Collections.emptyList()))); + final List expected = asList(block1, block2, block3); + + final RespondingEthPeer respondingPeer = + EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 1000); + final RespondingEthPeer.Responder responder = responderForFakeBlocks(expected); + + when(mockProtocolSchedule.getByBlockHeader((eq(header1)))).thenReturn(mockParisSpec); + when(mockParisSpec.getWithdrawalsProcessor()).thenReturn(Optional.empty()); + when(mockProtocolSchedule.getByBlockHeader((eq(header3)))).thenReturn(mockShanghaiSpec); + when(mockShanghaiSpec.getWithdrawalsProcessor()) + .thenReturn(Optional.of(mockWithdrawalsProcessor)); + + final EthTask> task = + CompleteBlocksTask.forHeaders( + mockProtocolSchedule, + ethContext, + List.of(header1, header2, header3), + maxRetries, + new NoOpMetricsSystem()); + + final CompletableFuture> runningTask = task.run(); + + assertThat(runningTask).isNotDone(); + respondingPeer.respond(responder); + assertThat(runningTask).isCompletedWithValue(expected); + } + + private RespondingEthPeer.Responder responderForFakeBlocks(final List blocks) { + final Blockchain mockBlockchain = spy(blockchain); + for (Block block : blocks) { + when(mockBlockchain.getBlockBody(block.getHash())).thenReturn(Optional.of(block.getBody())); + } + + return RespondingEthPeer.blockchainResponder(mockBlockchain); + } + @SuppressWarnings("unchecked") @Test public void shouldReduceTheBlockSegmentSizeAfterEachRetry() { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java index b829594587..162f8845a3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionPoolFactoryTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.transactions; import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule.DEFAULT_CHAIN_ID; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.atLeast; @@ -23,12 +24,15 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.BlockAddedObserver; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.MiningParameters; +import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthMessages; @@ -37,17 +41,22 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; +import org.hyperledger.besu.ethereum.eth.transactions.sorter.BaseFeePendingTransactionsSorter; import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolScheduleBuilder; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpecAdapters; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.testutil.TestClock; import java.math.BigInteger; import java.util.Collections; import java.util.Optional; +import java.util.function.Function; import org.assertj.core.api.Condition; import org.junit.Before; @@ -80,6 +89,8 @@ public class TransactionPoolFactoryTest { EthProtocolManager ethProtocolManager; + ProtocolContext protocolContext; + @Before public void setup() { when(blockchain.getBlockHashByNumber(anyLong())).thenReturn(Optional.of(mock(Hash.class))); @@ -245,4 +256,58 @@ public class TransactionPoolFactoryTest { mock(EthScheduler.class), mock(ForkIdManager.class)); } + + @Test + public void createTransactionPool_shouldUseBaseFeePendingTransactionsSorter_whenLondonEnabled() { + setupScheduleWith(new StubGenesisConfigOptions().londonBlock(0)); + + final TransactionPool pool = createTransactionPool(); + + assertThat(pool.getPendingTransactions()).isInstanceOf(BaseFeePendingTransactionsSorter.class); + } + + @Test + public void + createTransactionPool_shouldUseGasPricePendingTransactionsSorter_whenLondonNotEnabled() { + setupScheduleWith(new StubGenesisConfigOptions().berlinBlock(0)); + + final TransactionPool pool = createTransactionPool(); + + assertThat(pool.getPendingTransactions()).isInstanceOf(GasPricePendingTransactionsSorter.class); + } + + private void setupScheduleWith(final StubGenesisConfigOptions config) { + schedule = + new ProtocolScheduleBuilder( + config, + DEFAULT_CHAIN_ID, + ProtocolSpecAdapters.create(0, Function.identity()), + PrivacyParameters.DEFAULT, + false, + false, + EvmConfiguration.DEFAULT) + .createProtocolSchedule(); + + protocolContext = mock(ProtocolContext.class); + when(protocolContext.getBlockchain()).thenReturn(blockchain); + when(blockchain.getChainHeadHeader()).thenReturn(new BlockHeaderTestFixture().buildHeader()); + + syncState = new SyncState(blockchain, ethPeers, true, Optional.empty()); + } + + private TransactionPool createTransactionPool() { + return TransactionPoolFactory.createTransactionPool( + schedule, + protocolContext, + ethContext, + TestClock.fixed(), + new NoOpMetricsSystem(), + syncState, + new MiningParameters.Builder().minTransactionGasPrice(Wei.ONE).build(), + ImmutableTransactionPoolConfiguration.builder() + .txPoolMaxSize(1) + .txMessageKeepAliveSeconds(1) + .pendingTxRetentionPeriod(1) + .build()); + } } diff --git a/ethereum/evmtool/build.gradle b/ethereum/evmtool/build.gradle index bba853d443..ee3c6bf4ce 100644 --- a/ethereum/evmtool/build.gradle +++ b/ethereum/evmtool/build.gradle @@ -31,7 +31,6 @@ jar { } dependencies { - api 'org.slf4j:slf4j-api' implementation project(':besu') implementation project(':config') @@ -56,7 +55,6 @@ dependencies { implementation 'com.google.guava:guava' implementation 'info.picocli:picocli' implementation 'io.vertx:vertx-core' - implementation 'org.apache.logging.log4j:log4j-core' annotationProcessor 'com.google.dagger:dagger-compiler' diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java index 299448cd48..3a1f7e583b 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.referencetests.BlockchainReferenceTestCaseSpec.ReferenceTestBlockHeader; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.util.LogConfigurator; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -129,6 +130,7 @@ public class B11rSubCommand implements Runnable { @Override public void run() { + LogConfigurator.setLevel("", "OFF"); objectMapper.setDefaultPrettyPrinter( (new DefaultPrettyPrinter()) .withSpacesInObjectEntries() diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/CodeValidateSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/CodeValidateSubCommand.java index 606b389965..210efe014c 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/CodeValidateSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/CodeValidateSubCommand.java @@ -20,6 +20,7 @@ import static org.hyperledger.besu.evmtool.CodeValidateSubCommand.COMMAND_NAME; import org.hyperledger.besu.evm.code.CodeFactory; import org.hyperledger.besu.evm.code.CodeInvalid; import org.hyperledger.besu.evm.code.EOFLayout; +import org.hyperledger.besu.util.LogConfigurator; import java.io.BufferedReader; import java.io.File; @@ -68,6 +69,7 @@ public class CodeValidateSubCommand implements Runnable { @Override public void run() { + LogConfigurator.setLevel("", "OFF"); if (cliCode.isEmpty() && codeFile == null) { try (BufferedReader in = new BufferedReader(new InputStreamReader(input, UTF_8))) { checkCodeFromBufferedReader(in); diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java index d51781cfc4..f752ae15c1 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java @@ -41,7 +41,7 @@ import org.hyperledger.besu.evm.tracing.OperationTracer; import org.hyperledger.besu.evm.tracing.StandardJsonTracer; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; @@ -60,7 +60,6 @@ import java.util.Optional; import com.google.common.base.Joiner; import com.google.common.base.Stopwatch; import io.vertx.core.json.JsonObject; -import org.apache.logging.log4j.Level; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; @@ -251,6 +250,7 @@ public class EvmToolCommand implements Runnable { @Override public void run() { + LogConfigurator.setLevel("", "OFF"); try { final EvmToolComponent component = DaggerEvmToolComponent.builder() @@ -285,14 +285,9 @@ public class EvmToolCommand implements Runnable { .blockHeaderFunctions(new MainnetBlockHeaderFunctions()) .buildBlockHeader(); - Log4j2ConfiguratorUtil.setAllLevels("", repeat == 0 ? Level.INFO : Level.OFF); int remainingIters = this.repeat; - Log4j2ConfiguratorUtil.setLevel( - "org.hyperledger.besu.ethereum.mainnet.AbstractProtocolScheduleBuilder", Level.OFF); final ProtocolSpec protocolSpec = component.getProtocolSpec().apply(BlockHeaderBuilder.createDefault().buildBlockHeader()); - Log4j2ConfiguratorUtil.setLevel( - "org.hyperledger.besu.ethereum.mainnet.AbstractProtocolScheduleBuilder", null); final Transaction tx = new Transaction( 0, diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java index 0f2f266055..02c2d212f6 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java @@ -44,7 +44,7 @@ import org.hyperledger.besu.evm.tracing.StandardJsonTracer; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evmtool.exception.UnsupportedForkException; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import java.io.BufferedReader; import java.io.File; @@ -62,7 +62,6 @@ import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Stopwatch; -import org.apache.logging.log4j.Level; import org.apache.tuweni.bytes.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,6 +106,7 @@ public class StateTestSubCommand implements Runnable { @Override public void run() { + LogConfigurator.setLevel("", "OFF"); final ObjectMapper stateTestMapper = new ObjectMapper(); stateTestMapper.disable(Feature.AUTO_CLOSE_SOURCE); final JavaType javaType = @@ -162,12 +162,7 @@ public class StateTestSubCommand implements Runnable { } private void traceTestSpecs(final String test, final List specs) { - Log4j2ConfiguratorUtil.setLevel( - "org.hyperledger.besu.ethereum.mainnet.AbstractProtocolScheduleBuilder", Level.OFF); - final ReferenceTestProtocolSchedules referenceTestProtocolSchedules = - ReferenceTestProtocolSchedules.create(); - Log4j2ConfiguratorUtil.setLevel( - "org.hyperledger.besu.ethereum.mainnet.AbstractProtocolScheduleBuilder", null); + final var referenceTestProtocolSchedules = ReferenceTestProtocolSchedules.create(); final OperationTracer tracer = // You should have picked Mercy. parentCommand.showJsonResults diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java index 1e70d709e7..319ab014fb 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java @@ -43,6 +43,7 @@ import org.hyperledger.besu.ethereum.referencetests.ReferenceTestBlockchain; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; @@ -54,7 +55,7 @@ import org.hyperledger.besu.evm.tracing.StandardJsonTracer; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evmtool.exception.UnsupportedForkException; import org.hyperledger.besu.plugin.data.TransactionType; -import org.hyperledger.besu.util.Log4j2ConfiguratorUtil; +import org.hyperledger.besu.util.LogConfigurator; import java.io.FileOutputStream; import java.io.FileReader; @@ -65,6 +66,7 @@ import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -83,7 +85,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Stopwatch; -import org.apache.logging.log4j.Level; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; @@ -152,7 +153,6 @@ public class T8nSubCommand implements Runnable { description = "The summary of the transition") private final Path outResult = Path.of("result.json"); - @SuppressWarnings("UnusedVariable") @Option( names = {"--output.body"}, paramLabel = "file name", @@ -165,12 +165,12 @@ public class T8nSubCommand implements Runnable { description = "The chain Id to use") private final Long chainId = 1L; - @SuppressWarnings("UnusedVariable") + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) @Option( names = {"--state.reward"}, paramLabel = "block mining reward", description = "The block reward to use in block tess") - private final Wei reward = null; + private String rewardString = null; @ParentCommand private final EvmToolCommand parentCommand; @@ -188,6 +188,7 @@ public class T8nSubCommand implements Runnable { @Override public void run() { + LogConfigurator.setLevel("", "OFF"); final ObjectMapper objectMapper = new ObjectMapper(); objectMapper.setDefaultPrettyPrinter( (new DefaultPrettyPrinter()) @@ -231,7 +232,15 @@ public class T8nSubCommand implements Runnable { initialWorldState = objectMapper.convertValue(config.get("alloc"), ReferenceTestWorldState.class); initialWorldState.persist(null); - Iterator it = config.get("txs").elements(); + var node = config.get("txs"); + Iterator it; + if (node.isArray()) { + it = config.get("txs").elements(); + } else if (node == null || node.isNull()) { + it = Collections.emptyIterator(); + } else { + it = List.of(node).iterator(); + } transactions = extractTransactions(it); if (!outDir.toString().isBlank()) { @@ -246,13 +255,9 @@ public class T8nSubCommand implements Runnable { return; } - Log4j2ConfiguratorUtil.setLevel( - "org.hyperledger.besu.ethereum.mainnet.AbstractProtocolScheduleBuilder", Level.OFF); final ReferenceTestProtocolSchedules referenceTestProtocolSchedules = ReferenceTestProtocolSchedules.create( new StubGenesisConfigOptions().chainId(BigInteger.valueOf(chainId))); - Log4j2ConfiguratorUtil.setLevel( - "org.hyperledger.besu.ethereum.mainnet.AbstractProtocolScheduleBuilder", null); final MutableWorldState worldState = new DefaultMutableWorldState(initialWorldState); @@ -379,11 +384,16 @@ public class T8nSubCommand implements Runnable { final ObjectNode resultObject = objectMapper.createObjectNode(); // block reward - if (!validTransactions.isEmpty()) { + // The max production reward was 5 Eth, longs can hold over 18 Eth. + if (!validTransactions.isEmpty() && (rewardString == null || Long.decode(rewardString) > 0)) { + Wei reward = + (rewardString == null) + ? protocolSpec.getBlockReward() + : Wei.of(Long.decode(rewardString)); worldStateUpdater .getOrCreateSenderAccount(blockHeader.getCoinbase()) .getMutable() - .incrementBalance((reward == null) ? protocolSpec.getBlockReward() : reward); + .incrementBalance(reward); } // Invoke the withdrawal processor to handle CL withdrawals. @@ -478,14 +488,14 @@ public class T8nSubCommand implements Runnable { BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); rlpOut.writeList(transactions, Transaction::writeTo); - Bytes bodyBytes = rlpOut.encoded(); + TextNode bodyBytes = TextNode.valueOf(rlpOut.encoded().toHexString()); if (outBody.equals((stdoutPath))) { - outputObject.set("body", TextNode.valueOf(bodyBytes.toHexString())); + outputObject.set("body", bodyBytes); } else { try (PrintStream fileOut = new PrintStream(new FileOutputStream(outDir.resolve(outBody).toFile()))) { - fileOut.print(bodyBytes.toHexString()); + fileOut.println(bodyBytes); } } @@ -506,57 +516,70 @@ public class T8nSubCommand implements Runnable { } } - private static List extractTransactions(final Iterator it) { + private List extractTransactions(final Iterator it) { List transactions = new ArrayList<>(); while (it.hasNext()) { JsonNode txNode = it.next(); - if (txNode.has("txBytes")) { - Transaction tx = Transaction.readFrom(Bytes.fromHexString(txNode.get("txbytes").asText())); - transactions.add(tx); - } else { - Transaction.Builder builder = Transaction.builder(); - int type = Bytes.fromHexStringLenient(txNode.get("type").textValue()).toInt(); - TransactionType transactionType = TransactionType.of(type == 0 ? 0xf8 : type); - builder.type(transactionType); - builder.nonce(Bytes.fromHexStringLenient(txNode.get("nonce").textValue()).toLong()); - builder.gasPrice(Wei.fromHexString(txNode.get("gasPrice").textValue())); - builder.gasLimit(Bytes.fromHexStringLenient(txNode.get("gas").textValue()).toLong()); - builder.value(Wei.fromHexString(txNode.get("value").textValue())); - builder.payload(Bytes.fromHexString(txNode.get("input").textValue())); - if (txNode.has("to")) { - builder.to(Address.fromHexString(txNode.get("to").textValue())); - } - - if (transactionType.requiresChainId() - || !txNode.has("protected") - || txNode.get("protected").booleanValue()) { - // chainid if protected - builder.chainId( - new BigInteger( - 1, - Bytes.fromHexStringLenient(txNode.get("chainId").textValue()).toArrayUnsafe())); + if (txNode.isTextual()) { + BytesValueRLPInput rlpInput = + new BytesValueRLPInput(Bytes.fromHexString(txNode.asText()), false); + rlpInput.enterList(); + while (!rlpInput.isEndOfCurrentList()) { + Transaction tx = Transaction.readFrom(rlpInput); + transactions.add(tx); } + } else if (txNode.isObject()) { + if (txNode.has("txBytes")) { + Transaction tx = + Transaction.readFrom(Bytes.fromHexString(txNode.get("txbytes").asText())); + transactions.add(tx); + } else { + Transaction.Builder builder = Transaction.builder(); + int type = Bytes.fromHexStringLenient(txNode.get("type").textValue()).toInt(); + TransactionType transactionType = TransactionType.of(type == 0 ? 0xf8 : type); + builder.type(transactionType); + builder.nonce(Bytes.fromHexStringLenient(txNode.get("nonce").textValue()).toLong()); + builder.gasPrice(Wei.fromHexString(txNode.get("gasPrice").textValue())); + builder.gasLimit(Bytes.fromHexStringLenient(txNode.get("gas").textValue()).toLong()); + builder.value(Wei.fromHexString(txNode.get("value").textValue())); + builder.payload(Bytes.fromHexString(txNode.get("input").textValue())); + if (txNode.has("to")) { + builder.to(Address.fromHexString(txNode.get("to").textValue())); + } - if (txNode.has("secretKey")) { - SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance(); - KeyPair keys = - signatureAlgorithm.createKeyPair( - signatureAlgorithm.createPrivateKey( - Bytes32.fromHexString(txNode.get("secretKey").textValue()))); + if (transactionType.requiresChainId() + || !txNode.has("protected") + || txNode.get("protected").booleanValue()) { + // chainid if protected + builder.chainId( + new BigInteger( + 1, + Bytes.fromHexStringLenient(txNode.get("chainId").textValue()).toArrayUnsafe())); + } - transactions.add(builder.signAndBuild(keys)); - } else { - builder.signature( - SignatureAlgorithmFactory.getInstance() - .createSignature( - Bytes.fromHexString(txNode.get("r").textValue()).toUnsignedBigInteger(), - Bytes.fromHexString(txNode.get("s").textValue()).toUnsignedBigInteger(), - Bytes.fromHexString(txNode.get("v").textValue()) - .toUnsignedBigInteger() - .subtract(Transaction.REPLAY_UNPROTECTED_V_BASE) - .byteValueExact())); - transactions.add(builder.build()); + if (txNode.has("secretKey")) { + SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance(); + KeyPair keys = + signatureAlgorithm.createKeyPair( + signatureAlgorithm.createPrivateKey( + Bytes32.fromHexString(txNode.get("secretKey").textValue()))); + + transactions.add(builder.signAndBuild(keys)); + } else { + builder.signature( + SignatureAlgorithmFactory.getInstance() + .createSignature( + Bytes.fromHexString(txNode.get("r").textValue()).toUnsignedBigInteger(), + Bytes.fromHexString(txNode.get("s").textValue()).toUnsignedBigInteger(), + Bytes.fromHexString(txNode.get("v").textValue()) + .toUnsignedBigInteger() + .subtract(Transaction.REPLAY_UNPROTECTED_V_BASE) + .byteValueExact())); + transactions.add(builder.build()); + } } + } else { + parentCommand.out.printf("TX json node unparseable: %s%n", txNode); } } return transactions; diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-negative-cli-reward.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-negative-cli-reward.json new file mode 100644 index 0000000000..4ef273955c --- /dev/null +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-negative-cli-reward.json @@ -0,0 +1,96 @@ +{ + "cli": [ + "t8n", + "--input.alloc=stdin", + "--input.txs=stdin", + "--input.env=stdin", + "--output.result=stdout", + "--output.alloc=stdout", + "--output.body=stdout", + "--state.fork=Berlin", + "--state.chainid=1", + "--state.reward=-1" + ], + "stdin": { + "alloc": { + "0x1000000000000000000000000000000000000000": { + "nonce": "0x00", + "balance": "0x0ba1a9ce0ba1a9ce", + "code": "0x6010565b6000828201905092915050565b601a600260016003565b60005560206000f3", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "nonce": "0x00", + "balance": "0x0ba1a9ce0ba1a9ce", + "code": "0x", + "storage": {} + } + }, + "txs": [ + { + "type": "0x0", + "chainId": "0x0", + "nonce": "0x0", + "gasPrice": "0xa", + "gas": "0x7a120", + "value": "0x0", + "input": "0x", + "to": "0x1000000000000000000000000000000000000000", + "protected": false, + "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "v": "0x0", + "r": "0x0", + "s": "0x0" + } + ], + "env": { + "currentCoinbase": "0x0000000000000000000000000000000000000000", + "currentDifficulty": "0x0", + "currentGasLimit": "0x0", + "currentNumber": "0", + "currentTimestamp": "0" + } + }, + "stdout": { + "alloc": { + "0x0000000000000000000000000000000000000000": { + "balance": "0x696c2" + }, + "0x1000000000000000000000000000000000000000": { + "code": "0x6010565b6000828201905092915050565b601a600260016003565b60005560206000f3", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x0000000000000000000000000000000000000000000000000000000000000003" + }, + "balance": "0xba1a9ce0ba1a9ce" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0xba1a9ce0b9b130c", + "nonce": "0x1" + } + }, + "body": "0xf862f860800a8307a12094100000000000000000000000000000000000000080801ca091df80a63242846dcf1cc16c35e6ba39212cde92aad946f9c286fa0029782b54a0446170c2fb714c717885485ea7d93bf9277f3fb3d89ffd73e9b7f5f8eda3997b", + "result": { + "stateRoot": "0xde762e485411037c02119ff1115b422945b37efe91f4135dd442c3346629d0ef", + "txRoot": "0x8e987be72f36f97a98838e03d9e5d1b3d22795606240f16684532b1c994b4ee7", + "receiptsRoot": "0x0812e2d0238699cad1dd4c1274804ed2dde9a35df8ee002d48bbf3ef87857a63", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [ + { + "root": "0x", + "status": "0x1", + "cumulativeGasUsed": "0xa8ad", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0x6c4a22ee7d2d9ae9307b09aac69a34600978e34313088917a7e69fd12e8f959a", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0xa8ad", + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x0" + } + ], + "currentDifficulty": null, + "gasUsed": "0xa8ad" + } + } +} \ No newline at end of file diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-hex.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-hex.json index 15ae26d33c..502ac628de 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-hex.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-hex.json @@ -64,9 +64,6 @@ }, "stdout": { "alloc": { - "0x0000000000000000000000000000000000000000": { - "balance": "0x0" - }, "0x1000000000000000000000000000000000000000": { "code": "0x6010565b6000828201905092915050565b601a600260016003565b60005560206000f3", "storage": { @@ -81,7 +78,7 @@ }, "body": "0xf862f860800a8307a12094100000000000000000000000000000000000000080801ca091df80a63242846dcf1cc16c35e6ba39212cde92aad946f9c286fa0029782b54a0446170c2fb714c717885485ea7d93bf9277f3fb3d89ffd73e9b7f5f8eda3997b", "result": { - "stateRoot": "0x32c517c68424f758724f80f6f1d6e470d43c7be876c62b32096f49574f5d541e", + "stateRoot": "0x5775e461ac316f8c02466d23e4492d24bbe546336f6fc632dd1311d6a5b67b22", "txRoot": "0x8e987be72f36f97a98838e03d9e5d1b3d22795606240f16684532b1c994b4ee7", "receiptsRoot": "0x0812e2d0238699cad1dd4c1274804ed2dde9a35df8ee002d48bbf3ef87857a63", "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-invalidTx.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-invalidTx.json new file mode 100644 index 0000000000..7ec8a1d776 --- /dev/null +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-invalidTx.json @@ -0,0 +1,87 @@ +{ + "cli": [ + "t8n", + "--input.alloc=stdin", + "--input.txs=stdin", + "--input.env=stdin", + "--output.result=stdout", + "--output.alloc=stdout", + "--output.body=stdout", + "--state.fork=Shanghai", + "--state.chainid=1", + "--state.reward=2000000000000000000" + ], + "stdin": { + "alloc": { + "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0", + "code": "0x", + "nonce": "1", + "storage": { + } + }, + "095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "1000000000000000000", + "code": "0x600160010160005500", + "nonce": "0", + "storage": { + } + }, + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "1000000000000000000", + "code": "0x", + "nonce": "0", + "storage": { + } + } + }, + "txs": "0xf864f862800a8203e894095e7baea6a6c7c4c2dfeb977efac326af552d87830186a0801ba0bb6249049eb38732aa28ef23385945e51c9582c7fab78bd33f6086bd4181615da035118c5432cd6a3bdd6f989d4eee3d43d11736e640bc249d9a2c37e01db3d4b5", + "env": { + "currentCoinbase": "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentGasLimit": "0xFF112233445566", + "currentNumber": "1", + "currentTimestamp": "1000", + "currentDifficulty": "0x20000", + "currentBaseFee": "0xa", + "blockHashes": { + "0": "0x5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" + }, + "ommers": [], + "previousHash": "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" + } + }, + "stdout": { + "alloc": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "code": "0x600160010160005500", + "balance": "0x1000000000000000000" + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x0", + "nonce": "0x1" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x1000000000000000000" + } + }, + "body": "0xf864f862800a8203e894095e7baea6a6c7c4c2dfeb977efac326af552d87830186a0801ba0bb6249049eb38732aa28ef23385945e51c9582c7fab78bd33f6086bd4181615da035118c5432cd6a3bdd6f989d4eee3d43d11736e640bc249d9a2c37e01db3d4b5", + "result": { + "stateRoot": "0xec92f4c572101075d17eb5aaf15c33df92b6d5519cbed635fc53353b99e8e6da", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "rejected": [ + { + "index": 0, + "error": "intrinsic gas cost 21000 exceeds gas limit 1000" + } + ], + "currentDifficulty": "0x20000", + "gasUsed": "0x0", + "currentBaseFee": "0xa", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + } + } +} \ No newline at end of file diff --git a/ethereum/referencetests/src/reference-test/external-resources b/ethereum/referencetests/src/reference-test/external-resources index 638da7ca7b..69c4c2a98d 160000 --- a/ethereum/referencetests/src/reference-test/external-resources +++ b/ethereum/referencetests/src/reference-test/external-resources @@ -1 +1 @@ -Subproject commit 638da7ca7b2ca371ebb33d7d7d2562981aafd9b2 +Subproject commit 69c4c2a98dc8a712d4c6f5a817da4f21ff719006 diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java index a2e5230ffd..7c25fab1ba 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java @@ -80,7 +80,7 @@ public class BlockchainReferenceTestTools { // Inconclusive fork choice rule, since in merge CL should be choosing forks and setting the // chain head. // Perfectly valid test pre-merge. - params.ignore("UncleFromSideChain_Merge"); + params.ignore("UncleFromSideChain_(Merge|Shanghai|Cancun|Prague|Osaka|Bogota)"); // EIP tests are explicitly meant to be works-in-progress with known failing tests // We want to however include withdrawals even though they are EIP tests diff --git a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/NoRewardProtocolScheduleWrapper.java b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/NoRewardProtocolScheduleWrapper.java index e072101d2f..64c5d9c0f2 100644 --- a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/NoRewardProtocolScheduleWrapper.java +++ b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/NoRewardProtocolScheduleWrapper.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.ethereum.retesteth; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.BlockValidator; import org.hyperledger.besu.ethereum.MainnetBlockValidator; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockImporter; import org.hyperledger.besu.ethereum.core.TransactionFilter; import org.hyperledger.besu.ethereum.mainnet.BlockProcessor; @@ -24,10 +25,12 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockImporter; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.ScheduledProtocolSpec; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.math.BigInteger; import java.util.Optional; +import java.util.function.Predicate; import java.util.stream.Stream; public class NoRewardProtocolScheduleWrapper implements ProtocolSchedule { @@ -91,6 +94,16 @@ public class NoRewardProtocolScheduleWrapper implements ProtocolSchedule { return delegate.streamMilestoneBlocks(); } + @Override + public boolean anyMatch(final Predicate predicate) { + return delegate.anyMatch(predicate); + } + + @Override + public boolean isOnMilestoneBoundary(final BlockHeader blockHeader) { + return delegate.isOnMilestoneBoundary(blockHeader); + } + @Override public Optional getChainId() { return delegate.getChainId(); diff --git a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java index 270cc1e777..76da2e86c8 100644 --- a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java +++ b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java @@ -196,11 +196,7 @@ public class RetestethContext { 1000, 8); - blockReplay = - new BlockReplay( - protocolSchedule, - blockchainQueries.getBlockchain(), - blockchainQueries.getWorldStateArchive()); + blockReplay = new BlockReplay(protocolSchedule, blockchainQueries.getBlockchain()); // mining support diff --git a/gradle.properties b/gradle.properties index 305c046ef1..232f6f25fb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,8 +1,8 @@ -version=23.1.1 +version=23.1.2 org.gradle.welcome=never # Set exports/opens flags required by Google Java Format and ErrorProne plugins. (JEP-396) -org.gradle.jvmargs=-Xmx2g \ +org.gradle.jvmargs=-Xmx4g \ --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED \ diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index a922547577..30277e1b89 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -4262,6 +4262,11 @@ + + + + + @@ -5355,6 +5360,22 @@ + + + + + + + + + + + + + + + + @@ -5365,6 +5386,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index a716be8879..72373dcb57 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -193,6 +193,7 @@ dependencyManagement { dependency 'org.rocksdb:rocksdbjni:7.7.3' dependency 'org.slf4j:slf4j-api:2.0.6' + dependency 'org.slf4j:slf4j-nop:2.0.6' dependency 'org.springframework.security:spring-security-crypto:5.7.2' diff --git a/gradlew b/gradlew index 1b6c787337..aef870434d 100755 --- a/gradlew +++ b/gradlew @@ -86,7 +86,7 @@ APP_NAME="Gradle" APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +DEFAULT_JVM_OPTS='"-Xmx1024m" "-Xms256m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 55dac9face..a79bc87a9d 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -31,6 +31,7 @@ dependencies { api 'org.apache.commons:commons-lang3' api 'org.apache.tuweni:tuweni-bytes' api 'org.apache.tuweni:tuweni-units' + implementation 'com.google.guava:guava' } configurations { testArtifacts } @@ -66,7 +67,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'sh5/mQlcilXjb/z51QbcvJTlqpI2uBe2EMli1WBV2tw=' + knownHash = 'LhQ25UazqtTJuZJN1bj76imJ7QdkFRR6dUyxt7Ig5cs=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SnappedKeyValueStorage.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SnappedKeyValueStorage.java index 9170f3f5b2..032563e925 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SnappedKeyValueStorage.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SnappedKeyValueStorage.java @@ -24,11 +24,4 @@ public interface SnappedKeyValueStorage extends KeyValueStorage { * @return the snapshot transaction */ KeyValueStorageTransaction getSnapshotTransaction(); - - /** - * Clone from snapshot. - * - * @return the snapped key value storage - */ - SnappedKeyValueStorage cloneFromSnapshot(); } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java index c7768b1d5c..1c40716f43 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java @@ -26,19 +26,28 @@ import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDbSegmentIdenti import java.io.IOException; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; import org.rocksdb.OptimisticTransactionDB; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** The RocksDb columnar key value snapshot. */ public class RocksDBColumnarKeyValueSnapshot implements SnappedKeyValueStorage { + + private static final Logger LOG = LoggerFactory.getLogger(RocksDBColumnarKeyValueSnapshot.class); + /** The Db. */ final OptimisticTransactionDB db; + /** The Snap tx. */ final RocksDBSnapshotTransaction snapTx; + private final AtomicBoolean closed = new AtomicBoolean(false); + /** * Instantiates a new RocksDb columnar key value snapshot. * @@ -54,29 +63,27 @@ public class RocksDBColumnarKeyValueSnapshot implements SnappedKeyValueStorage { this.snapTx = new RocksDBSnapshotTransaction(db, segment.get(), metrics); } - private RocksDBColumnarKeyValueSnapshot( - final OptimisticTransactionDB db, final RocksDBSnapshotTransaction snapTx) { - this.db = db; - this.snapTx = snapTx; - } - @Override public Optional get(final byte[] key) throws StorageException { + throwIfClosed(); return snapTx.get(key); } @Override public Stream> stream() { + throwIfClosed(); return snapTx.stream(); } @Override public Stream streamKeys() { + throwIfClosed(); return snapTx.streamKeys(); } @Override public boolean tryDelete(final byte[] key) throws StorageException { + throwIfClosed(); snapTx.remove(key); return true; } @@ -109,21 +116,26 @@ public class RocksDBColumnarKeyValueSnapshot implements SnappedKeyValueStorage { @Override public boolean containsKey(final byte[] key) throws StorageException { + throwIfClosed(); return snapTx.get(key).isPresent(); } @Override public void close() throws IOException { - snapTx.close(); + if (closed.compareAndSet(false, true)) { + snapTx.close(); + } } - @Override - public KeyValueStorageTransaction getSnapshotTransaction() { - return snapTx; + private void throwIfClosed() { + if (closed.get()) { + LOG.error("Attempting to use a closed RocksDBKeyValueStorage"); + throw new IllegalStateException("Storage has been closed"); + } } @Override - public SnappedKeyValueStorage cloneFromSnapshot() { - return new RocksDBColumnarKeyValueSnapshot(db, snapTx.copy()); + public KeyValueStorageTransaction getSnapshotTransaction() { + return snapTx; } } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java index 6bf870df6c..a3d31cb2a4 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java @@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory; public class RocksDBSnapshotTransaction implements KeyValueStorageTransaction, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(RocksDBSnapshotTransaction.class); private static final String NO_SPACE_LEFT_ON_DEVICE = "No space left on device"; - private final RocksDBMetrics metrics; private final OptimisticTransactionDB db; private final ColumnFamilyHandle columnFamilyHandle; diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java index a2d6f28a3b..1b1df6621e 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBKeyValueStorage.java @@ -97,6 +97,7 @@ public class RocksDBKeyValueStorage implements KeyValueStorage { @Override public void clear() throws StorageException { + throwIfClosed(); try (final RocksIterator rocksIterator = db.newIterator()) { rocksIterator.seekToFirst(); if (rocksIterator.isValid()) { @@ -140,6 +141,7 @@ public class RocksDBKeyValueStorage implements KeyValueStorage { @Override public Stream> stream() { + throwIfClosed(); final RocksIterator rocksIterator = db.newIterator(); rocksIterator.seekToFirst(); return RocksDbIterator.create(rocksIterator).toStream(); @@ -147,6 +149,7 @@ public class RocksDBKeyValueStorage implements KeyValueStorage { @Override public Stream streamKeys() { + throwIfClosed(); final RocksIterator rocksIterator = db.newIterator(); rocksIterator.seekToFirst(); return RocksDbIterator.create(rocksIterator).toStreamKeys(); @@ -162,6 +165,7 @@ public class RocksDBKeyValueStorage implements KeyValueStorage { @Override public boolean tryDelete(final byte[] key) { + throwIfClosed(); try { db.delete(tryDeleteOptions, key); return true; diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java index 86fbaf836f..627313f71b 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java @@ -42,8 +42,11 @@ import org.apache.tuweni.bytes.Bytes; public class InMemoryKeyValueStorage implements SnappedKeyValueStorage, SnappableKeyValueStorage, KeyValueStorage { - private final Map hashValueStore; - private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); + /** protected access for the backing hash map. */ + protected final Map> hashValueStore; + + /** protected access to the rw lock. */ + protected final ReadWriteLock rwLock = new ReentrantReadWriteLock(); /** Instantiates a new In memory key value storage. */ public InMemoryKeyValueStorage() { @@ -55,7 +58,7 @@ public class InMemoryKeyValueStorage * * @param hashValueStore the hash value store */ - protected InMemoryKeyValueStorage(final Map hashValueStore) { + protected InMemoryKeyValueStorage(final Map> hashValueStore) { this.hashValueStore = hashValueStore; } @@ -80,7 +83,7 @@ public class InMemoryKeyValueStorage final Lock lock = rwLock.readLock(); lock.lock(); try { - return Optional.ofNullable(hashValueStore.get(Bytes.wrap(key))); + return hashValueStore.getOrDefault(Bytes.wrap(key), Optional.empty()); } finally { lock.unlock(); } @@ -108,7 +111,10 @@ public class InMemoryKeyValueStorage lock.lock(); try { return ImmutableSet.copyOf(hashValueStore.entrySet()).stream() - .map(bytesEntry -> Pair.of(bytesEntry.getKey().toArrayUnsafe(), bytesEntry.getValue())); + .filter(bytesEntry -> bytesEntry.getValue().isPresent()) + .map( + bytesEntry -> + Pair.of(bytesEntry.getKey().toArrayUnsafe(), bytesEntry.getValue().get())); } finally { lock.unlock(); } @@ -167,19 +173,17 @@ public class InMemoryKeyValueStorage return startTransaction(); } - @Override - public SnappedKeyValueStorage cloneFromSnapshot() { - return takeSnapshot(); - } - - private class InMemoryTransaction implements KeyValueStorageTransaction { + /** In memory transaction. */ + public class InMemoryTransaction implements KeyValueStorageTransaction { - private Map updatedValues = new HashMap<>(); - private Set removedKeys = new HashSet<>(); + /** protected access to updatedValues map for the transaction. */ + protected Map> updatedValues = new HashMap<>(); + /** protected access to deletedValues set for the transaction. */ + protected Set removedKeys = new HashSet<>(); @Override public void put(final byte[] key, final byte[] value) { - updatedValues.put(Bytes.wrap(key), value); + updatedValues.put(Bytes.wrap(key), Optional.of(value)); removedKeys.remove(Bytes.wrap(key)); } @@ -219,8 +223,14 @@ public class InMemoryKeyValueStorage final Lock lock = rwLock.readLock(); lock.lock(); try { - hashValueStore.forEach( - (k, v) -> ps.printf(" %s : %s%n", k.toHexString(), Bytes.wrap(v).toHexString())); + ImmutableSet.copyOf(hashValueStore.entrySet()).stream() + .filter(bytesEntry -> bytesEntry.getValue().isPresent()) + .forEach( + entry -> + ps.printf( + " %s : %s%n", + entry.getKey().toHexString(), + Bytes.wrap(entry.getValue().get()).toHexString())); } finally { lock.unlock(); } diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java new file mode 100644 index 0000000000..b8144e8bba --- /dev/null +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java @@ -0,0 +1,158 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.services.kvstore; + +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.stream.Stream; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Streams; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.tuweni.bytes.Bytes; + +/** Key value storage which stores in memory all updates to a parent worldstate storage. */ +public class LayeredKeyValueStorage extends InMemoryKeyValueStorage + implements SnappedKeyValueStorage { + + private final KeyValueStorage parent; + + /** + * Instantiates a new Layered key value storage. + * + * @param parent the parent key value storage for this layered storage. + */ + public LayeredKeyValueStorage(final KeyValueStorage parent) { + this(new ConcurrentHashMap<>(), parent); + } + + /** + * Constructor which takes an explicit backing map for the layered key value storage. + * + * @param map the backing map + * @param parent the parent key value storage for this layered storage. + */ + public LayeredKeyValueStorage( + final Map> map, final KeyValueStorage parent) { + super(map); + this.parent = parent; + } + + @Override + public boolean containsKey(final byte[] key) throws StorageException { + return get(key).isPresent(); + } + + @Override + public Optional get(final byte[] key) throws StorageException { + final Lock lock = rwLock.readLock(); + lock.lock(); + try { + Bytes wrapKey = Bytes.wrap(key); + final Optional foundKey = hashValueStore.get(wrapKey); + if (foundKey == null) { + return parent.get(key); + } else { + return foundKey; + } + } finally { + lock.unlock(); + } + } + + @Override + public Stream> stream() { + final Lock lock = rwLock.readLock(); + lock.lock(); + try { + // immutable copy of our in memory store to use for streaming and filtering: + Map> ourLayerState = ImmutableMap.copyOf(hashValueStore); + + return Streams.concat( + ourLayerState.entrySet().stream() + .filter(entry -> entry.getValue().isPresent()) + .map( + bytesEntry -> + Pair.of(bytesEntry.getKey().toArrayUnsafe(), bytesEntry.getValue().get())) + // since we are layered, concat a parent stream filtered by our map entries: + , + parent.stream().filter(e -> !ourLayerState.containsKey(Bytes.of(e.getLeft())))); + } finally { + lock.unlock(); + } + } + + @Override + public Stream streamKeys() { + final Lock lock = rwLock.readLock(); + lock.lock(); + try { + // immutable copy of our in memory store to use for streaming and filtering: + Map> ourLayerState = ImmutableMap.copyOf(hashValueStore); + + return Streams.concat( + ourLayerState.entrySet().stream() + .filter(entry -> entry.getValue().isPresent()) + .map(bytesEntry -> bytesEntry.getKey().toArrayUnsafe()) + // since we are layered, concat a parent stream filtered by our map entries: + , + parent.streamKeys().filter(e -> !ourLayerState.containsKey(Bytes.of(e)))); + + } finally { + lock.unlock(); + } + } + + @Override + public boolean tryDelete(final byte[] key) { + hashValueStore.put(Bytes.wrap(key), Optional.empty()); + return true; + } + + @Override + public KeyValueStorageTransaction startTransaction() { + return new KeyValueStorageTransactionTransitionValidatorDecorator( + new InMemoryTransaction() { + @Override + public void commit() throws StorageException { + + final Lock lock = rwLock.writeLock(); + lock.lock(); + try { + hashValueStore.putAll(updatedValues); + removedKeys.forEach(key -> hashValueStore.put(key, Optional.empty())); + // put empty and not removed to not ask parent in case of deletion + updatedValues = null; + removedKeys = null; + } finally { + lock.unlock(); + } + } + }); + } + + @Override + public SnappedKeyValueStorage clone() { + return new LayeredKeyValueStorage(hashValueStore, parent); + } +} diff --git a/util/src/main/java/org/hyperledger/besu/util/Log4j2ConfiguratorUtil.java b/util/src/main/java/org/hyperledger/besu/util/Log4j2ConfiguratorUtil.java index ca5d7a2bd2..d47294fc16 100644 --- a/util/src/main/java/org/hyperledger/besu/util/Log4j2ConfiguratorUtil.java +++ b/util/src/main/java/org/hyperledger/besu/util/Log4j2ConfiguratorUtil.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.util; +import static java.util.Objects.requireNonNull; + import java.util.Map; import java.util.Set; @@ -26,7 +28,7 @@ import org.apache.logging.slf4j.Log4jLoggerFactory; import org.slf4j.LoggerFactory; /** The Log4j2 configurator util. */ -public class Log4j2ConfiguratorUtil { +class Log4j2ConfiguratorUtil { private Log4j2ConfiguratorUtil() {} @@ -36,18 +38,20 @@ public class Log4j2ConfiguratorUtil { * @param parentLogger the parent logger * @param level the level */ - public static void setAllLevels(final String parentLogger, final Level level) { + static void setAllLevels(final String parentLogger, final String level) { // 1) get logger config // 2) if exact match, use it, if not, create it. // 3) set level on logger config // 4) update child logger configs with level // 5) update loggers + Level log4JLevel = Level.toLevel(level, null); + requireNonNull(log4JLevel); final LoggerContext loggerContext = getLoggerContext(); final Configuration config = loggerContext.getConfiguration(); - boolean set = setLevel(parentLogger, level, config); + boolean set = setLevel(parentLogger, log4JLevel, config); for (final Map.Entry entry : config.getLoggers().entrySet()) { if (entry.getKey().startsWith(parentLogger)) { - set |= setLevel(entry.getValue(), level); + set |= setLevel(entry.getValue(), log4JLevel); } } if (set) { @@ -60,8 +64,8 @@ public class Log4j2ConfiguratorUtil { * * @param loggerName the logger name */ - public static void setLevelDebug(final String loggerName) { - setLevel(loggerName, Level.DEBUG); + static void setLevelDebug(final String loggerName) { + setLevel(loggerName, "DEBUG"); } /** @@ -70,11 +74,13 @@ public class Log4j2ConfiguratorUtil { * @param loggerName the logger name * @param level the level */ - public static void setLevel(final String loggerName, final Level level) { + static void setLevel(final String loggerName, final String level) { + Level log4jLevel = Level.toLevel(level, null); + requireNonNull(log4jLevel); final LoggerContext loggerContext = getLoggerContext(); if (Strings.isEmpty(loggerName)) { - setRootLevel(loggerContext, level); - } else if (setLevel(loggerName, level, loggerContext.getConfiguration())) { + setRootLevel(loggerContext, log4jLevel); + } else if (setLevel(loggerName, log4jLevel, loggerContext.getConfiguration())) { loggerContext.updateLoggers(); } } @@ -111,7 +117,7 @@ public class Log4j2ConfiguratorUtil { } /** Reconfigure. */ - public static void reconfigure() { + static void reconfigure() { getLoggerContext().reconfigure(); } @@ -122,7 +128,7 @@ public class Log4j2ConfiguratorUtil { } /** Shutdown. */ - public static void shutdown() { + static void shutdown() { getLoggerContext().terminate(); } } diff --git a/util/src/main/java/org/hyperledger/besu/util/LogConfigurator.java b/util/src/main/java/org/hyperledger/besu/util/LogConfigurator.java new file mode 100644 index 0000000000..b0ca0bcd46 --- /dev/null +++ b/util/src/main/java/org/hyperledger/besu/util/LogConfigurator.java @@ -0,0 +1,54 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.util; + +import java.util.NoSuchElementException; + +/** The library independent logger configurator util. */ +@SuppressWarnings("CatchAndPrintStackTrace") +public interface LogConfigurator { + + /** + * Sets level to specified logger. + * + * @param parentLogger the logger name + * @param level the level + */ + static void setLevel(final String parentLogger, final String level) { + try { + Log4j2ConfiguratorUtil.setAllLevels(parentLogger, level); + } catch (NoClassDefFoundError | ClassCastException | NoSuchElementException e) { + // This is expected when Log4j support is not in the classpath, so ignore + } + } + + /** Reconfigure. */ + static void reconfigure() { + try { + Log4j2ConfiguratorUtil.reconfigure(); + } catch (NoClassDefFoundError | ClassCastException | NoSuchElementException e) { + // This is expected when Log4j support is not in the classpath, so ignore + } + } + + /** Shutdown. */ + static void shutdown() { + try { + Log4j2ConfiguratorUtil.shutdown(); + } catch (NoClassDefFoundError | ClassCastException | NoSuchElementException e) { + // This is expected when Log4j support is not in the classpath, so ignore + } + } +}