From 180c75197c4a32e4a2d4cf6b732a580197fbf496 Mon Sep 17 00:00:00 2001 From: matkt Date: Tue, 13 Jun 2023 16:11:06 +0200 Subject: [PATCH] Healing Mechanism for Flat Database in Besu (#5319) The proposed pull request introduces a feature that allows healing of the flat database by streaming the flat database data and validating it by generating a proof from the trie structure. If the proof is found to be invalid, the code traverses the trie to fix the invalid range. To optimize the process and avoid checking the entire flat database, the PR includes enhancements such as tracking the accounts that need to be repaired during SnapSync. By implementing these optimizations, the PR aims to significantly reduce the time and resources required for repairing the flat database. Signed-off-by: Karim TAAM --- CHANGELOG.md | 2 + .../org/hyperledger/besu/cli/BesuCommand.java | 8 + .../options/unstable/SynchronizerOptions.java | 158 ++++++--- .../hyperledger/besu/cli/BesuCommandTest.java | 26 ++ .../bonsai/BonsaiWorldStateProvider.java | 8 + ...nsaiSnapshotWorldStateKeyValueStorage.java | 2 + .../BonsaiWorldStateKeyValueStorage.java | 259 +++++++------- .../storage/BonsaiWorldStateLayerStorage.java | 6 + .../storage/flat/FlatDbReaderStrategy.java | 174 ++++++++++ .../flat/FullFlatDbReaderStrategy.java | 98 ++++++ .../flat/PartialFlatDbReaderStrategy.java | 140 ++++++++ .../proof/WorldStateProofProvider.java | 38 +- .../keyvalue/WorldStateKeyValueStorage.java | 17 +- .../besu/ethereum/worldstate/FlatDbMode.java | 57 +++ .../worldstate/WorldStateStorage.java | 35 +- .../BonsaiWorldStateKeyValueStorageTest.java | 136 +++++++- .../besu/ethereum/bonsai/LogRollingTests.java | 58 +--- .../besu/ethereum/bonsai/RollingImport.java | 13 +- .../eth/sync/DefaultSynchronizer.java | 6 +- .../CheckpointDownloaderFactory.java | 17 +- .../eth/sync/fastsync/FastSyncDownloader.java | 4 +- .../AccountTrieNodeDataRequest.java | 5 +- .../StorageTrieNodeDataRequest.java | 3 +- .../eth/sync/snapsync/CompleteTaskStep.java | 9 +- ...er.java => DynamicPivotBlockSelector.java} | 15 +- .../eth/sync/snapsync/LoadLocalDataStep.java | 17 +- .../eth/sync/snapsync/PersistDataStep.java | 53 ++- .../eth/sync/snapsync/RangeManager.java | 14 + .../eth/sync/snapsync/RequestDataStep.java | 107 +++++- .../sync/snapsync/SnapDownloaderFactory.java | 14 +- .../sync/snapsync/SnapSyncConfiguration.java | 33 +- .../eth/sync/snapsync/SnapSyncDownloader.java | 2 +- ...ncState.java => SnapSyncProcessState.java} | 29 +- .../sync/snapsync/SnapWorldDownloadState.java | 144 ++++++-- .../SnapWorldStateDownloadProcess.java | 132 +++++-- .../snapsync/SnapWorldStateDownloader.java | 52 ++- .../sync/snapsync/SnapsyncMetricsManager.java | 155 ++++++--- .../ethereum/eth/sync/snapsync/StackTrie.java | 27 +- .../SnapSyncStatePersistenceManager.java} | 35 +- .../request/AccountRangeDataRequest.java | 33 +- .../snapsync/request/BytecodeRequest.java | 14 +- .../snapsync/request/SnapDataRequest.java | 58 +++- .../request/StorageRangeDataRequest.java | 57 ++- ...ccountFlatDatabaseHealingRangeRequest.java | 210 +++++++++++ .../AccountTrieNodeHealingRequest.java} | 24 +- ...torageFlatDatabaseHealingRangeRequest.java | 190 ++++++++++ .../StorageTrieNodeHealingRequest.java} | 47 ++- .../TrieNodeHealingRequest.java} | 28 +- .../ethereum/eth/sync/state/SyncState.java | 2 +- .../sync/fastsync/FastSyncDownloaderTest.java | 2 + .../sync/snapsync/CompleteTaskStepTest.java | 2 +- .../DynamicPivotBlockManagerTest.java | 6 +- .../sync/snapsync/LoadLocalDataStepTest.java | 17 +- .../sync/snapsync/PersistDataStepTest.java | 6 +- .../eth/sync/snapsync/RangeManagerTest.java | 37 +- .../snapsync/SnapWorldDownloadStateTest.java | 105 ++++-- ...ntFlatDatabaseHealingRangeRequestTest.java | 297 ++++++++++++++++ ...geFlatDatabaseHealingRangeRequestTest.java | 325 ++++++++++++++++++ .../trie/RangeStorageEntriesCollector.java | 4 +- .../besu/ethereum/trie/TrieIterator.java | 19 +- .../RangeStorageEntriesCollectorTest.java | 85 +++++ .../besu/ethereum/trie/TrieIteratorTest.java | 1 + plugin-api/build.gradle | 2 +- .../services/storage/KeyValueStorage.java | 11 + .../RocksDBColumnarKeyValueSnapshot.java | 6 + .../RocksDBColumnarKeyValueStorage.java | 8 + .../unsegmented/RocksDBKeyValueStorage.java | 6 + .../kvstore/InMemoryKeyValueStorage.java | 5 + .../kvstore/LayeredKeyValueStorage.java | 5 + .../LimitedInMemoryKeyValueStorage.java | 5 + .../kvstore/SegmentedKeyValueStorage.java | 11 + .../SegmentedKeyValueStorageAdapter.java | 5 + .../besu/services/pipeline/Pipeline.java | 2 +- 73 files changed, 3130 insertions(+), 613 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FullFlatDbReaderStrategy.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/PartialFlatDbReaderStrategy.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/FlatDbMode.java rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/{DynamicPivotBlockManager.java => DynamicPivotBlockSelector.java} (95%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/{SnapSyncState.java => SnapSyncProcessState.java} (68%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/{SnapPersistedContext.java => context/SnapSyncStatePersistenceManager.java} (77%) create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/{AccountTrieNodeDataRequest.java => heal/AccountTrieNodeHealingRequest.java} (84%) create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/{StorageTrieNodeDataRequest.java => heal/StorageTrieNodeHealingRequest.java} (58%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/{TrieNodeDataRequest.java => heal/TrieNodeHealingRequest.java} (81%) create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java create mode 100644 ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 274918ccf0..786d9f43b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and in case a rollback is needed, before installing a previous version, the migr - Tidy DEBUG logs by moving engine API full logging to TRACE [#5529](https://github.com/hyperledger/besu/pull/5529) - remove PoW validation if merge is enabled as it is not needed anymore [#5538](https://github.com/hyperledger/besu/pull/5538) - Use BlobDB for blockchain storage to reduce initial sync time and write amplification [#5475](https://github.com/hyperledger/besu/pull/5475) +- Update to Tuweni 2.4.1. [#5513](https://github.com/hyperledger/besu/pull/5513) +- Add healing flat db mechanism with experimental CLI options `--Xsnapsync-synchronizer-flat-db-healing-enabled=true` [#5319](https://github.com/hyperledger/besu/pull/5319) ### Bug Fixes - check to ensure storage and transactions are not closed prior to reading/writing [#5527](https://github.com/hyperledger/besu/pull/5527) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index d77dac0ce9..c9a9b599dc 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2168,6 +2168,14 @@ public class BesuCommand implements DefaultCommandValues, Runnable { SyncMode.X_CHECKPOINT.equals(getDefaultSyncModeIfNotSet()), singletonList("--Xcheckpoint-post-merge-enabled")); + CommandLineUtils.failIfOptionDoesntMeetRequirement( + commandLine, + "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true", + unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled(), + asList( + "--Xsnapsync-synchronizer-flat-account-healed-count-per-request", + "--Xsnapsync-synchronizer-flat-slot-healed-count-per-request")); + if (!securityModuleName.equals(DEFAULT_SECURITY_MODULE) && nodePrivateKeyFileOption.getNodePrivateKeyFile() != null) { logger.warn( diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index 7c7e08742c..8760258fba 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -73,6 +73,15 @@ public class SynchronizerOptions implements CLIOptions