mirror of https://github.com/hyperledger/besu
Healing Mechanism for Flat Database in Besu (#5319)
The proposed pull request introduces a feature that allows healing of the flat database by streaming the flat database data and validating it by generating a proof from the trie structure. If the proof is found to be invalid, the code traverses the trie to fix the invalid range. To optimize the process and avoid checking the entire flat database, the PR includes enhancements such as tracking the accounts that need to be repaired during SnapSync. By implementing these optimizations, the PR aims to significantly reduce the time and resources required for repairing the flat database. Signed-off-by: Karim TAAM <karim.t2am@gmail.com>pull/5601/head
matkt
1 year ago
committed by
GitHub
parent
ddacdc37c2
commit
180c75197c
@ -0,0 +1,174 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
* |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.bonsai.storage.flat; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.datatypes.StorageSlotKey; |
||||
import org.hyperledger.besu.ethereum.trie.NodeLoader; |
||||
import org.hyperledger.besu.metrics.BesuMetricCategory; |
||||
import org.hyperledger.besu.plugin.services.MetricsSystem; |
||||
import org.hyperledger.besu.plugin.services.metrics.Counter; |
||||
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; |
||||
|
||||
import java.util.Map; |
||||
import java.util.Optional; |
||||
import java.util.TreeMap; |
||||
import java.util.function.Supplier; |
||||
import java.util.stream.Collectors; |
||||
import java.util.stream.Stream; |
||||
|
||||
import kotlin.Pair; |
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.apache.tuweni.rlp.RLP; |
||||
|
||||
/** |
||||
* This class represents a FlatDbReaderStrategy, which is responsible for reading data from flat |
||||
* databases. It implements various methods for retrieving account data, code data, and storage data |
||||
* from the corresponding KeyValueStorage. |
||||
*/ |
||||
public abstract class FlatDbReaderStrategy { |
||||
|
||||
protected final MetricsSystem metricsSystem; |
||||
protected final Counter getAccountCounter; |
||||
protected final Counter getAccountFoundInFlatDatabaseCounter; |
||||
|
||||
protected final Counter getStorageValueCounter; |
||||
protected final Counter getStorageValueFlatDatabaseCounter; |
||||
|
||||
public FlatDbReaderStrategy(final MetricsSystem metricsSystem) { |
||||
this.metricsSystem = metricsSystem; |
||||
|
||||
getAccountCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_account_total", |
||||
"Total number of calls to getAccount"); |
||||
|
||||
getAccountFoundInFlatDatabaseCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_account_flat_database", |
||||
"Number of accounts found in the flat database"); |
||||
|
||||
getStorageValueCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_storagevalue_total", |
||||
"Total number of calls to getStorageValueBySlotHash"); |
||||
|
||||
getStorageValueFlatDatabaseCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_storagevalue_flat_database", |
||||
"Number of storage slots found in the flat database"); |
||||
} |
||||
|
||||
/* |
||||
* Retrieves the account data for the given account hash, using the world state root hash supplier and node loader. |
||||
*/ |
||||
public abstract Optional<Bytes> getAccount( |
||||
Supplier<Optional<Bytes>> worldStateRootHashSupplier, |
||||
NodeLoader nodeLoader, |
||||
Hash accountHash, |
||||
KeyValueStorage accountStorage); |
||||
|
||||
/* |
||||
* Retrieves the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader. |
||||
*/ |
||||
|
||||
public abstract Optional<Bytes> getStorageValueByStorageSlotKey( |
||||
Supplier<Optional<Bytes>> worldStateRootHashSupplier, |
||||
Supplier<Optional<Hash>> storageRootSupplier, |
||||
NodeLoader nodeLoader, |
||||
Hash accountHash, |
||||
StorageSlotKey storageSlotKey, |
||||
KeyValueStorage storageStorage); |
||||
|
||||
/* |
||||
* Retrieves the code data for the given code hash and account hash. |
||||
*/ |
||||
public Optional<Bytes> getCode( |
||||
final Bytes32 codeHash, final Hash accountHash, final KeyValueStorage codeStorage) { |
||||
if (codeHash.equals(Hash.EMPTY)) { |
||||
return Optional.of(Bytes.EMPTY); |
||||
} else { |
||||
return codeStorage |
||||
.get(accountHash.toArrayUnsafe()) |
||||
.map(Bytes::wrap) |
||||
.filter(b -> Hash.hash(b).equals(codeHash)); |
||||
} |
||||
} |
||||
|
||||
public void clearAll( |
||||
final KeyValueStorage accountStorage, |
||||
final KeyValueStorage storageStorage, |
||||
final KeyValueStorage codeStorage) { |
||||
accountStorage.clear(); |
||||
storageStorage.clear(); |
||||
codeStorage.clear(); |
||||
} |
||||
|
||||
public void resetOnResync( |
||||
final KeyValueStorage accountStorage, final KeyValueStorage storageStorage) { |
||||
accountStorage.clear(); |
||||
storageStorage.clear(); |
||||
} |
||||
|
||||
public Map<Bytes32, Bytes> streamAccountFlatDatabase( |
||||
final KeyValueStorage accountStorage, |
||||
final Bytes startKeyHash, |
||||
final Bytes32 endKeyHash, |
||||
final long max) { |
||||
final Stream<Pair<Bytes32, Bytes>> pairStream = |
||||
accountStorage |
||||
.streamFromKey(startKeyHash.toArrayUnsafe()) |
||||
.limit(max) |
||||
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue()))) |
||||
.takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0); |
||||
|
||||
final TreeMap<Bytes32, Bytes> collected = |
||||
pairStream.collect( |
||||
Collectors.toMap(Pair::getFirst, Pair::getSecond, (v1, v2) -> v1, TreeMap::new)); |
||||
pairStream.close(); |
||||
return collected; |
||||
} |
||||
|
||||
public Map<Bytes32, Bytes> streamStorageFlatDatabase( |
||||
final KeyValueStorage storageStorage, |
||||
final Hash accountHash, |
||||
final Bytes startKeyHash, |
||||
final Bytes32 endKeyHash, |
||||
final long max) { |
||||
final Stream<Pair<Bytes32, Bytes>> pairStream = |
||||
storageStorage |
||||
.streamFromKey(Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe()) |
||||
.takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash)) |
||||
.limit(max) |
||||
.map( |
||||
pair -> |
||||
new Pair<>( |
||||
Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)), |
||||
RLP.encodeValue(Bytes.wrap(pair.getValue()).trimLeadingZeros()))) |
||||
.takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0); |
||||
|
||||
final TreeMap<Bytes32, Bytes> collected = |
||||
pairStream.collect( |
||||
Collectors.toMap(Pair::getFirst, Pair::getSecond, (v1, v2) -> v1, TreeMap::new)); |
||||
pairStream.close(); |
||||
return collected; |
||||
} |
||||
} |
||||
@ -0,0 +1,98 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
* |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.bonsai.storage.flat; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.datatypes.StorageSlotKey; |
||||
import org.hyperledger.besu.ethereum.trie.NodeLoader; |
||||
import org.hyperledger.besu.metrics.BesuMetricCategory; |
||||
import org.hyperledger.besu.plugin.services.MetricsSystem; |
||||
import org.hyperledger.besu.plugin.services.metrics.Counter; |
||||
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; |
||||
|
||||
import java.util.Optional; |
||||
import java.util.function.Supplier; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes; |
||||
|
||||
public class FullFlatDbReaderStrategy extends FlatDbReaderStrategy { |
||||
|
||||
protected final Counter getAccountNotFoundInFlatDatabaseCounter; |
||||
|
||||
protected final Counter getStorageValueNotFoundInFlatDatabaseCounter; |
||||
|
||||
public FullFlatDbReaderStrategy(final MetricsSystem metricsSystem) { |
||||
super(metricsSystem); |
||||
|
||||
getAccountNotFoundInFlatDatabaseCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_account_missing_flat_database", |
||||
"Number of accounts not found in the flat database"); |
||||
|
||||
getStorageValueNotFoundInFlatDatabaseCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_storagevalue_missing_flat_database", |
||||
"Number of storage slots not found in the flat database"); |
||||
} |
||||
|
||||
@Override |
||||
public Optional<Bytes> getAccount( |
||||
final Supplier<Optional<Bytes>> worldStateRootHashSupplier, |
||||
final NodeLoader nodeLoader, |
||||
final Hash accountHash, |
||||
final KeyValueStorage accountStorage) { |
||||
getAccountCounter.inc(); |
||||
final Optional<Bytes> accountFound = |
||||
accountStorage.get(accountHash.toArrayUnsafe()).map(Bytes::wrap); |
||||
if (accountFound.isPresent()) { |
||||
getAccountFoundInFlatDatabaseCounter.inc(); |
||||
} else { |
||||
getAccountNotFoundInFlatDatabaseCounter.inc(); |
||||
} |
||||
return accountFound; |
||||
} |
||||
|
||||
@Override |
||||
public Optional<Bytes> getStorageValueByStorageSlotKey( |
||||
final Supplier<Optional<Bytes>> worldStateRootHashSupplier, |
||||
final Supplier<Optional<Hash>> storageRootSupplier, |
||||
final NodeLoader nodeLoader, |
||||
final Hash accountHash, |
||||
final StorageSlotKey storageSlotKey, |
||||
final KeyValueStorage storageStorage) { |
||||
getStorageValueCounter.inc(); |
||||
final Optional<Bytes> storageFound = |
||||
storageStorage |
||||
.get(Bytes.concatenate(accountHash, storageSlotKey.getSlotHash()).toArrayUnsafe()) |
||||
.map(Bytes::wrap); |
||||
if (storageFound.isPresent()) { |
||||
getStorageValueFlatDatabaseCounter.inc(); |
||||
} else { |
||||
getStorageValueNotFoundInFlatDatabaseCounter.inc(); |
||||
} |
||||
|
||||
return storageFound; |
||||
} |
||||
|
||||
@Override |
||||
public void resetOnResync( |
||||
final KeyValueStorage accountStorage, final KeyValueStorage storageStorage) { |
||||
// NOOP
|
||||
// not need to reset anything in full mode
|
||||
} |
||||
} |
||||
@ -0,0 +1,140 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
* |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.bonsai.storage.flat; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.datatypes.StorageSlotKey; |
||||
import org.hyperledger.besu.ethereum.trie.NodeLoader; |
||||
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; |
||||
import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; |
||||
import org.hyperledger.besu.metrics.BesuMetricCategory; |
||||
import org.hyperledger.besu.plugin.services.MetricsSystem; |
||||
import org.hyperledger.besu.plugin.services.metrics.Counter; |
||||
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; |
||||
|
||||
import java.util.Optional; |
||||
import java.util.function.Function; |
||||
import java.util.function.Supplier; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.apache.tuweni.rlp.RLP; |
||||
|
||||
/** |
||||
* This class represents a strategy for reading data from a partial flat database. It extends the |
||||
* FlatDbReaderStrategy and provides additional functionality for reading data from a merkle trie. |
||||
* If data is missing in the flat database, this strategy falls back to the merkle trie to retrieve |
||||
* the data. It adds a fallback mechanism for the `getAccount` and `getStorageValueByStorageSlotKey` |
||||
* methods, which checks if the data is present in the flat database, and if not, queries the merkle |
||||
* trie |
||||
*/ |
||||
public class PartialFlatDbReaderStrategy extends FlatDbReaderStrategy { |
||||
|
||||
protected final Counter getAccountMerkleTrieCounter; |
||||
protected final Counter getAccountMissingMerkleTrieCounter; |
||||
|
||||
protected final Counter getStorageValueMerkleTrieCounter; |
||||
protected final Counter getStorageValueMissingMerkleTrieCounter; |
||||
|
||||
public PartialFlatDbReaderStrategy(final MetricsSystem metricsSystem) { |
||||
super(metricsSystem); |
||||
getAccountMerkleTrieCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_account_merkle_trie", |
||||
"Number of accounts not found in the flat database, but found in the merkle trie"); |
||||
|
||||
getAccountMissingMerkleTrieCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_account_missing_merkle_trie", |
||||
"Number of accounts not found (either in the flat database or the merkle trie)"); |
||||
|
||||
getStorageValueMerkleTrieCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_storagevalue_merkle_trie", |
||||
"Number of storage slots not found in the flat database, but found in the merkle trie"); |
||||
|
||||
getStorageValueMissingMerkleTrieCounter = |
||||
metricsSystem.createCounter( |
||||
BesuMetricCategory.BLOCKCHAIN, |
||||
"get_storagevalue_missing_merkle_trie", |
||||
"Number of storage slots not found (either in the flat database or in the merkle trie)"); |
||||
} |
||||
|
||||
@Override |
||||
public Optional<Bytes> getAccount( |
||||
final Supplier<Optional<Bytes>> worldStateRootHashSupplier, |
||||
final NodeLoader nodeLoader, |
||||
final Hash accountHash, |
||||
final KeyValueStorage accountStorage) { |
||||
getAccountCounter.inc(); |
||||
Optional<Bytes> response = accountStorage.get(accountHash.toArrayUnsafe()).map(Bytes::wrap); |
||||
if (response.isEmpty()) { |
||||
// after a snapsync/fastsync we only have the trie branches.
|
||||
final Optional<Bytes> worldStateRootHash = worldStateRootHashSupplier.get(); |
||||
if (worldStateRootHash.isPresent()) { |
||||
response = |
||||
new StoredMerklePatriciaTrie<>( |
||||
new StoredNodeFactory<>(nodeLoader, Function.identity(), Function.identity()), |
||||
Bytes32.wrap(worldStateRootHash.get())) |
||||
.get(accountHash); |
||||
if (response.isEmpty()) { |
||||
getAccountMissingMerkleTrieCounter.inc(); |
||||
} else { |
||||
getAccountMerkleTrieCounter.inc(); |
||||
} |
||||
} |
||||
} else { |
||||
getAccountFoundInFlatDatabaseCounter.inc(); |
||||
} |
||||
|
||||
return response; |
||||
} |
||||
|
||||
@Override |
||||
public Optional<Bytes> getStorageValueByStorageSlotKey( |
||||
final Supplier<Optional<Bytes>> worldStateRootHashSupplier, |
||||
final Supplier<Optional<Hash>> storageRootSupplier, |
||||
final NodeLoader nodeLoader, |
||||
final Hash accountHash, |
||||
final StorageSlotKey storageSlotKey, |
||||
final KeyValueStorage storageStorage) { |
||||
getStorageValueCounter.inc(); |
||||
Optional<Bytes> response = |
||||
storageStorage |
||||
.get(Bytes.concatenate(accountHash, storageSlotKey.getSlotHash()).toArrayUnsafe()) |
||||
.map(Bytes::wrap); |
||||
if (response.isEmpty()) { |
||||
final Optional<Hash> storageRoot = storageRootSupplier.get(); |
||||
final Optional<Bytes> worldStateRootHash = worldStateRootHashSupplier.get(); |
||||
if (storageRoot.isPresent() && worldStateRootHash.isPresent()) { |
||||
response = |
||||
new StoredMerklePatriciaTrie<>( |
||||
new StoredNodeFactory<>(nodeLoader, Function.identity(), Function.identity()), |
||||
storageRoot.get()) |
||||
.get(storageSlotKey.getSlotHash()) |
||||
.map(bytes -> Bytes32.leftPad(RLP.decodeValue(bytes))); |
||||
if (response.isEmpty()) getStorageValueMissingMerkleTrieCounter.inc(); |
||||
else getStorageValueMerkleTrieCounter.inc(); |
||||
} |
||||
} else { |
||||
getStorageValueFlatDatabaseCounter.inc(); |
||||
} |
||||
return response; |
||||
} |
||||
} |
||||
@ -0,0 +1,57 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
* |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.worldstate; |
||||
|
||||
import java.util.stream.Stream; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes; |
||||
|
||||
/** |
||||
* The FlatDbMode enum represents the different modes of the flat database. It has two modes: |
||||
* PARTIAL and FULL. |
||||
* |
||||
* <p>- PARTIAL: Not all the leaves are present inside the flat database. The trie serves as a |
||||
* fallback to retrieve missing data. The PARTIAL mode is primarily used for backward compatibility |
||||
* purposes, where the flat database may not have all the required data, and the trie is utilized to |
||||
* fill in the gaps. |
||||
* |
||||
* <p>- FULL: The flat database contains the complete representation of the world state, and there |
||||
* is no need for a fallback mechanism. The FULL mode represents a fully synchronized state where |
||||
* the flat database encompasses all the necessary data. |
||||
*/ |
||||
public enum FlatDbMode { |
||||
NO_FLATTENED(Bytes.EMPTY), |
||||
PARTIAL(Bytes.of(0x00)), |
||||
FULL(Bytes.of(0x01)); |
||||
|
||||
final Bytes version; |
||||
|
||||
FlatDbMode(final Bytes version) { |
||||
this.version = version; |
||||
} |
||||
|
||||
public Bytes getVersion() { |
||||
return version; |
||||
} |
||||
|
||||
public static FlatDbMode fromVersion(final Bytes version) { |
||||
return Stream.of(FlatDbMode.values()) |
||||
.filter(mode -> mode.getVersion().equals(version)) |
||||
.findFirst() |
||||
.orElseThrow( |
||||
() -> new IllegalArgumentException("Unknown flat DB mode version: " + version)); |
||||
} |
||||
} |
||||
@ -0,0 +1,210 @@ |
||||
/* |
||||
* Copyright contributors to Hyperledger Besu |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; |
||||
|
||||
import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; |
||||
import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; |
||||
import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.HEAL_FLAT; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; |
||||
import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; |
||||
import org.hyperledger.besu.ethereum.rlp.RLP; |
||||
import org.hyperledger.besu.ethereum.trie.CompactEncoding; |
||||
import org.hyperledger.besu.ethereum.trie.MerkleTrie; |
||||
import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; |
||||
import org.hyperledger.besu.ethereum.trie.TrieIterator; |
||||
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; |
||||
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; |
||||
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; |
||||
|
||||
import java.math.BigInteger; |
||||
import java.util.ArrayList; |
||||
import java.util.Collection; |
||||
import java.util.List; |
||||
import java.util.TreeMap; |
||||
import java.util.function.Function; |
||||
import java.util.stream.Stream; |
||||
|
||||
import kotlin.collections.ArrayDeque; |
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
|
||||
/** |
||||
* The AccountFlatDatabaseHealingRangeRequest class represents a request to heal a range of account |
||||
* in the flat databases. It encapsulates the necessary information to identify the range and |
||||
* initiate the healing process. |
||||
*/ |
||||
public class AccountFlatDatabaseHealingRangeRequest extends SnapDataRequest { |
||||
|
||||
private final Bytes32 startKeyHash; |
||||
private final Bytes32 endKeyHash; |
||||
private TreeMap<Bytes32, Bytes> existingAccounts; |
||||
|
||||
private TreeMap<Bytes32, Bytes> removedAccounts; |
||||
private boolean isProofValid; |
||||
|
||||
public AccountFlatDatabaseHealingRangeRequest( |
||||
final Hash originalRootHash, final Bytes32 startKeyHash, final Bytes32 endKeyHash) { |
||||
super(RequestType.ACCOUNT_RANGE, originalRootHash); |
||||
this.startKeyHash = startKeyHash; |
||||
this.endKeyHash = endKeyHash; |
||||
this.existingAccounts = new TreeMap<>(); |
||||
this.removedAccounts = new TreeMap<>(); |
||||
this.isProofValid = false; |
||||
} |
||||
|
||||
@Override |
||||
public Stream<SnapDataRequest> getChildRequests( |
||||
final SnapWorldDownloadState downloadState, |
||||
final WorldStateStorage worldStateStorage, |
||||
final SnapSyncProcessState snapSyncState) { |
||||
final List<SnapDataRequest> childRequests = new ArrayList<>(); |
||||
if (!existingAccounts.isEmpty()) { |
||||
// new request is added if the response does not match all the requested range
|
||||
RangeManager.generateRanges( |
||||
existingAccounts.lastKey().toUnsignedBigInteger().add(BigInteger.ONE), |
||||
endKeyHash.toUnsignedBigInteger(), |
||||
1) |
||||
.forEach( |
||||
(key, value) -> { |
||||
downloadState.getMetricsManager().notifyRangeProgress(HEAL_FLAT, key, endKeyHash); |
||||
final AccountFlatDatabaseHealingRangeRequest storageRangeDataRequest = |
||||
createAccountFlatHealingRangeRequest(getRootHash(), key, value); |
||||
childRequests.add(storageRangeDataRequest); |
||||
}); |
||||
} else { |
||||
downloadState.getMetricsManager().notifyRangeProgress(HEAL_FLAT, endKeyHash, endKeyHash); |
||||
} |
||||
|
||||
Stream.of(existingAccounts.entrySet(), removedAccounts.entrySet()) |
||||
.flatMap(Collection::stream) |
||||
.forEach( |
||||
account -> { |
||||
if (downloadState |
||||
.getAccountsToBeRepaired() |
||||
.contains(CompactEncoding.bytesToPath(account.getKey()))) { |
||||
final StateTrieAccountValue accountValue = |
||||
StateTrieAccountValue.readFrom(RLP.input(account.getValue())); |
||||
childRequests.add( |
||||
createStorageFlatHealingRangeRequest( |
||||
getRootHash(), |
||||
account.getKey(), |
||||
accountValue.getStorageRoot(), |
||||
MIN_RANGE, |
||||
MAX_RANGE)); |
||||
} |
||||
}); |
||||
return childRequests.stream(); |
||||
} |
||||
|
||||
public Bytes32 getStartKeyHash() { |
||||
return startKeyHash; |
||||
} |
||||
|
||||
public Bytes32 getEndKeyHash() { |
||||
return endKeyHash; |
||||
} |
||||
|
||||
@Override |
||||
public boolean isResponseReceived() { |
||||
return true; |
||||
} |
||||
|
||||
public void addLocalData( |
||||
final WorldStateProofProvider worldStateProofProvider, |
||||
final TreeMap<Bytes32, Bytes> accounts, |
||||
final ArrayDeque<Bytes> proofs) { |
||||
if (!accounts.isEmpty() && !proofs.isEmpty()) { |
||||
// very proof in order to check if the local flat database is valid or not
|
||||
isProofValid = |
||||
worldStateProofProvider.isValidRangeProof( |
||||
startKeyHash, endKeyHash, getRootHash(), proofs, accounts); |
||||
this.existingAccounts = accounts; |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
protected int doPersist( |
||||
final WorldStateStorage worldStateStorage, |
||||
final WorldStateStorage.Updater updater, |
||||
final SnapWorldDownloadState downloadState, |
||||
final SnapSyncProcessState snapSyncState, |
||||
final SnapSyncConfiguration syncConfig) { |
||||
|
||||
if (!isProofValid) { // if proof is not valid we need to fix the flat database
|
||||
|
||||
final BonsaiWorldStateKeyValueStorage.Updater bonsaiUpdater = |
||||
(BonsaiWorldStateKeyValueStorage.Updater) updater; |
||||
|
||||
final MerkleTrie<Bytes, Bytes> accountTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
worldStateStorage::getAccountStateTrieNode, |
||||
getRootHash(), |
||||
Function.identity(), |
||||
Function.identity()); |
||||
|
||||
// retrieve the data from the trie in order to know what to fix in the flat database
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
startKeyHash, |
||||
existingAccounts.isEmpty() ? endKeyHash : existingAccounts.lastKey(), |
||||
existingAccounts.isEmpty() |
||||
? syncConfig.getLocalFlatAccountCountToHealPerRequest() |
||||
: Integer.MAX_VALUE, |
||||
Integer.MAX_VALUE); |
||||
|
||||
// put all flat accounts in the list, and gradually keep only those that are not in the trie
|
||||
// to remove and heal them.
|
||||
removedAccounts = new TreeMap<>(existingAccounts); |
||||
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
existingAccounts = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
accountTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, startKeyHash)); |
||||
|
||||
// doing the fix
|
||||
existingAccounts.forEach( |
||||
(key, value) -> { |
||||
if (removedAccounts.containsKey(key)) { |
||||
removedAccounts.remove(key); |
||||
} else { |
||||
final Hash accountHash = Hash.wrap(key); |
||||
// if the account was missing in the flat db we need to heal the storage
|
||||
downloadState.addAccountsToBeRepaired(CompactEncoding.bytesToPath(accountHash)); |
||||
bonsaiUpdater.putAccountInfoState(accountHash, value); |
||||
} |
||||
}); |
||||
|
||||
removedAccounts.forEach( |
||||
(key, value) -> { |
||||
final Hash accountHash = Hash.wrap(key); |
||||
// if the account was removed we will have to heal the storage
|
||||
downloadState.addAccountsToBeRepaired(CompactEncoding.bytesToPath(accountHash)); |
||||
bonsaiUpdater.removeAccountInfoState(accountHash); |
||||
}); |
||||
} |
||||
return existingAccounts.size() + removedAccounts.size(); |
||||
} |
||||
} |
||||
@ -0,0 +1,190 @@ |
||||
/* |
||||
* Copyright contributors to Hyperledger Besu |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; |
||||
|
||||
import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.getRangeCount; |
||||
import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.STORAGE_RANGE; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; |
||||
import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; |
||||
import org.hyperledger.besu.ethereum.trie.MerkleTrie; |
||||
import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; |
||||
import org.hyperledger.besu.ethereum.trie.TrieIterator; |
||||
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; |
||||
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; |
||||
|
||||
import java.math.BigInteger; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.TreeMap; |
||||
import java.util.function.Function; |
||||
import java.util.stream.Stream; |
||||
|
||||
import kotlin.collections.ArrayDeque; |
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.apache.tuweni.rlp.RLP; |
||||
|
||||
/** |
||||
* The StorageFlatDatabaseHealingRangeRequest class represents a request to heal a range of storage |
||||
* in the flat databases. It encapsulates the necessary information to identify the range and |
||||
* initiate the healing process. |
||||
*/ |
||||
public class StorageFlatDatabaseHealingRangeRequest extends SnapDataRequest { |
||||
|
||||
private final Hash accountHash; |
||||
private final Bytes32 storageRoot; |
||||
private final Bytes32 startKeyHash; |
||||
private final Bytes32 endKeyHash; |
||||
private TreeMap<Bytes32, Bytes> slots; |
||||
private boolean isProofValid; |
||||
|
||||
public StorageFlatDatabaseHealingRangeRequest( |
||||
final Hash rootHash, |
||||
final Bytes32 accountHash, |
||||
final Bytes32 storageRoot, |
||||
final Bytes32 startKeyHash, |
||||
final Bytes32 endKeyHash) { |
||||
super(STORAGE_RANGE, rootHash); |
||||
this.accountHash = Hash.wrap(accountHash); |
||||
this.storageRoot = storageRoot; |
||||
this.startKeyHash = startKeyHash; |
||||
this.endKeyHash = endKeyHash; |
||||
this.isProofValid = false; |
||||
} |
||||
|
||||
@Override |
||||
public Stream<SnapDataRequest> getChildRequests( |
||||
final SnapWorldDownloadState downloadState, |
||||
final WorldStateStorage worldStateStorage, |
||||
final SnapSyncProcessState snapSyncState) { |
||||
final List<SnapDataRequest> childRequests = new ArrayList<>(); |
||||
if (!slots.isEmpty()) { |
||||
// new request is added if the response does not match all the requested range
|
||||
final int nbRanges = getRangeCount(startKeyHash, endKeyHash, slots); |
||||
RangeManager.generateRanges( |
||||
slots.lastKey().toUnsignedBigInteger().add(BigInteger.ONE), |
||||
endKeyHash.toUnsignedBigInteger(), |
||||
nbRanges) |
||||
.forEach( |
||||
(key, value) -> { |
||||
final StorageFlatDatabaseHealingRangeRequest storageRangeDataRequest = |
||||
createStorageFlatHealingRangeRequest( |
||||
getRootHash(), accountHash, storageRoot, key, value); |
||||
childRequests.add(storageRangeDataRequest); |
||||
}); |
||||
} |
||||
return childRequests.stream(); |
||||
} |
||||
|
||||
public Hash getAccountHash() { |
||||
return accountHash; |
||||
} |
||||
|
||||
public Bytes32 getStorageRoot() { |
||||
return storageRoot; |
||||
} |
||||
|
||||
public Bytes32 getStartKeyHash() { |
||||
return startKeyHash; |
||||
} |
||||
|
||||
public Bytes32 getEndKeyHash() { |
||||
return endKeyHash; |
||||
} |
||||
|
||||
@Override |
||||
public boolean isResponseReceived() { |
||||
return true; |
||||
} |
||||
|
||||
public void addLocalData( |
||||
final WorldStateProofProvider worldStateProofProvider, |
||||
final TreeMap<Bytes32, Bytes> slots, |
||||
final ArrayDeque<Bytes> proofs) { |
||||
if (!slots.isEmpty() && !proofs.isEmpty()) { |
||||
// very proof in order to check if the local flat database is valid or not
|
||||
isProofValid = |
||||
worldStateProofProvider.isValidRangeProof( |
||||
startKeyHash, endKeyHash, storageRoot, proofs, slots); |
||||
} |
||||
this.slots = slots; |
||||
} |
||||
|
||||
@Override |
||||
protected int doPersist( |
||||
final WorldStateStorage worldStateStorage, |
||||
final WorldStateStorage.Updater updater, |
||||
final SnapWorldDownloadState downloadState, |
||||
final SnapSyncProcessState snapSyncState, |
||||
final SnapSyncConfiguration snapSyncConfiguration) { |
||||
|
||||
if (!isProofValid) { |
||||
// If the proof is not valid, it indicates that the flat database needs to be fixed.
|
||||
|
||||
final BonsaiWorldStateKeyValueStorage.Updater bonsaiUpdater = |
||||
(BonsaiWorldStateKeyValueStorage.Updater) updater; |
||||
|
||||
final MerkleTrie<Bytes, Bytes> storageTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
(location, hash) -> |
||||
worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), |
||||
storageRoot, |
||||
Function.identity(), |
||||
Function.identity()); |
||||
|
||||
Map<Bytes32, Bytes> remainingKeys = new TreeMap<>(slots); |
||||
|
||||
// Retrieve the data from the trie in order to know what needs to be fixed in the flat
|
||||
// database
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
startKeyHash, |
||||
slots.isEmpty() ? endKeyHash : slots.lastKey(), |
||||
slots.isEmpty() |
||||
? snapSyncConfiguration.getLocalFlatStorageCountToHealPerRequest() |
||||
: Integer.MAX_VALUE, |
||||
Integer.MAX_VALUE); |
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
slots = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
storageTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, startKeyHash)); |
||||
|
||||
// Perform the fix by updating the flat database
|
||||
slots.forEach( |
||||
(key, value) -> { |
||||
if (remainingKeys.containsKey(key)) { |
||||
remainingKeys.remove(key); |
||||
} else { |
||||
bonsaiUpdater.putStorageValueBySlotHash( |
||||
accountHash, Hash.wrap(key), Bytes32.leftPad(RLP.decodeValue(value))); |
||||
} |
||||
}); |
||||
remainingKeys.forEach( |
||||
(key, value) -> bonsaiUpdater.removeStorageValueBySlotHash(accountHash, Hash.wrap(key))); |
||||
} |
||||
return slots.size(); |
||||
} |
||||
} |
||||
@ -0,0 +1,297 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; |
||||
import org.hyperledger.besu.ethereum.core.TrieGenerator; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; |
||||
import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; |
||||
import org.hyperledger.besu.ethereum.storage.StorageProvider; |
||||
import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.trie.CompactEncoding; |
||||
import org.hyperledger.besu.ethereum.trie.MerkleTrie; |
||||
import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; |
||||
import org.hyperledger.besu.ethereum.trie.TrieIterator; |
||||
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; |
||||
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; |
||||
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; |
||||
|
||||
import java.util.HashSet; |
||||
import java.util.Iterator; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.TreeMap; |
||||
import java.util.stream.Collectors; |
||||
import java.util.stream.Stream; |
||||
|
||||
import kotlin.collections.ArrayDeque; |
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.assertj.core.api.Assertions; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.MockitoJUnitRunner; |
||||
|
||||
@RunWith(MockitoJUnitRunner.class) |
||||
public class AccountFlatDatabaseHealingRangeRequestTest { |
||||
|
||||
@Mock private SnapWorldDownloadState downloadState; |
||||
@Mock private SnapSyncProcessState snapSyncState; |
||||
|
||||
@Before |
||||
public void setup() { |
||||
Mockito.when(downloadState.getMetricsManager()) |
||||
.thenReturn(Mockito.mock(SnapsyncMetricsManager.class)); |
||||
Mockito.when(downloadState.getAccountsToBeRepaired()).thenReturn(new HashSet<>()); |
||||
} |
||||
|
||||
@Test |
||||
public void shouldReturnChildRequests() { |
||||
final WorldStateStorage worldStateStorage = |
||||
new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); |
||||
final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); |
||||
final MerkleTrie<Bytes, Bytes> accountStateTrie = |
||||
TrieGenerator.generateTrie(worldStateStorage, 15); |
||||
|
||||
// Create a collector to gather account entries within a specific range
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
Hash.ZERO, RangeManager.MAX_RANGE, 10, Integer.MAX_VALUE); |
||||
|
||||
// Create a visitor for the range collector
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
|
||||
// Collect the account entries within the specified range using the trie and range collector
|
||||
final TreeMap<Bytes32, Bytes> accounts = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
accountStateTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, Hash.ZERO)); |
||||
|
||||
// Retrieve the proof related nodes for the account trie
|
||||
final List<Bytes> proofs = |
||||
proofProvider.getAccountProofRelatedNodes( |
||||
Hash.wrap(accountStateTrie.getRootHash()), Hash.ZERO); |
||||
proofs.addAll( |
||||
proofProvider.getAccountProofRelatedNodes( |
||||
Hash.wrap(accountStateTrie.getRootHash()), accounts.lastKey())); |
||||
|
||||
// Create a request for healing the flat database with a range from MIN_RANGE to MAX_RANGE
|
||||
final AccountFlatDatabaseHealingRangeRequest request = |
||||
new AccountFlatDatabaseHealingRangeRequest( |
||||
Hash.EMPTY, RangeManager.MIN_RANGE, RangeManager.MAX_RANGE); |
||||
// Add local data to the request, including the proof provider, accounts TreeMap, and proofs as
|
||||
// an ArrayDeque
|
||||
request.addLocalData(proofProvider, accounts, new ArrayDeque<>(proofs)); |
||||
|
||||
// Verify that the start key hash of the snapDataRequest is greater than the last key in the
|
||||
// accounts TreeMap
|
||||
List<SnapDataRequest> childRequests = |
||||
request.getChildRequests(downloadState, worldStateStorage, snapSyncState).toList(); |
||||
Assertions.assertThat(childRequests).hasSize(1); |
||||
AccountFlatDatabaseHealingRangeRequest snapDataRequest = |
||||
(AccountFlatDatabaseHealingRangeRequest) childRequests.get(0); |
||||
Assertions.assertThat(snapDataRequest.getStartKeyHash()).isGreaterThan(accounts.lastKey()); |
||||
|
||||
// Verify that we have storage healing request when the account need to be repaired
|
||||
Mockito.when(downloadState.getAccountsToBeRepaired()) |
||||
.thenReturn( |
||||
new HashSet<>( |
||||
accounts.keySet().stream() |
||||
.map(CompactEncoding::bytesToPath) |
||||
.collect(Collectors.toList()))); |
||||
childRequests = |
||||
request.getChildRequests(downloadState, worldStateStorage, snapSyncState).toList(); |
||||
Assertions.assertThat(childRequests).hasSizeGreaterThan(1); |
||||
Assertions.assertThat(childRequests) |
||||
.hasAtLeastOneElementOfType(AccountFlatDatabaseHealingRangeRequest.class); |
||||
Assertions.assertThat(childRequests) |
||||
.hasAtLeastOneElementOfType(StorageFlatDatabaseHealingRangeRequest.class); |
||||
} |
||||
|
||||
@Test |
||||
public void shouldNotReturnChildRequestsWhenNoMoreAccounts() { |
||||
final WorldStateStorage worldStateStorage = |
||||
new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); |
||||
final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); |
||||
final MerkleTrie<Bytes, Bytes> accountStateTrie = |
||||
TrieGenerator.generateTrie(worldStateStorage, 15); |
||||
|
||||
// Create a collector to gather account entries within a specific range
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
Hash.ZERO, RangeManager.MAX_RANGE, 15, Integer.MAX_VALUE); |
||||
|
||||
// Create a visitor for the range collector
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
|
||||
// Collect the account entries within the specified range using the trie and range collector
|
||||
final TreeMap<Bytes32, Bytes> accounts = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
accountStateTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, Hash.ZERO)); |
||||
|
||||
// Create a request for healing the flat database with no more accounts
|
||||
final AccountFlatDatabaseHealingRangeRequest request = |
||||
new AccountFlatDatabaseHealingRangeRequest( |
||||
Hash.EMPTY, accounts.lastKey(), RangeManager.MAX_RANGE); |
||||
|
||||
// Add local data to the request
|
||||
request.addLocalData(proofProvider, new TreeMap<>(), new ArrayDeque<>()); |
||||
|
||||
// Verify that no child requests are returned from the request
|
||||
final Stream<SnapDataRequest> childRequests = |
||||
request.getChildRequests(downloadState, worldStateStorage, snapSyncState); |
||||
Assertions.assertThat(childRequests).isEmpty(); |
||||
} |
||||
|
||||
@Test |
||||
public void doNotPersistWhenProofIsValid() { |
||||
|
||||
final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); |
||||
|
||||
final WorldStateStorage worldStateStorage = |
||||
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); |
||||
final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); |
||||
final MerkleTrie<Bytes, Bytes> accountStateTrie = |
||||
TrieGenerator.generateTrie(worldStateStorage, 15); |
||||
// Create a collector to gather account entries within a specific range
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
Hash.ZERO, RangeManager.MAX_RANGE, 10, Integer.MAX_VALUE); |
||||
|
||||
// Create a visitor for the range collector
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
|
||||
// Collect the account entries within the specified range using the trie and range collector
|
||||
final TreeMap<Bytes32, Bytes> accounts = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
accountStateTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, Hash.ZERO)); |
||||
|
||||
// Retrieve the proof related nodes for the account trie
|
||||
final List<Bytes> proofs = |
||||
proofProvider.getAccountProofRelatedNodes( |
||||
Hash.wrap(accountStateTrie.getRootHash()), Hash.ZERO); |
||||
proofs.addAll( |
||||
proofProvider.getAccountProofRelatedNodes( |
||||
Hash.wrap(accountStateTrie.getRootHash()), accounts.lastKey())); |
||||
|
||||
// Create a request for healing the flat database with a range from MIN_RANGE to MAX_RANGE
|
||||
final AccountFlatDatabaseHealingRangeRequest request = |
||||
new AccountFlatDatabaseHealingRangeRequest( |
||||
Hash.wrap(accountStateTrie.getRootHash()), |
||||
RangeManager.MIN_RANGE, |
||||
RangeManager.MAX_RANGE); |
||||
// Add local data to the request, including the proof provider, accounts TreeMap, and proofs as
|
||||
// an ArrayDeque
|
||||
request.addLocalData(proofProvider, accounts, new ArrayDeque<>(proofs)); |
||||
|
||||
WorldStateStorage.Updater updater = Mockito.spy(worldStateStorage.updater()); |
||||
request.doPersist( |
||||
worldStateStorage, |
||||
updater, |
||||
downloadState, |
||||
snapSyncState, |
||||
SnapSyncConfiguration.getDefault()); |
||||
Mockito.verifyNoInteractions(updater); |
||||
} |
||||
|
||||
@Test |
||||
public void doHealAndPersistWhenProofIsInvalid() { |
||||
|
||||
final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); |
||||
|
||||
final WorldStateStorage worldStateStorage = |
||||
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); |
||||
final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); |
||||
final MerkleTrie<Bytes, Bytes> accountStateTrie = |
||||
TrieGenerator.generateTrie(worldStateStorage, 15); |
||||
// Create a collector to gather account entries within a specific range
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
Hash.ZERO, RangeManager.MAX_RANGE, 15, Integer.MAX_VALUE); |
||||
|
||||
// Create a visitor for the range collector
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
|
||||
// Collect the account entries within the specified range using the trie and range collector
|
||||
final TreeMap<Bytes32, Bytes> accounts = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
accountStateTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, Hash.ZERO)); |
||||
|
||||
// Retrieve the proof related nodes for the account trie
|
||||
final List<Bytes> proofs = |
||||
proofProvider.getAccountProofRelatedNodes( |
||||
Hash.wrap(accountStateTrie.getRootHash()), Hash.ZERO); |
||||
proofs.addAll( |
||||
proofProvider.getAccountProofRelatedNodes( |
||||
Hash.wrap(accountStateTrie.getRootHash()), accounts.lastKey())); |
||||
|
||||
// Remove an account in the middle of the range
|
||||
final Iterator<Map.Entry<Bytes32, Bytes>> iterator = accounts.entrySet().iterator(); |
||||
Map.Entry<Bytes32, Bytes> removedAccount = null; |
||||
int i = 0; |
||||
while (iterator.hasNext()) { |
||||
if (i == 7) { |
||||
removedAccount = Map.Entry.copyOf(iterator.next()); |
||||
iterator.remove(); |
||||
} else { |
||||
iterator.next(); |
||||
} |
||||
i++; |
||||
} |
||||
|
||||
// Create a request for healing the flat database with a range from MIN_RANGE to MAX_RANGE
|
||||
final AccountFlatDatabaseHealingRangeRequest request = |
||||
new AccountFlatDatabaseHealingRangeRequest( |
||||
Hash.wrap(accountStateTrie.getRootHash()), |
||||
RangeManager.MIN_RANGE, |
||||
RangeManager.MAX_RANGE); |
||||
// Add local data to the request, including the proof provider, accounts TreeMap, and proofs as
|
||||
// an ArrayDeque
|
||||
request.addLocalData(proofProvider, accounts, new ArrayDeque<>(proofs)); |
||||
|
||||
BonsaiWorldStateKeyValueStorage.Updater updater = |
||||
(BonsaiWorldStateKeyValueStorage.Updater) Mockito.spy(worldStateStorage.updater()); |
||||
request.doPersist( |
||||
worldStateStorage, |
||||
updater, |
||||
downloadState, |
||||
snapSyncState, |
||||
SnapSyncConfiguration.getDefault()); |
||||
// check add the missing account to the updater
|
||||
Mockito.verify(updater) |
||||
.putAccountInfoState(Hash.wrap(removedAccount.getKey()), removedAccount.getValue()); |
||||
} |
||||
} |
||||
@ -0,0 +1,325 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; |
||||
|
||||
import static org.apache.tuweni.rlp.RLP.decodeValue; |
||||
|
||||
import org.hyperledger.besu.datatypes.Address; |
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; |
||||
import org.hyperledger.besu.ethereum.core.TrieGenerator; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; |
||||
import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; |
||||
import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; |
||||
import org.hyperledger.besu.ethereum.rlp.RLP; |
||||
import org.hyperledger.besu.ethereum.storage.StorageProvider; |
||||
import org.hyperledger.besu.ethereum.trie.MerkleTrie; |
||||
import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; |
||||
import org.hyperledger.besu.ethereum.trie.TrieIterator; |
||||
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; |
||||
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; |
||||
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; |
||||
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; |
||||
|
||||
import java.util.Iterator; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.TreeMap; |
||||
import java.util.stream.Collectors; |
||||
import java.util.stream.Stream; |
||||
|
||||
import kotlin.collections.ArrayDeque; |
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.assertj.core.api.Assertions; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.MockitoJUnitRunner; |
||||
|
||||
@RunWith(MockitoJUnitRunner.class) |
||||
public class StorageFlatDatabaseHealingRangeRequestTest { |
||||
|
||||
@Mock private SnapWorldDownloadState downloadState; |
||||
@Mock private SnapSyncProcessState snapSyncState; |
||||
|
||||
final List<Address> accounts = |
||||
List.of( |
||||
Address.fromHexString("0xdeadbeef"), |
||||
Address.fromHexString("0xdeadbeee"), |
||||
Address.fromHexString("0xdeadbeea"), |
||||
Address.fromHexString("0xdeadbeeb")); |
||||
|
||||
private MerkleTrie<Bytes, Bytes> trie; |
||||
private BonsaiWorldStateKeyValueStorage worldStateStorage; |
||||
private WorldStateProofProvider proofProvider; |
||||
private Hash account0Hash; |
||||
private Hash account0StorageRoot; |
||||
|
||||
@Before |
||||
public void setup() { |
||||
final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); |
||||
worldStateStorage = |
||||
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); |
||||
proofProvider = new WorldStateProofProvider(worldStateStorage); |
||||
trie = |
||||
TrieGenerator.generateTrie( |
||||
worldStateStorage, accounts.stream().map(Hash::hash).collect(Collectors.toList())); |
||||
account0Hash = Hash.hash(accounts.get(0)); |
||||
account0StorageRoot = |
||||
trie.get(account0Hash) |
||||
.map(RLP::input) |
||||
.map(StateTrieAccountValue::readFrom) |
||||
.map(StateTrieAccountValue::getStorageRoot) |
||||
.orElseThrow(); |
||||
} |
||||
|
||||
@Test |
||||
public void shouldReturnChildRequests() { |
||||
|
||||
final StoredMerklePatriciaTrie<Bytes, Bytes> storageTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
(location, hash) -> |
||||
worldStateStorage.getAccountStorageTrieNode(account0Hash, location, hash), |
||||
account0StorageRoot, |
||||
b -> b, |
||||
b -> b); |
||||
|
||||
// Create a collector to gather slot entries within a specific range
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
Hash.ZERO, RangeManager.MAX_RANGE, 1, Integer.MAX_VALUE); |
||||
|
||||
// Create a visitor for the range collector
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
|
||||
// Collect the slot entries within the specified range using the trie and range collector
|
||||
final TreeMap<Bytes32, Bytes> slots = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
storageTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, Hash.ZERO)); |
||||
|
||||
// Retrieve the proof related nodes for the account trie
|
||||
final List<Bytes> proofs = |
||||
proofProvider.getStorageProofRelatedNodes( |
||||
Hash.wrap(storageTrie.getRootHash()), account0Hash, slots.firstKey()); |
||||
proofs.addAll( |
||||
proofProvider.getStorageProofRelatedNodes( |
||||
Hash.wrap(storageTrie.getRootHash()), account0Hash, slots.lastKey())); |
||||
|
||||
// Create a request for healing the flat database with a range from MIN_RANGE to MAX_RANGE
|
||||
final StorageFlatDatabaseHealingRangeRequest request = |
||||
new StorageFlatDatabaseHealingRangeRequest( |
||||
Hash.EMPTY, |
||||
account0Hash, |
||||
account0StorageRoot, |
||||
RangeManager.MIN_RANGE, |
||||
RangeManager.MAX_RANGE); |
||||
// Add local data to the request, including the proof provider, accounts TreeMap, and proofs as
|
||||
// an ArrayDeque
|
||||
request.addLocalData(proofProvider, slots, new ArrayDeque<>(proofs)); |
||||
|
||||
// Verify that the start key hash of the snapDataRequest is greater than the last key in the
|
||||
// slots TreeMap
|
||||
List<SnapDataRequest> childRequests = |
||||
request.getChildRequests(downloadState, worldStateStorage, snapSyncState).toList(); |
||||
Assertions.assertThat(childRequests).hasSizeGreaterThan(1); |
||||
StorageFlatDatabaseHealingRangeRequest snapDataRequest = |
||||
(StorageFlatDatabaseHealingRangeRequest) childRequests.get(0); |
||||
Assertions.assertThat(snapDataRequest.getStartKeyHash()).isGreaterThan(slots.lastKey()); |
||||
} |
||||
|
||||
@Test |
||||
public void shouldNotReturnChildRequestsWhenNoMoreSlots() { |
||||
|
||||
final StoredMerklePatriciaTrie<Bytes, Bytes> storageTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
(location, hash) -> |
||||
worldStateStorage.getAccountStorageTrieNode(account0Hash, location, hash), |
||||
account0StorageRoot, |
||||
b -> b, |
||||
b -> b); |
||||
|
||||
// Create a collector to gather slot entries within a specific range
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
Hash.ZERO, RangeManager.MAX_RANGE, Integer.MAX_VALUE, Integer.MAX_VALUE); |
||||
|
||||
// Create a visitor for the range collector
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
|
||||
// Collect the slots entries within the specified range using the trie and range collector
|
||||
final TreeMap<Bytes32, Bytes> slots = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
storageTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, Hash.ZERO)); |
||||
|
||||
// Create a request for healing the flat database with no more slots
|
||||
final StorageFlatDatabaseHealingRangeRequest request = |
||||
new StorageFlatDatabaseHealingRangeRequest( |
||||
Hash.EMPTY, account0Hash, account0StorageRoot, slots.lastKey(), RangeManager.MAX_RANGE); |
||||
|
||||
// Add local data to the request
|
||||
request.addLocalData(proofProvider, new TreeMap<>(), new ArrayDeque<>()); |
||||
|
||||
// Verify that no child requests are returned from the request
|
||||
final Stream<SnapDataRequest> childRequests = |
||||
request.getChildRequests(downloadState, worldStateStorage, snapSyncState); |
||||
Assertions.assertThat(childRequests).isEmpty(); |
||||
} |
||||
|
||||
@Test |
||||
public void doNotPersistWhenProofIsValid() { |
||||
|
||||
final StoredMerklePatriciaTrie<Bytes, Bytes> storageTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
(location, hash) -> |
||||
worldStateStorage.getAccountStorageTrieNode(account0Hash, location, hash), |
||||
account0StorageRoot, |
||||
b -> b, |
||||
b -> b); |
||||
|
||||
// Create a collector to gather slots entries within a specific range
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
Hash.ZERO, RangeManager.MAX_RANGE, Integer.MAX_VALUE, Integer.MAX_VALUE); |
||||
|
||||
// Create a visitor for the range collector
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
|
||||
// Collect the slot entries within the specified range using the trie and range collector
|
||||
final TreeMap<Bytes32, Bytes> slots = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
storageTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, Hash.ZERO)); |
||||
|
||||
// Retrieve the proof related nodes for the account trie
|
||||
final List<Bytes> proofs = |
||||
proofProvider.getStorageProofRelatedNodes( |
||||
Hash.wrap(storageTrie.getRootHash()), account0Hash, slots.firstKey()); |
||||
proofs.addAll( |
||||
proofProvider.getStorageProofRelatedNodes( |
||||
Hash.wrap(storageTrie.getRootHash()), account0Hash, slots.lastKey())); |
||||
|
||||
// Create a request for healing the flat database with a range from MIN_RANGE to MAX_RANGE
|
||||
final StorageFlatDatabaseHealingRangeRequest request = |
||||
new StorageFlatDatabaseHealingRangeRequest( |
||||
Hash.wrap(trie.getRootHash()), |
||||
account0Hash, |
||||
Hash.wrap(storageTrie.getRootHash()), |
||||
RangeManager.MIN_RANGE, |
||||
RangeManager.MAX_RANGE); |
||||
|
||||
// Add local data to the request
|
||||
request.addLocalData(proofProvider, slots, new ArrayDeque<>(proofs)); |
||||
|
||||
WorldStateStorage.Updater updater = Mockito.spy(worldStateStorage.updater()); |
||||
request.doPersist( |
||||
worldStateStorage, |
||||
updater, |
||||
downloadState, |
||||
snapSyncState, |
||||
SnapSyncConfiguration.getDefault()); |
||||
Mockito.verifyNoInteractions(updater); |
||||
} |
||||
|
||||
@Test |
||||
public void doHealAndPersistWhenProofIsInvalid() { |
||||
|
||||
final StoredMerklePatriciaTrie<Bytes, Bytes> storageTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
(location, hash) -> |
||||
worldStateStorage.getAccountStorageTrieNode(account0Hash, location, hash), |
||||
account0StorageRoot, |
||||
b -> b, |
||||
b -> b); |
||||
|
||||
// Create a collector to gather slots entries within a specific range
|
||||
final RangeStorageEntriesCollector collector = |
||||
RangeStorageEntriesCollector.createCollector( |
||||
Hash.ZERO, RangeManager.MAX_RANGE, Integer.MAX_VALUE, Integer.MAX_VALUE); |
||||
|
||||
// Create a visitor for the range collector
|
||||
final TrieIterator<Bytes> visitor = RangeStorageEntriesCollector.createVisitor(collector); |
||||
|
||||
// Collect the slot entries within the specified range using the trie and range collector
|
||||
final TreeMap<Bytes32, Bytes> slots = |
||||
(TreeMap<Bytes32, Bytes>) |
||||
storageTrie.entriesFrom( |
||||
root -> |
||||
RangeStorageEntriesCollector.collectEntries( |
||||
collector, visitor, root, Hash.ZERO)); |
||||
|
||||
// Retrieve the proof related nodes for the account trie
|
||||
final List<Bytes> proofs = |
||||
proofProvider.getStorageProofRelatedNodes( |
||||
Hash.wrap(storageTrie.getRootHash()), account0Hash, slots.firstKey()); |
||||
proofs.addAll( |
||||
proofProvider.getStorageProofRelatedNodes( |
||||
Hash.wrap(storageTrie.getRootHash()), account0Hash, slots.lastKey())); |
||||
|
||||
// Remove a slot in the middle of the range
|
||||
final Iterator<Map.Entry<Bytes32, Bytes>> iterator = slots.entrySet().iterator(); |
||||
Map.Entry<Bytes32, Bytes> removedSlot = null; |
||||
int i = 0; |
||||
while (iterator.hasNext()) { |
||||
if (i == 1) { |
||||
removedSlot = Map.Entry.copyOf(iterator.next()); |
||||
iterator.remove(); |
||||
} else { |
||||
iterator.next(); |
||||
} |
||||
i++; |
||||
} |
||||
|
||||
// Create a request for healing the flat database with a range from MIN_RANGE to MAX_RANGE
|
||||
final StorageFlatDatabaseHealingRangeRequest request = |
||||
new StorageFlatDatabaseHealingRangeRequest( |
||||
Hash.wrap(trie.getRootHash()), |
||||
account0Hash, |
||||
Hash.wrap(storageTrie.getRootHash()), |
||||
RangeManager.MIN_RANGE, |
||||
RangeManager.MAX_RANGE); |
||||
// Add local data to the request
|
||||
request.addLocalData(proofProvider, slots, new ArrayDeque<>(proofs)); |
||||
|
||||
BonsaiWorldStateKeyValueStorage.Updater updater = |
||||
(BonsaiWorldStateKeyValueStorage.Updater) Mockito.spy(worldStateStorage.updater()); |
||||
request.doPersist( |
||||
worldStateStorage, |
||||
updater, |
||||
downloadState, |
||||
snapSyncState, |
||||
SnapSyncConfiguration.getDefault()); |
||||
// check add the missing slot to the updater
|
||||
Mockito.verify(updater) |
||||
.putStorageValueBySlotHash( |
||||
account0Hash, |
||||
Hash.wrap(removedSlot.getKey()), |
||||
Bytes32.leftPad(decodeValue(removedSlot.getValue()))); |
||||
} |
||||
} |
||||
@ -0,0 +1,85 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.ethereum.trie; |
||||
|
||||
import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; |
||||
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; |
||||
|
||||
import java.util.List; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.assertj.core.api.Assertions; |
||||
import org.junit.Test; |
||||
|
||||
public class RangeStorageEntriesCollectorTest { |
||||
|
||||
@Test |
||||
public void shouldRetrieveAllLeavesInRangeWhenStartFromZero() { |
||||
InMemoryKeyValueStorage worldStateStorage = new InMemoryKeyValueStorage(); |
||||
final MerkleTrie<Bytes, Bytes> accountStateTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
(location, hash) -> worldStateStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), |
||||
b -> b, |
||||
b -> b); |
||||
final List<Bytes32> lists = |
||||
List.of( |
||||
Bytes32.rightPad(Bytes.of(1, 1, 3, 0)), |
||||
Bytes32.rightPad(Bytes.of(1, 1, 3, 1)), |
||||
Bytes32.rightPad(Bytes.of(1, 2, 0, 0))); |
||||
lists.forEach(bytes -> accountStateTrie.put(bytes, Bytes.of(1, 2, 3))); |
||||
Assertions.assertThat( |
||||
accountStateTrie.entriesFrom(Bytes32.rightPad(Bytes.of(0, 0, 0, 0)), 3).keySet()) |
||||
.containsAll(lists); |
||||
} |
||||
|
||||
@Test |
||||
public void shouldRetrieveAllLeavesInRangeWhenStartFromSpecificRange() { |
||||
InMemoryKeyValueStorage worldStateStorage = new InMemoryKeyValueStorage(); |
||||
final MerkleTrie<Bytes, Bytes> accountStateTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
(location, hash) -> worldStateStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), |
||||
b -> b, |
||||
b -> b); |
||||
final List<Bytes32> lists = |
||||
List.of( |
||||
Bytes32.rightPad(Bytes.of(1, 1, 3, 0)), |
||||
Bytes32.rightPad(Bytes.of(1, 1, 3, 1)), |
||||
Bytes32.rightPad(Bytes.of(1, 2, 0, 0))); |
||||
lists.forEach(bytes -> accountStateTrie.put(bytes, Bytes.of(1, 2, 3))); |
||||
Assertions.assertThat( |
||||
accountStateTrie.entriesFrom(Bytes32.rightPad(Bytes.of(1, 1, 2, 1)), 3).keySet()) |
||||
.containsAll(lists); |
||||
} |
||||
|
||||
@Test |
||||
public void shouldExcludeLeavesNotInRange() { |
||||
InMemoryKeyValueStorage worldStateStorage = new InMemoryKeyValueStorage(); |
||||
final MerkleTrie<Bytes, Bytes> accountStateTrie = |
||||
new StoredMerklePatriciaTrie<>( |
||||
(location, hash) -> worldStateStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), |
||||
b -> b, |
||||
b -> b); |
||||
final List<Bytes32> lists = |
||||
List.of( |
||||
Bytes32.rightPad(Bytes.of(1, 1, 3, 0)), |
||||
Bytes32.rightPad(Bytes.of(1, 1, 3, 1)), |
||||
Bytes32.rightPad(Bytes.of(1, 2, 0, 0))); |
||||
lists.forEach(bytes -> accountStateTrie.put(bytes, Bytes.of(1, 2, 3))); |
||||
Assertions.assertThat( |
||||
accountStateTrie.entriesFrom(Bytes32.rightPad(Bytes.of(1, 1, 9, 9)), 1).keySet()) |
||||
.contains(Bytes32.rightPad(Bytes.of(1, 2, 0, 0))); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue