diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java new file mode 100644 index 0000000000..ea44362f0e --- /dev/null +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java @@ -0,0 +1,95 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; +import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +public class TrieGenerator { + + public static MerklePatriciaTrie generateTrie( + final WorldStateStorage worldStateStorage, final int nbAccounts) { + final List accountHash = new ArrayList<>(); + final MerklePatriciaTrie accountStateTrie = + emptyAccountStateTrie(worldStateStorage); + // Add some storage values + for (int i = 0; i < nbAccounts; i++) { + final WorldStateStorage.Updater updater = worldStateStorage.updater(); + + accountHash.add(Hash.wrap(Bytes32.leftPad(Bytes.of(i + 1)))); + final MerklePatriciaTrie storageTrie = + emptyStorageTrie(worldStateStorage, accountHash.get(i)); + writeStorageValue(storageTrie, UInt256.ONE, UInt256.valueOf(2L)); + writeStorageValue(storageTrie, UInt256.valueOf(2L), UInt256.valueOf(4L)); + writeStorageValue(storageTrie, UInt256.valueOf(3L), UInt256.valueOf(6L)); + int accountIndex = i; + storageTrie.commit( + (location, hash, value) -> + updater.putAccountStorageTrieNode( + accountHash.get(accountIndex), location, hash, value)); + final Hash codeHash = Hash.hash(Bytes32.leftPad(Bytes.of(i + 10))); + final StateTrieAccountValue accountValue = + new StateTrieAccountValue(1L, Wei.of(2L), Hash.wrap(storageTrie.getRootHash()), codeHash); + accountStateTrie.put(accountHash.get(i), RLP.encode(accountValue::writeTo)); + accountStateTrie.commit(updater::putAccountStateTrieNode); + + // Persist updates + updater.commit(); + } + return accountStateTrie; + } + + private static void writeStorageValue( + final MerklePatriciaTrie storageTrie, + final UInt256 key, + final UInt256 value) { + storageTrie.put(storageKeyHash(key), encodeStorageValue(value)); + } + + private static Bytes32 storageKeyHash(final UInt256 storageKey) { + return Hash.hash(storageKey); + } + + private static Bytes encodeStorageValue(final UInt256 storageValue) { + return RLP.encode(out -> out.writeBytes(storageValue.toMinimalBytes())); + } + + public static MerklePatriciaTrie emptyStorageTrie( + final WorldStateStorage worldStateStorage, final Hash accountHash) { + return new StoredMerklePatriciaTrie<>( + (location, hash) -> + worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), + b -> b, + b -> b); + } + + public static MerklePatriciaTrie emptyAccountStateTrie( + final WorldStateStorage worldStateStorage) { + return new StoredMerklePatriciaTrie<>( + worldStateStorage::getAccountStateTrieNode, b -> b, b -> b); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java index af9b9cedb8..373628591e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java @@ -17,14 +17,11 @@ package org.hyperledger.besu.ethereum.proof; import static org.assertj.core.api.Assertions.assertThat; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; -import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -36,7 +33,6 @@ import java.util.TreeMap; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; import org.junit.Before; import org.junit.Test; @@ -57,7 +53,8 @@ public class WorldStateRangeProofProviderTest { @Test public void rangeProofValidationNominalCase() { - final MerklePatriciaTrie accountStateTrie = generateTrie(); + final MerklePatriciaTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorage, 15); // collect accounts in range final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector(Hash.ZERO, MAX_RANGE, 10, Integer.MAX_VALUE); @@ -85,7 +82,8 @@ public class WorldStateRangeProofProviderTest { @Test public void rangeProofValidationMissingAccount() { - MerklePatriciaTrie accountStateTrie = generateTrie(); + MerklePatriciaTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorage, 15); // collect accounts in range final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector(Hash.ZERO, MAX_RANGE, 10, Integer.MAX_VALUE); @@ -122,7 +120,8 @@ public class WorldStateRangeProofProviderTest { @Test public void rangeProofValidationNoMonotonicIncreasing() { - MerklePatriciaTrie accountStateTrie = generateTrie(); + MerklePatriciaTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorage, 15); // generate the invalid proof final RangeStorageEntriesCollector collector = @@ -158,7 +157,8 @@ public class WorldStateRangeProofProviderTest { @Test public void rangeProofValidationEmptyProof() { - MerklePatriciaTrie accountStateTrie = generateTrie(); + MerklePatriciaTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorage, 15); // generate the invalid proof final RangeStorageEntriesCollector collector = @@ -185,7 +185,8 @@ public class WorldStateRangeProofProviderTest { @Test public void rangeProofValidationInvalidEmptyProof() { - MerklePatriciaTrie accountStateTrie = generateTrie(); + MerklePatriciaTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorage, 15); // generate the invalid proof final RangeStorageEntriesCollector collector = @@ -209,61 +210,4 @@ public class WorldStateRangeProofProviderTest { accounts); assertThat(isValidRangeProof).isFalse(); } - - private MerklePatriciaTrie generateTrie() { - final List accountHash = new ArrayList<>(); - final MerklePatriciaTrie accountStateTrie = emptyAccountStateTrie(); - // Add some storage values - for (int i = 0; i < 15; i++) { - final WorldStateStorage.Updater updater = worldStateStorage.updater(); - - accountHash.add(Hash.wrap(Bytes32.leftPad(Bytes.of(i + 1)))); - final MerklePatriciaTrie storageTrie = emptyStorageTrie(accountHash.get(i)); - writeStorageValue(storageTrie, UInt256.ONE, UInt256.valueOf(2L)); - writeStorageValue(storageTrie, UInt256.valueOf(2L), UInt256.valueOf(4L)); - writeStorageValue(storageTrie, UInt256.valueOf(3L), UInt256.valueOf(6L)); - int accountIndex = i; - storageTrie.commit( - (location, hash, value) -> - updater.putAccountStorageTrieNode( - accountHash.get(accountIndex), location, hash, value)); - final Hash codeHash = Hash.hash(Bytes32.leftPad(Bytes.of(i + 10))); - final StateTrieAccountValue accountValue = - new StateTrieAccountValue(1L, Wei.of(2L), Hash.wrap(storageTrie.getRootHash()), codeHash); - accountStateTrie.put(accountHash.get(i), RLP.encode(accountValue::writeTo)); - accountStateTrie.commit(updater::putAccountStateTrieNode); - - // Persist updates - updater.commit(); - } - return accountStateTrie; - } - - private void writeStorageValue( - final MerklePatriciaTrie storageTrie, - final UInt256 key, - final UInt256 value) { - storageTrie.put(storageKeyHash(key), encodeStorageValue(value)); - } - - private Bytes32 storageKeyHash(final UInt256 storageKey) { - return Hash.hash(storageKey); - } - - private Bytes encodeStorageValue(final UInt256 storageValue) { - return RLP.encode(out -> out.writeBytes(storageValue.toMinimalBytes())); - } - - private MerklePatriciaTrie emptyStorageTrie(final Hash accountHash) { - return new StoredMerklePatriciaTrie<>( - (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), - b -> b, - b -> b); - } - - private static MerklePatriciaTrie emptyAccountStateTrie() { - return new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, b -> b, b -> b); - } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeer.java index a0ce294816..bbf4670fdd 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeer.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.eth.messages.GetReceiptsMessage; import org.hyperledger.besu.ethereum.eth.messages.snap.GetAccountRangeMessage; import org.hyperledger.besu.ethereum.eth.messages.snap.GetByteCodesMessage; import org.hyperledger.besu.ethereum.eth.messages.snap.GetStorageRangeMessage; -import org.hyperledger.besu.ethereum.eth.messages.snap.GetTrieNodes; +import org.hyperledger.besu.ethereum.eth.messages.snap.GetTrieNodesMessage; import org.hyperledger.besu.ethereum.eth.messages.snap.SnapV1; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; @@ -57,6 +57,7 @@ import java.util.function.Consumer; import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -291,26 +292,45 @@ public class EthPeer { requestManagers.get(protocolName).get(EthPV65.GET_POOLED_TRANSACTIONS), message); } - public RequestManager.ResponseStream getSnapAccountRange(final GetAccountRangeMessage message) + public RequestManager.ResponseStream getSnapAccountRange( + final Hash stateRoot, final Bytes32 startKeyHash, final Bytes32 endKeyHash) throws PeerNotConnected { + final GetAccountRangeMessage getAccountRangeMessage = + GetAccountRangeMessage.create(stateRoot, startKeyHash, endKeyHash); + getAccountRangeMessage.setRootHash(Optional.of(stateRoot)); return sendRequest( - requestManagers.get(SnapProtocol.NAME).get(SnapV1.GET_ACCOUNT_RANGE), message); + requestManagers.get(SnapProtocol.NAME).get(SnapV1.GET_ACCOUNT_RANGE), + getAccountRangeMessage); } - public RequestManager.ResponseStream getSnapStorageRange(final GetStorageRangeMessage message) + public RequestManager.ResponseStream getSnapStorageRange( + final Hash stateRoot, + final List accountHashes, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash) throws PeerNotConnected { + final GetStorageRangeMessage getStorageRangeMessage = + GetStorageRangeMessage.create(stateRoot, accountHashes, startKeyHash, endKeyHash); + getStorageRangeMessage.setRootHash(Optional.of(stateRoot)); return sendRequest( - requestManagers.get(SnapProtocol.NAME).get(SnapV1.GET_STORAGE_RANGE), message); + requestManagers.get(SnapProtocol.NAME).get(SnapV1.GET_STORAGE_RANGE), + getStorageRangeMessage); } - public RequestManager.ResponseStream getSnapBytecode(final GetByteCodesMessage message) - throws PeerNotConnected { - return sendRequest(requestManagers.get(SnapProtocol.NAME).get(SnapV1.GET_BYTECODES), message); + public RequestManager.ResponseStream getSnapBytecode( + final Hash stateRoot, final List codeHashes) throws PeerNotConnected { + final GetByteCodesMessage getByteCodes = GetByteCodesMessage.create(codeHashes); + getByteCodes.setRootHash(Optional.of(stateRoot)); + return sendRequest( + requestManagers.get(SnapProtocol.NAME).get(SnapV1.GET_BYTECODES), getByteCodes); } - public RequestManager.ResponseStream getSnapTrieNode(final GetTrieNodes message) - throws PeerNotConnected { - return sendRequest(requestManagers.get(SnapProtocol.NAME).get(SnapV1.GET_TRIE_NODES), message); + public RequestManager.ResponseStream getSnapTrieNode( + final Hash stateRoot, final List> paths) throws PeerNotConnected { + final GetTrieNodesMessage getTrieNodes = GetTrieNodesMessage.create(stateRoot, paths); + getTrieNodes.setRootHash(Optional.of(stateRoot)); + return sendRequest( + requestManagers.get(SnapProtocol.NAME).get(SnapV1.GET_TRIE_NODES), getTrieNodes); } private RequestManager.ResponseStream sendRequest( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetAccountRangeFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetAccountRangeFromPeerTask.java new file mode 100644 index 0000000000..c52758eeb9 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetAccountRangeFromPeerTask.java @@ -0,0 +1,93 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.snap; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.PendingPeerRequest; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractPeerRequestTask; +import org.hyperledger.besu.ethereum.eth.messages.snap.AccountRangeMessage; +import org.hyperledger.besu.ethereum.eth.messages.snap.SnapV1; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GetAccountRangeFromPeerTask + extends AbstractPeerRequestTask { + + private static final Logger LOG = LoggerFactory.getLogger(GetAccountRangeFromPeerTask.class); + + private final Bytes32 startKeyHash; + private final Bytes32 endKeyHash; + private final BlockHeader blockHeader; + + private GetAccountRangeFromPeerTask( + final EthContext ethContext, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + super(ethContext, SnapV1.ACCOUNT_RANGE, metricsSystem); + this.startKeyHash = startKeyHash; + this.endKeyHash = endKeyHash; + this.blockHeader = blockHeader; + } + + public static GetAccountRangeFromPeerTask forAccountRange( + final EthContext ethContext, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + return new GetAccountRangeFromPeerTask( + ethContext, startKeyHash, endKeyHash, blockHeader, metricsSystem); + } + + @Override + protected PendingPeerRequest sendRequest() { + return sendRequestToPeer( + peer -> { + LOG.trace( + "Requesting account range [{} ,{}] for state root {} from peer {} .", + startKeyHash, + endKeyHash, + blockHeader.getStateRoot(), + peer); + return peer.getSnapAccountRange(blockHeader.getStateRoot(), startKeyHash, endKeyHash); + }, + blockHeader.getNumber()); + } + + @Override + protected Optional processResponse( + final boolean streamClosed, final MessageData message, final EthPeer peer) { + + if (streamClosed) { + // We don't record this as a useless response because it's impossible to know if a peer has + // the data we're requesting. + return Optional.empty(); + } + final AccountRangeMessage accountRangeMessage = AccountRangeMessage.readFrom(message); + final AccountRangeMessage.AccountRangeData accountRangeData = + accountRangeMessage.accountData(true); + return Optional.of(accountRangeData); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetBytecodeFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetBytecodeFromPeerTask.java new file mode 100644 index 0000000000..357157484a --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetBytecodeFromPeerTask.java @@ -0,0 +1,104 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.snap; + +import static java.util.Collections.emptyMap; +import static org.slf4j.LoggerFactory.getLogger; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.PendingPeerRequest; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractPeerRequestTask; +import org.hyperledger.besu.ethereum.eth.messages.snap.ByteCodesMessage; +import org.hyperledger.besu.ethereum.eth.messages.snap.SnapV1; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import kotlin.collections.ArrayDeque; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; + +public class GetBytecodeFromPeerTask extends AbstractPeerRequestTask> { + + private static final Logger LOG = getLogger(GetBytecodeFromPeerTask.class); + + private final List codeHashes; + private final BlockHeader blockHeader; + + private GetBytecodeFromPeerTask( + final EthContext ethContext, + final List codeHashes, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + super(ethContext, SnapV1.STORAGE_RANGE, metricsSystem); + this.codeHashes = codeHashes; + this.blockHeader = blockHeader; + } + + public static GetBytecodeFromPeerTask forBytecode( + final EthContext ethContext, + final List codeHashes, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + return new GetBytecodeFromPeerTask(ethContext, codeHashes, blockHeader, metricsSystem); + } + + @Override + protected PendingPeerRequest sendRequest() { + return sendRequestToPeer( + peer -> { + LOG.trace("Requesting {} Bytecodes from {} .", codeHashes.size(), peer); + return peer.getSnapBytecode(blockHeader.getStateRoot(), codeHashes); + }, + blockHeader.getNumber()); + } + + @Override + protected Optional> processResponse( + final boolean streamClosed, final MessageData message, final EthPeer peer) { + + if (streamClosed) { + // We don't record this as a useless response because it's impossible to know if a peer has + // the data we're requesting. + return Optional.of(emptyMap()); + } + final ByteCodesMessage byteCodesMessage = ByteCodesMessage.readFrom(message); + final ArrayDeque bytecodes = byteCodesMessage.bytecodes(true).codes(); + if (bytecodes.size() > codeHashes.size()) { + // Can't be the response to our request + return Optional.empty(); + } + return mapCodeByHash(bytecodes); + } + + private Optional> mapCodeByHash(final ArrayDeque bytecodes) { + final Map codeByHash = new HashMap<>(); + for (int i = 0; i < bytecodes.size(); i++) { + final Hash hash = Hash.hash(bytecodes.get(i)); + if (codeHashes.get(i).equals(hash)) { + codeByHash.put(hash, bytecodes.get(i)); + } + } + return Optional.of(codeByHash); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetStorageRangeFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetStorageRangeFromPeerTask.java new file mode 100644 index 0000000000..48b3426ae4 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetStorageRangeFromPeerTask.java @@ -0,0 +1,97 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.snap; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.PendingPeerRequest; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractPeerRequestTask; +import org.hyperledger.besu.ethereum.eth.messages.snap.SnapV1; +import org.hyperledger.besu.ethereum.eth.messages.snap.StorageRangeMessage; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GetStorageRangeFromPeerTask + extends AbstractPeerRequestTask { + + private static final Logger LOG = LoggerFactory.getLogger(GetStorageRangeFromPeerTask.class); + + private final List accountHashes; + private final Bytes32 startKeyHash; + private final Bytes32 endKeyHash; + private final BlockHeader blockHeader; + + private GetStorageRangeFromPeerTask( + final EthContext ethContext, + final List accountHashes, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + super(ethContext, SnapV1.STORAGE_RANGE, metricsSystem); + this.accountHashes = accountHashes; + this.startKeyHash = startKeyHash; + this.endKeyHash = endKeyHash; + this.blockHeader = blockHeader; + } + + public static GetStorageRangeFromPeerTask forStorageRange( + final EthContext ethContext, + final List accountHashes, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + return new GetStorageRangeFromPeerTask( + ethContext, accountHashes, startKeyHash, endKeyHash, blockHeader, metricsSystem); + } + + @Override + protected PendingPeerRequest sendRequest() { + return sendRequestToPeer( + peer -> { + LOG.trace( + "Requesting storage range [{} ,{}] for {} accounts from peer {} .", + startKeyHash, + endKeyHash, + accountHashes.size(), + peer); + return peer.getSnapStorageRange( + blockHeader.getStateRoot(), accountHashes, startKeyHash, endKeyHash); + }, + blockHeader.getNumber()); + } + + @Override + protected Optional processResponse( + final boolean streamClosed, final MessageData message, final EthPeer peer) { + + if (streamClosed) { + // We don't record this as a useless response because it's impossible to know if a peer has + // the data we're requesting. + return Optional.empty(); + } + + return Optional.of(StorageRangeMessage.readFrom(message).slotsData(true)); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetTrieNodeFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetTrieNodeFromPeerTask.java new file mode 100644 index 0000000000..56c7d9a235 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/GetTrieNodeFromPeerTask.java @@ -0,0 +1,113 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.snap; + +import static java.util.Collections.emptyMap; +import static org.slf4j.LoggerFactory.getLogger; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.PendingPeerRequest; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractPeerRequestTask; +import org.hyperledger.besu.ethereum.eth.messages.snap.SnapV1; +import org.hyperledger.besu.ethereum.eth.messages.snap.TrieNodesMessage; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.google.common.collect.Lists; +import kotlin.collections.ArrayDeque; +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; + +public class GetTrieNodeFromPeerTask extends AbstractPeerRequestTask> { + + private static final Logger LOG = getLogger(GetTrieNodeFromPeerTask.class); + + private final List> paths; + private final BlockHeader blockHeader; + + private GetTrieNodeFromPeerTask( + final EthContext ethContext, + final List> paths, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + super(ethContext, SnapV1.TRIE_NODES, metricsSystem); + this.paths = paths; + this.blockHeader = blockHeader; + } + + public static GetTrieNodeFromPeerTask forTrieNodes( + final EthContext ethContext, + final Map> paths, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + return new GetTrieNodeFromPeerTask( + ethContext, + paths.entrySet().stream() + .map(entry -> Lists.asList(entry.getKey(), entry.getValue().toArray(new Bytes[0]))) + .collect(Collectors.toList()), + blockHeader, + metricsSystem); + } + + @Override + protected PendingPeerRequest sendRequest() { + return sendRequestToPeer( + peer -> { + LOG.trace("Requesting {} trie nodes from peer {}", paths.size(), peer); + return peer.getSnapTrieNode(blockHeader.getStateRoot(), paths); + }, + blockHeader.getNumber()); + } + + @Override + protected Optional> processResponse( + final boolean streamClosed, final MessageData message, final EthPeer peer) { + if (streamClosed) { + // We don't record this as a useless response because it's impossible to know if a peer has + // the data we're requesting. + return Optional.of(emptyMap()); + } + final TrieNodesMessage trieNodes = TrieNodesMessage.readFrom(message); + final ArrayDeque nodes = trieNodes.nodes(true); + return mapNodeDataByPath(nodes); + } + + private Optional> mapNodeDataByPath(final ArrayDeque nodeData) { + final Map nodeDataByPath = new HashMap<>(); + paths.forEach( + list -> { + int i = 1; + if (!nodeData.isEmpty() && i == list.size()) { + Bytes bytes = nodeData.removeFirst(); + nodeDataByPath.put(list.get(0), bytes); + } else { + while (!nodeData.isEmpty() && i < list.size()) { + Bytes bytes = nodeData.removeFirst(); + nodeDataByPath.put(Bytes.concatenate(list.get(0), list.get(i)), bytes); + i++; + } + } + }); + return Optional.of(nodeDataByPath); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetAccountRangeFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetAccountRangeFromPeerTask.java new file mode 100644 index 0000000000..33254cf1b8 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetAccountRangeFromPeerTask.java @@ -0,0 +1,78 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.snap; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractRetryingPeerTask; +import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; +import org.hyperledger.besu.ethereum.eth.messages.snap.AccountRangeMessage; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import org.apache.tuweni.bytes.Bytes32; + +public class RetryingGetAccountRangeFromPeerTask + extends AbstractRetryingPeerTask { + + private final EthContext ethContext; + private final Bytes32 startKeyHash; + private final Bytes32 endKeyHash; + private final BlockHeader blockHeader; + private final MetricsSystem metricsSystem; + + private RetryingGetAccountRangeFromPeerTask( + final EthContext ethContext, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + super( + ethContext, 3, data -> data.accounts().isEmpty() && data.proofs().isEmpty(), metricsSystem); + this.ethContext = ethContext; + this.startKeyHash = startKeyHash; + this.endKeyHash = endKeyHash; + this.blockHeader = blockHeader; + this.metricsSystem = metricsSystem; + } + + public static EthTask forAccountRange( + final EthContext ethContext, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + return new RetryingGetAccountRangeFromPeerTask( + ethContext, startKeyHash, endKeyHash, blockHeader, metricsSystem); + } + + @Override + protected CompletableFuture executePeerTask( + final Optional assignedPeer) { + final GetAccountRangeFromPeerTask task = + GetAccountRangeFromPeerTask.forAccountRange( + ethContext, startKeyHash, endKeyHash, blockHeader, metricsSystem); + assignedPeer.ifPresent(task::assignPeer); + return executeSubTask(task::run) + .thenApply( + peerResult -> { + result.complete(peerResult.getResult()); + return peerResult.getResult(); + }); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetBytecodeFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetBytecodeFromPeerTask.java new file mode 100644 index 0000000000..cd5dc69959 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetBytecodeFromPeerTask.java @@ -0,0 +1,72 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.snap; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractRetryingPeerTask; +import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public class RetryingGetBytecodeFromPeerTask extends AbstractRetryingPeerTask> { + + private final EthContext ethContext; + private final List codeHashes; + private final BlockHeader blockHeader; + private final MetricsSystem metricsSystem; + + private RetryingGetBytecodeFromPeerTask( + final EthContext ethContext, + final List codeHashes, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + super(ethContext, 3, Map::isEmpty, metricsSystem); + this.ethContext = ethContext; + this.codeHashes = codeHashes; + this.blockHeader = blockHeader; + this.metricsSystem = metricsSystem; + } + + public static EthTask> forByteCode( + final EthContext ethContext, + final List codeHashes, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + return new RetryingGetBytecodeFromPeerTask(ethContext, codeHashes, blockHeader, metricsSystem); + } + + @Override + protected CompletableFuture> executePeerTask( + final Optional assignedPeer) { + final GetBytecodeFromPeerTask task = + GetBytecodeFromPeerTask.forBytecode(ethContext, codeHashes, blockHeader, metricsSystem); + assignedPeer.ifPresent(task::assignPeer); + return executeSubTask(task::run) + .thenApply( + peerResult -> { + result.complete(peerResult.getResult()); + return peerResult.getResult(); + }); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetStorageRangeFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetStorageRangeFromPeerTask.java new file mode 100644 index 0000000000..d7bc8a14ec --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetStorageRangeFromPeerTask.java @@ -0,0 +1,82 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.snap; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractRetryingPeerTask; +import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; +import org.hyperledger.besu.ethereum.eth.messages.snap.StorageRangeMessage; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import org.apache.tuweni.bytes.Bytes32; + +public class RetryingGetStorageRangeFromPeerTask + extends AbstractRetryingPeerTask { + + private final EthContext ethContext; + private final List accountHashes; + private final Bytes32 startKeyHash; + private final Bytes32 endKeyHash; + private final BlockHeader blockHeader; + private final MetricsSystem metricsSystem; + + private RetryingGetStorageRangeFromPeerTask( + final EthContext ethContext, + final List accountHashes, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + super(ethContext, 3, data -> data.proofs().isEmpty() && data.slots().isEmpty(), metricsSystem); + this.ethContext = ethContext; + this.accountHashes = accountHashes; + this.startKeyHash = startKeyHash; + this.endKeyHash = endKeyHash; + this.blockHeader = blockHeader; + this.metricsSystem = metricsSystem; + } + + public static EthTask forStorageRange( + final EthContext ethContext, + final List accountHashes, + final Bytes32 startKeyHash, + final Bytes32 endKeyHash, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + return new RetryingGetStorageRangeFromPeerTask( + ethContext, accountHashes, startKeyHash, endKeyHash, blockHeader, metricsSystem); + } + + @Override + protected CompletableFuture executePeerTask( + final Optional assignedPeer) { + final GetStorageRangeFromPeerTask task = + GetStorageRangeFromPeerTask.forStorageRange( + ethContext, accountHashes, startKeyHash, endKeyHash, blockHeader, metricsSystem); + assignedPeer.ifPresent(task::assignPeer); + return executeSubTask(task::run) + .thenApply( + peerResult -> { + result.complete(peerResult.getResult()); + return peerResult.getResult(); + }); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetTrieNodeFromPeerTask.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetTrieNodeFromPeerTask.java new file mode 100644 index 0000000000..9b162c8951 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/RetryingGetTrieNodeFromPeerTask.java @@ -0,0 +1,71 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.manager.snap; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractRetryingPeerTask; +import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import org.apache.tuweni.bytes.Bytes; + +public class RetryingGetTrieNodeFromPeerTask extends AbstractRetryingPeerTask> { + + private final EthContext ethContext; + private final Map> paths; + private final BlockHeader blockHeader; + private final MetricsSystem metricsSystem; + + private RetryingGetTrieNodeFromPeerTask( + final EthContext ethContext, + final Map> paths, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + super(ethContext, 3, Map::isEmpty, metricsSystem); + this.ethContext = ethContext; + this.paths = paths; + this.blockHeader = blockHeader; + this.metricsSystem = metricsSystem; + } + + public static EthTask> forTrieNodes( + final EthContext ethContext, + final Map> paths, + final BlockHeader blockHeader, + final MetricsSystem metricsSystem) { + return new RetryingGetTrieNodeFromPeerTask(ethContext, paths, blockHeader, metricsSystem); + } + + @Override + protected CompletableFuture> executePeerTask( + final Optional assignedPeer) { + final GetTrieNodeFromPeerTask task = + GetTrieNodeFromPeerTask.forTrieNodes(ethContext, paths, blockHeader, metricsSystem); + assignedPeer.ifPresent(task::assignPeer); + return executeSubTask(task::run) + .thenApply( + peerResult -> { + result.complete(peerResult.getResult()); + return peerResult.getResult(); + }); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java index fbac0a4869..17fb0a246c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java @@ -19,7 +19,7 @@ import org.hyperledger.besu.ethereum.eth.messages.snap.AccountRangeMessage; import org.hyperledger.besu.ethereum.eth.messages.snap.ByteCodesMessage; import org.hyperledger.besu.ethereum.eth.messages.snap.SnapV1; import org.hyperledger.besu.ethereum.eth.messages.snap.StorageRangeMessage; -import org.hyperledger.besu.ethereum.eth.messages.snap.TrieNodes; +import org.hyperledger.besu.ethereum.eth.messages.snap.TrieNodesMessage; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; @@ -74,6 +74,6 @@ class SnapServer { private MessageData constructGetTrieNodesResponse( final WorldStateArchive worldStateArchive, final MessageData message) { - return TrieNodes.create(new ArrayDeque<>()); + return TrieNodesMessage.create(new ArrayDeque<>()); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java index abc8ba5edb..417bbd27fc 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import java.math.BigInteger; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -34,6 +35,10 @@ import org.immutables.value.Value; public final class AccountRangeMessage extends AbstractSnapMessageData { + public AccountRangeMessage(final Bytes data) { + super(data); + } + public static AccountRangeMessage readFrom(final MessageData message) { if (message instanceof AccountRangeMessage) { return (AccountRangeMessage) message; @@ -47,14 +52,14 @@ public final class AccountRangeMessage extends AbstractSnapMessageData { } public static AccountRangeMessage create( - final Map accounts, final ArrayDeque proof) { + final Map accounts, final List proof) { return create(Optional.empty(), accounts, proof); } public static AccountRangeMessage create( final Optional requestId, final Map accounts, - final ArrayDeque proof) { + final List proof) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); requestId.ifPresent(tmp::writeBigIntegerScalar); @@ -71,10 +76,6 @@ public final class AccountRangeMessage extends AbstractSnapMessageData { return new AccountRangeMessage(tmp.encoded()); } - public AccountRangeMessage(final Bytes data) { - super(data); - } - @Override protected Bytes wrap(final BigInteger requestId) { final AccountRangeData accountData = accountData(false); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/ByteCodesMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/ByteCodesMessage.java index 007e94993a..b86fcdb41a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/ByteCodesMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/ByteCodesMessage.java @@ -30,6 +30,10 @@ import org.immutables.value.Value; public final class ByteCodesMessage extends AbstractSnapMessageData { + public ByteCodesMessage(final Bytes data) { + super(data); + } + public static ByteCodesMessage readFrom(final MessageData message) { if (message instanceof ByteCodesMessage) { return (ByteCodesMessage) message; @@ -56,10 +60,6 @@ public final class ByteCodesMessage extends AbstractSnapMessageData { return new ByteCodesMessage(tmp.encoded()); } - public ByteCodesMessage(final Bytes data) { - super(data); - } - @Override protected Bytes wrap(final BigInteger requestId) { final ByteCodes bytecodes = bytecodes(false); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessage.java index fefc6ddc9e..9d1eccce98 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessage.java @@ -29,6 +29,10 @@ import org.immutables.value.Value; public final class GetAccountRangeMessage extends AbstractSnapMessageData { + public GetAccountRangeMessage(final Bytes data) { + super(data); + } + public static GetAccountRangeMessage readFrom(final MessageData message) { if (message instanceof GetAccountRangeMessage) { return (GetAccountRangeMessage) message; @@ -42,24 +46,17 @@ public final class GetAccountRangeMessage extends AbstractSnapMessageData { } public static GetAccountRangeMessage create( - final Hash worldStateRootHash, - final Bytes32 startKeyHash, - final Bytes32 endKeyHash, - final BigInteger responseBytes) { + final Hash worldStateRootHash, final Bytes32 startKeyHash, final Bytes32 endKeyHash) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); tmp.writeBytes(worldStateRootHash); tmp.writeBytes(startKeyHash); tmp.writeBytes(endKeyHash); - tmp.writeBigIntegerScalar(responseBytes); + tmp.writeBigIntegerScalar(SIZE_REQUEST); tmp.endList(); return new GetAccountRangeMessage(tmp.encoded()); } - public GetAccountRangeMessage(final Bytes data) { - super(data); - } - @Override protected Bytes wrap(final BigInteger requestId) { final Range range = range(false); @@ -86,7 +83,7 @@ public final class GetAccountRangeMessage extends AbstractSnapMessageData { final Hash worldStateRootHash = Hash.wrap(Bytes32.wrap(input.readBytes32())); final ImmutableRange range = ImmutableRange.builder() - .worldStateRootHash(getOverrideStateRoot().orElse(worldStateRootHash)) + .worldStateRootHash(getRootHash().orElse(worldStateRootHash)) .startKeyHash(Hash.wrap(Bytes32.wrap(input.readBytes32()))) .endKeyHash(Hash.wrap(Bytes32.wrap(input.readBytes32()))) .responseBytes(input.readBigIntegerScalar()) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetByteCodesMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetByteCodesMessage.java index a4d02712f9..25302084d1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetByteCodesMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetByteCodesMessage.java @@ -21,16 +21,19 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLPInput; import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; -import kotlin.collections.ArrayDeque; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.immutables.value.Value; public final class GetByteCodesMessage extends AbstractSnapMessageData { - final Optional> accountHashes; + public GetByteCodesMessage(final Bytes data) { + super(data); + } public static GetByteCodesMessage readFrom(final MessageData message) { if (message instanceof GetByteCodesMessage) { @@ -41,40 +44,27 @@ public final class GetByteCodesMessage extends AbstractSnapMessageData { throw new IllegalArgumentException( String.format("Message has code %d and thus is not a GetByteCodesMessage.", code)); } - return new GetByteCodesMessage(Optional.empty(), message.getData()); + return new GetByteCodesMessage(message.getData()); } - public static GetByteCodesMessage create( - final Optional> accountHashes, - final ArrayDeque codeHashes, - final BigInteger responseBytes) { - return create(Optional.empty(), accountHashes, codeHashes, responseBytes); + public static GetByteCodesMessage create(final List codeHashes) { + return create(Optional.empty(), codeHashes); } public static GetByteCodesMessage create( - final Optional requestId, - final Optional> accountHashes, - final ArrayDeque codeHashes, - final BigInteger responseBytes) { + final Optional requestId, final List codeHashes) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); requestId.ifPresent(tmp::writeBigIntegerScalar); tmp.writeList(codeHashes, (hash, rlpOutput) -> rlpOutput.writeBytes(hash)); - tmp.writeBigIntegerScalar(responseBytes); + tmp.writeBigIntegerScalar(SIZE_REQUEST); tmp.endList(); - return new GetByteCodesMessage(accountHashes, tmp.encoded()); - } - - public GetByteCodesMessage(final Optional> accountHashes, final Bytes data) { - super(data); - this.accountHashes = accountHashes; + return new GetByteCodesMessage(tmp.encoded()); } @Override protected Bytes wrap(final BigInteger requestId) { - final CodeHashes request = codeHashes(false); - return create(Optional.of(requestId), accountHashes, request.hashes(), request.responseBytes()) - .getData(); + return create(Optional.of(requestId), codeHashes(false).hashes()).getData(); } @Override @@ -83,7 +73,7 @@ public final class GetByteCodesMessage extends AbstractSnapMessageData { } public CodeHashes codeHashes(final boolean withRequestId) { - final ArrayDeque hashes = new ArrayDeque<>(); + final List hashes = new ArrayList<>(); final BigInteger responseBytes; final RLPInput input = new BytesValueRLPInput(data, false); input.enterList(); @@ -98,14 +88,10 @@ public final class GetByteCodesMessage extends AbstractSnapMessageData { return ImmutableCodeHashes.builder().hashes(hashes).responseBytes(responseBytes).build(); } - public Optional> getAccountHashes() { - return accountHashes; - } - @Value.Immutable public interface CodeHashes { - ArrayDeque hashes(); + List hashes(); BigInteger responseBytes(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessage.java index 1f1fb8d56c..fe27f9eb5b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessage.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLPInput; import java.math.BigInteger; +import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -32,7 +33,9 @@ import org.immutables.value.Value; public final class GetStorageRangeMessage extends AbstractSnapMessageData { - private final Optional> storageRoots; + public GetStorageRangeMessage(final Bytes data) { + super(data); + } public static GetStorageRangeMessage readFrom(final MessageData message) { if (message instanceof GetStorageRangeMessage) { @@ -48,29 +51,18 @@ public final class GetStorageRangeMessage extends AbstractSnapMessageData { public static GetStorageRangeMessage create( final Hash worldStateRootHash, - final ArrayDeque accountHashes, - final Optional> storageRoots, + final List accountHashes, final Bytes32 startKeyHash, - final Bytes32 endKeyHash, - final BigInteger responseBytes) { - return create( - Optional.empty(), - worldStateRootHash, - accountHashes, - storageRoots, - startKeyHash, - endKeyHash, - responseBytes); + final Bytes32 endKeyHash) { + return create(Optional.empty(), worldStateRootHash, accountHashes, startKeyHash, endKeyHash); } public static GetStorageRangeMessage create( final Optional requestId, final Hash worldStateRootHash, - final ArrayDeque accountHashes, - final Optional> storageRoots, + final List accountHashes, final Bytes32 startKeyHash, - final Bytes32 endKeyHash, - final BigInteger responseBytes) { + final Bytes32 endKeyHash) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); requestId.ifPresent(tmp::writeBigIntegerScalar); @@ -78,23 +70,9 @@ public final class GetStorageRangeMessage extends AbstractSnapMessageData { tmp.writeList(accountHashes, (hash, rlpOutput) -> rlpOutput.writeBytes(hash)); tmp.writeBytes(startKeyHash); tmp.writeBytes(endKeyHash); - tmp.writeBigIntegerScalar(responseBytes); + tmp.writeBigIntegerScalar(SIZE_REQUEST); tmp.endList(); - return new GetStorageRangeMessage(tmp.encoded(), storageRoots); - } - - public GetStorageRangeMessage(final Bytes data) { - this(data, Optional.empty()); - } - - public GetStorageRangeMessage( - final Bytes data, final Optional> storageRoots) { - super(data); - this.storageRoots = storageRoots; - } - - public Optional> getStorageRoots() { - return storageRoots; + return new GetStorageRangeMessage(tmp.encoded()); } @Override @@ -104,10 +82,8 @@ public final class GetStorageRangeMessage extends AbstractSnapMessageData { Optional.of(requestId), range.worldStateRootHash(), range.hashes(), - storageRoots, range.startKeyHash(), - range.endKeyHash(), - range.responseBytes()) + range.endKeyHash()) .getData(); } @@ -124,7 +100,7 @@ public final class GetStorageRangeMessage extends AbstractSnapMessageData { final Hash worldStateRootHash = Hash.wrap(Bytes32.wrap(input.readBytes32())); final ImmutableStorageRange.Builder range = ImmutableStorageRange.builder() - .worldStateRootHash(getOverrideStateRoot().orElse(worldStateRootHash)); + .worldStateRootHash(getRootHash().orElse(worldStateRootHash)); input.enterList(); while (!input.isEndOfCurrentList()) { hashes.add(input.readBytes32()); @@ -149,11 +125,6 @@ public final class GetStorageRangeMessage extends AbstractSnapMessageData { return range.build(); } - @Override - public String toString() { - return "GetStorageRangeMessage{" + "storageRoots=" + storageRoots + ", data=" + data + '}'; - } - @Value.Immutable public interface StorageRange { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodes.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodesMessage.java similarity index 72% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodes.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodesMessage.java index 4234cde883..1d184231ed 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodes.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodesMessage.java @@ -29,39 +29,33 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.immutables.value.Value; -public final class GetTrieNodes extends AbstractSnapMessageData { +public final class GetTrieNodesMessage extends AbstractSnapMessageData { - public static GetTrieNodes readFrom(final MessageData message) { - if (message instanceof GetTrieNodes) { - return (GetTrieNodes) message; + public GetTrieNodesMessage(final Bytes data) { + super(data); + } + + public static GetTrieNodesMessage readFrom(final MessageData message) { + if (message instanceof GetTrieNodesMessage) { + return (GetTrieNodesMessage) message; } final int code = message.getCode(); if (code != SnapV1.GET_TRIE_NODES) { throw new IllegalArgumentException( String.format("Message has code %d and thus is not a GetTrieNodes.", code)); } - return new GetTrieNodes(message.getData()); + return new GetTrieNodesMessage(message.getData()); } - /*public static GetTrieNodes createWithRequest( - final Hash worldStateRootHash, - final TrieNodeDataRequest request, - final BigInteger responseBytes) { - return create(Optional.empty(), worldStateRootHash, request.getPaths(), responseBytes); - }*/ - - public static GetTrieNodes create( - final Hash worldStateRootHash, - final List> requests, - final BigInteger responseBytes) { - return create(Optional.empty(), worldStateRootHash, requests, responseBytes); + public static GetTrieNodesMessage create( + final Hash worldStateRootHash, final List> requests) { + return create(Optional.empty(), worldStateRootHash, requests); } - public static GetTrieNodes create( + public static GetTrieNodesMessage create( final Optional requestId, final Hash worldStateRootHash, - final List> paths, - final BigInteger responseBytes) { + final List> paths) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); requestId.ifPresent(tmp::writeBigIntegerScalar); @@ -70,24 +64,15 @@ public final class GetTrieNodes extends AbstractSnapMessageData { paths, (path, rlpOutput) -> rlpOutput.writeList(path, (b, subRlpOutput) -> subRlpOutput.writeBytes(b))); - tmp.writeBigIntegerScalar(responseBytes); + tmp.writeBigIntegerScalar(SIZE_REQUEST); tmp.endList(); - return new GetTrieNodes(tmp.encoded()); - } - - public GetTrieNodes(final Bytes data) { - super(data); + return new GetTrieNodesMessage(tmp.encoded()); } @Override protected Bytes wrap(final BigInteger requestId) { final TrieNodesPaths paths = paths(false); - return create( - Optional.of(requestId), - paths.worldStateRootHash(), - paths.paths(), - paths.responseBytes()) - .getData(); + return create(Optional.of(requestId), paths.worldStateRootHash(), paths.paths()).getData(); } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java index 712227ef41..49d39c9ee1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.rlp.RLPInput; import java.math.BigInteger; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -32,6 +33,10 @@ import org.immutables.value.Value; public final class StorageRangeMessage extends AbstractSnapMessageData { + public StorageRangeMessage(final Bytes data) { + super(data); + } + public static StorageRangeMessage readFrom(final MessageData message) { if (message instanceof StorageRangeMessage) { return (StorageRangeMessage) message; @@ -45,13 +50,13 @@ public final class StorageRangeMessage extends AbstractSnapMessageData { } public static StorageRangeMessage create( - final ArrayDeque> slots, final List proof) { + final ArrayDeque> slots, final List proof) { return create(Optional.empty(), slots, proof); } public static StorageRangeMessage create( final Optional requestId, - final ArrayDeque> slots, + final ArrayDeque> slots, final List proof) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); @@ -72,10 +77,6 @@ public final class StorageRangeMessage extends AbstractSnapMessageData { return new StorageRangeMessage(tmp.encoded()); } - public StorageRangeMessage(final Bytes data) { - super(data); - } - @Override protected Bytes wrap(final BigInteger requestId) { final SlotRangeData slotsData = slotsData(false); @@ -88,7 +89,7 @@ public final class StorageRangeMessage extends AbstractSnapMessageData { } public SlotRangeData slotsData(final boolean withRequestId) { - final ArrayDeque> slots = new ArrayDeque<>(); + final ArrayDeque> slots = new ArrayDeque<>(); final ArrayDeque proofs = new ArrayDeque<>(); final RLPInput input = new BytesValueRLPInput(data, false); input.enterList(); @@ -120,7 +121,7 @@ public final class StorageRangeMessage extends AbstractSnapMessageData { @Value.Immutable public interface SlotRangeData { - ArrayDeque> slots(); + ArrayDeque> slots(); ArrayDeque proofs(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodes.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodesMessage.java similarity index 81% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodes.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodesMessage.java index c1d49aeab1..15208e7b42 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodes.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodesMessage.java @@ -27,35 +27,36 @@ import java.util.Optional; import kotlin.collections.ArrayDeque; import org.apache.tuweni.bytes.Bytes; -public final class TrieNodes extends AbstractSnapMessageData { +public final class TrieNodesMessage extends AbstractSnapMessageData { - public static TrieNodes readFrom(final MessageData message) { - if (message instanceof TrieNodes) { - return (TrieNodes) message; + public TrieNodesMessage(final Bytes data) { + super(data); + } + + public static TrieNodesMessage readFrom(final MessageData message) { + if (message instanceof TrieNodesMessage) { + return (TrieNodesMessage) message; } final int code = message.getCode(); if (code != SnapV1.TRIE_NODES) { throw new IllegalArgumentException( String.format("Message has code %d and thus is not a TrieNodes.", code)); } - return new TrieNodes(message.getData()); + return new TrieNodesMessage(message.getData()); } - public static TrieNodes create(final List nodes) { + public static TrieNodesMessage create(final List nodes) { return create(Optional.empty(), nodes); } - public static TrieNodes create(final Optional requestId, final List nodes) { + public static TrieNodesMessage create( + final Optional requestId, final List nodes) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); requestId.ifPresent(tmp::writeBigIntegerScalar); tmp.writeList(nodes, (node, rlpOutput) -> rlpOutput.writeBytes(node)); tmp.endList(); - return new TrieNodes(tmp.encoded()); - } - - public TrieNodes(final Bytes data) { - super(data); + return new TrieNodesMessage(tmp.encoded()); } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java new file mode 100644 index 0000000000..6f4856aad8 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java @@ -0,0 +1,142 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.sync.snapsync; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.trie.InnerNodeDiscoveryManager; +import org.hyperledger.besu.ethereum.trie.MerkleTrieException; +import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; + +import java.math.BigInteger; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +/** + * This class helps to generate ranges according to several parameters (the start and the end of the + * range, its size) + */ +public class RangeManager { + + public static final Hash MIN_RANGE = + Hash.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"); + public static final Hash MAX_RANGE = + Hash.fromHexString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + + private RangeManager() {} + + public static Map generateAllRanges(final int sizeRange) { + if (sizeRange == 1) { + return Map.ofEntries(Map.entry(MIN_RANGE, MAX_RANGE)); + } + return generateRanges( + MIN_RANGE.toUnsignedBigInteger(), MAX_RANGE.toUnsignedBigInteger(), sizeRange); + } + + /** + * Generate a range + * + * @param min start of the range in bytes + * @param max the max end of the range in bytes + * @param nbRange number of ranges + * @return the start and end of the generated range + */ + public static Map generateRanges( + final Bytes32 min, final Bytes32 max, final int nbRange) { + return generateRanges(min.toUnsignedBigInteger(), max.toUnsignedBigInteger(), nbRange); + } + + /** + * Generate a range + * + * @param min start of the range + * @param max the max end of the range + * @param nbRange number of ranges + * @return the start and end of the generated range + */ + public static Map generateRanges( + final BigInteger min, final BigInteger max, final int nbRange) { + final BigInteger rangeSize = max.subtract(min).divide(BigInteger.valueOf(nbRange)); + final TreeMap ranges = new TreeMap<>(); + if (min.equals(max) || nbRange == 1) { + ranges.put(format(min), format(max)); + return ranges; + } + BigInteger currentStart = min; + BigInteger currentEnd = min; + while (max.subtract(currentEnd).compareTo(rangeSize) > 0) { + currentEnd = currentStart.add(rangeSize); + ranges.put(format(currentStart), format(currentEnd)); + currentStart = currentStart.add(rangeSize).add(BigInteger.ONE); + } + if (max.subtract(currentEnd).compareTo(BigInteger.ZERO) > 0) { + currentEnd = currentStart.add(max.subtract(currentEnd)).subtract(BigInteger.ONE); + ranges.put(format(currentStart), format(currentEnd)); + } + return ranges; + } + + /** + * Helps to create a new range according to the last data obtained. This happens when a peer + * doesn't return all of the data in a range. + * + * @param worldstateRootHash the root hash + * @param proofs proof received + * @param endKeyHash the end of the range initially wanted + * @param receivedKeys the last key received + * @return begin of the new range + */ + public static Optional findNewBeginElementInRange( + final Bytes32 worldstateRootHash, + final List proofs, + final TreeMap receivedKeys, + final Bytes32 endKeyHash) { + if (receivedKeys.isEmpty() || receivedKeys.lastKey().compareTo(endKeyHash) >= 0) { + return Optional.empty(); + } else { + final Map proofsEntries = new HashMap<>(); + for (Bytes proof : proofs) { + proofsEntries.put(Hash.hash(proof), proof); + } + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + new InnerNodeDiscoveryManager<>( + (location, key) -> Optional.ofNullable(proofsEntries.get(key)), + Function.identity(), + Function.identity(), + receivedKeys.lastKey(), + endKeyHash, + false), + worldstateRootHash); + + try { + storageTrie.visitAll(bytesNode -> {}); + } catch (MerkleTrieException e) { + return Optional.of(InnerNodeDiscoveryManager.decodePath(e.getLocation())); + } + return Optional.empty(); + } + } + + private static Bytes32 format(final BigInteger data) { + return Bytes32.leftPad(Bytes.of(data.toByteArray()).trimLeadingZeros()); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessageTest.java new file mode 100644 index 0000000000..bfb76576fe --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessageTest.java @@ -0,0 +1,57 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.messages.snap; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class AccountRangeMessageTest { + + @Test + public void roundTripTest() { + final Map keys = new HashMap<>(); + final StateTrieAccountValue accountValue = + new StateTrieAccountValue(1L, Wei.of(2L), Hash.EMPTY_TRIE_HASH, Hash.EMPTY); + keys.put(Hash.wrap(Bytes32.leftPad(Bytes.of(1))), RLP.encode(accountValue::writeTo)); + + final List proofs = new ArrayList<>(); + proofs.add(Bytes32.random()); + + // Perform round-trip transformation + final MessageData initialMessage = AccountRangeMessage.create(keys, proofs); + final MessageData raw = new RawMessage(SnapV1.ACCOUNT_RANGE, initialMessage.getData()); + + final AccountRangeMessage message = AccountRangeMessage.readFrom(raw); + + // check match originals. + final AccountRangeMessage.AccountRangeData range = message.accountData(false); + Assertions.assertThat(range.accounts()).isEqualTo(keys); + Assertions.assertThat(range.proofs()).isEqualTo(proofs); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/BytecodeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/BytecodeMessageTest.java new file mode 100644 index 0000000000..28b8b6883c --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/BytecodeMessageTest.java @@ -0,0 +1,49 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.messages.snap; + +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class BytecodeMessageTest { + + @Test + public void roundTripTest() { + + final List codes = new ArrayList<>(); + final int hashCount = 20; + for (int i = 0; i < hashCount; ++i) { + codes.add(Bytes32.random()); + } + + // Perform round-trip transformation + final MessageData initialMessage = ByteCodesMessage.create(codes); + final MessageData raw = new RawMessage(SnapV1.BYTECODES, initialMessage.getData()); + + final ByteCodesMessage message = ByteCodesMessage.readFrom(raw); + + // check match originals. + final ByteCodesMessage.ByteCodes response = message.bytecodes(false); + Assertions.assertThat(response.codes()).isEqualTo(codes); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java new file mode 100644 index 0000000000..c56629149d --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java @@ -0,0 +1,48 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.messages.snap; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; + +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class GetAccountRangeMessageTest { + + @Test + public void roundTripTest() { + final Hash rootHash = Hash.wrap(Bytes32.random()); + final Hash startKeyHash = RangeManager.MIN_RANGE; + final Hash endKeyHash = RangeManager.MAX_RANGE; + + // Perform round-trip transformation + final MessageData initialMessage = + GetAccountRangeMessage.create(rootHash, startKeyHash, endKeyHash); + final MessageData raw = new RawMessage(SnapV1.GET_ACCOUNT_RANGE, initialMessage.getData()); + + final GetAccountRangeMessage message = GetAccountRangeMessage.readFrom(raw); + + // check match originals. + final GetAccountRangeMessage.Range range = message.range(false); + Assertions.assertThat(range.worldStateRootHash()).isEqualTo(rootHash); + Assertions.assertThat(range.startKeyHash()).isEqualTo(startKeyHash); + Assertions.assertThat(range.responseBytes()).isEqualTo(AbstractSnapMessageData.SIZE_REQUEST); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetBytecodeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetBytecodeMessageTest.java new file mode 100644 index 0000000000..39f1688209 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetBytecodeMessageTest.java @@ -0,0 +1,51 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.messages.snap; + +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class GetBytecodeMessageTest { + + @Test + public void roundTripTest() { + + final List hashes = new ArrayList<>(); + final int hashCount = 20; + for (int i = 0; i < hashCount; ++i) { + hashes.add(Bytes32.random()); + } + + // Perform round-trip transformation + final MessageData initialMessage = GetByteCodesMessage.create(hashes); + final MessageData raw = new RawMessage(SnapV1.GET_BYTECODES, initialMessage.getData()); + + final GetByteCodesMessage message = GetByteCodesMessage.readFrom(raw); + + // check match originals. + final GetByteCodesMessage.CodeHashes codeHashes = message.codeHashes(false); + Assertions.assertThat(codeHashes.hashes()).isEqualTo(hashes); + Assertions.assertThat(codeHashes.responseBytes()) + .isEqualTo(AbstractSnapMessageData.SIZE_REQUEST); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java new file mode 100644 index 0000000000..49a43f9412 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java @@ -0,0 +1,52 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.messages.snap; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; + +import java.util.List; + +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class GetStorageRangeMessageTest { + + @Test + public void roundTripTest() { + final Hash rootHash = Hash.wrap(Bytes32.random()); + final List accountKeys = List.of(Bytes32.random()); + final Hash startKeyHash = RangeManager.MIN_RANGE; + final Hash endKeyHash = RangeManager.MAX_RANGE; + + // Perform round-trip transformation + final MessageData initialMessage = + GetStorageRangeMessage.create(rootHash, accountKeys, startKeyHash, endKeyHash); + final MessageData raw = new RawMessage(SnapV1.GET_STORAGE_RANGE, initialMessage.getData()); + + final GetStorageRangeMessage message = GetStorageRangeMessage.readFrom(raw); + + // check match originals. + final GetStorageRangeMessage.StorageRange range = message.range(false); + Assertions.assertThat(range.worldStateRootHash()).isEqualTo(rootHash); + Assertions.assertThat(range.hashes()).isEqualTo(accountKeys); + Assertions.assertThat(range.startKeyHash()).isEqualTo(startKeyHash); + Assertions.assertThat(range.responseBytes()).isEqualTo(AbstractSnapMessageData.SIZE_REQUEST); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodeMessageTest.java new file mode 100644 index 0000000000..67447005c8 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetTrieNodeMessageTest.java @@ -0,0 +1,53 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.messages.snap; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class GetTrieNodeMessageTest { + + @Test + public void roundTripTest() { + final Hash rootHash = Hash.wrap(Bytes32.random()); + final List paths = new ArrayList<>(); + final int hashCount = 20; + for (int i = 0; i < hashCount; ++i) { + paths.add(Bytes32.random()); + } + + // Perform round-trip transformation + final MessageData initialMessage = GetTrieNodesMessage.create(rootHash, List.of(paths)); + final MessageData raw = new RawMessage(SnapV1.GET_TRIE_NODES, initialMessage.getData()); + + final GetTrieNodesMessage message = GetTrieNodesMessage.readFrom(raw); + + // check match originals. + final GetTrieNodesMessage.TrieNodesPaths response = message.paths(false); + Assertions.assertThat(response.worldStateRootHash()).isEqualTo(rootHash); + Assertions.assertThat(response.paths()).contains(paths); + Assertions.assertThat(response.responseBytes()).isEqualTo(AbstractSnapMessageData.SIZE_REQUEST); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java new file mode 100644 index 0000000000..19554ec424 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java @@ -0,0 +1,56 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.messages.snap; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import kotlin.collections.ArrayDeque; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class StorageRangeMessageTest { + + @Test + public void roundTripTest() { + + final ArrayDeque> keys = new ArrayDeque<>(); + final Map storage = new HashMap<>(); + storage.put(Hash.wrap(Bytes32.leftPad(Bytes.of(1))), Bytes32.random()); + keys.add(storage); + + final List proofs = new ArrayList<>(); + proofs.add(Bytes32.random()); + + // Perform round-trip transformation + final MessageData initialMessage = StorageRangeMessage.create(keys, proofs); + final MessageData raw = new RawMessage(SnapV1.STORAGE_RANGE, initialMessage.getData()); + + final StorageRangeMessage message = StorageRangeMessage.readFrom(raw); + + // check match originals. + final StorageRangeMessage.SlotRangeData range = message.slotsData(false); + Assertions.assertThat(range.slots()).isEqualTo(keys); + Assertions.assertThat(range.proofs()).isEqualTo(proofs); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodeMessageTest.java new file mode 100644 index 0000000000..02481e7e1b --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/TrieNodeMessageTest.java @@ -0,0 +1,49 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.messages.snap; + +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; +import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; + +import java.util.ArrayList; +import java.util.List; + +import kotlin.collections.ArrayDeque; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class TrieNodeMessageTest { + + @Test + public void roundTripTest() { + final List nodes = new ArrayList<>(); + final int hashCount = 20; + for (int i = 0; i < hashCount; ++i) { + nodes.add(Bytes32.random()); + } + + // Perform round-trip transformation + final MessageData initialMessage = TrieNodesMessage.create(nodes); + final MessageData raw = new RawMessage(SnapV1.TRIE_NODES, initialMessage.getData()); + + final TrieNodesMessage message = TrieNodesMessage.readFrom(raw); + + // check match originals. + final ArrayDeque response = message.nodes(false); + Assertions.assertThat(response).isEqualTo(nodes); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java new file mode 100644 index 0000000000..37c8e6d77e --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java @@ -0,0 +1,174 @@ +/* + * Copyright contributors to Hyperledger Besu + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.sync.snapsync; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.TrieGenerator; +import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; +import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; +import org.hyperledger.besu.ethereum.trie.TrieIterator; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public final class RangeManagerTest { + + @Test + public void testGenerateAllRangesWithSize1() { + final Map expectedResult = new HashMap<>(); + expectedResult.put( + Bytes32.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), + Bytes32.fromHexString( + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + final Map ranges = RangeManager.generateAllRanges(1); + Assertions.assertThat(ranges.size()).isEqualTo(1); + Assertions.assertThat(ranges).isEqualTo(expectedResult); + } + + @Test + public void testGenerateAllRangesWithSize3() { + final Map expectedResult = new HashMap<>(); + expectedResult.put( + Bytes32.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), + Bytes32.fromHexString( + "0x5555555555555555555555555555555555555555555555555555555555555555")); + expectedResult.put( + Bytes32.fromHexString("0x5555555555555555555555555555555555555555555555555555555555555556"), + Bytes32.fromHexString( + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab")); + expectedResult.put( + Bytes32.fromHexString("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"), + Bytes32.fromHexString( + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + final Map ranges = RangeManager.generateAllRanges(3); + Assertions.assertThat(ranges.size()).isEqualTo(3); + Assertions.assertThat(ranges).isEqualTo(expectedResult); + } + + @Test + public void testGenerateRangesWithSize3() { + final Map expectedResult = new HashMap<>(); + expectedResult.put( + Bytes32.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), + Bytes32.fromHexString( + "0x5555555555555555555555555555555555555555555555555555555555555555")); + expectedResult.put( + Bytes32.fromHexString("0x5555555555555555555555555555555555555555555555555555555555555556"), + Bytes32.fromHexString( + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab")); + expectedResult.put( + Bytes32.fromHexString("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac"), + Bytes32.fromHexString( + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + final Map ranges = + RangeManager.generateRanges( + Bytes32.fromHexString( + "0x0000000000000000000000000000000000000000000000000000000000000000"), + Bytes32.fromHexString( + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + 3); + Assertions.assertThat(ranges.size()).isEqualTo(3); + Assertions.assertThat(ranges).isEqualTo(expectedResult); + } + + @Test + public void testFindNewBeginElement() { + + final WorldStateStorage worldStateStorage = + new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + + final MerklePatriciaTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorage, 15); + + final RangeStorageEntriesCollector collector = + RangeStorageEntriesCollector.createCollector( + Hash.ZERO, RangeManager.MAX_RANGE, 10, Integer.MAX_VALUE); + final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); + final TreeMap accounts = + (TreeMap) + accountStateTrie.entriesFrom( + root -> + RangeStorageEntriesCollector.collectEntries( + collector, visitor, root, Hash.ZERO)); + + final WorldStateProofProvider worldStateProofProvider = + new WorldStateProofProvider(worldStateStorage); + + // generate the proof + final List proofs = + worldStateProofProvider.getAccountProofRelatedNodes( + Hash.wrap(accountStateTrie.getRootHash()), Hash.ZERO); + proofs.addAll( + worldStateProofProvider.getAccountProofRelatedNodes( + Hash.wrap(accountStateTrie.getRootHash()), accounts.lastKey())); + + final Optional newBeginElementInRange = + RangeManager.findNewBeginElementInRange( + accountStateTrie.getRootHash(), proofs, accounts, RangeManager.MAX_RANGE); + + Assertions.assertThat(newBeginElementInRange) + .contains(Bytes32.leftPad(Bytes.wrap(Bytes.ofUnsignedShort(0x0b)))); + } + + @Test + public void testFindNewBeginElementWhenNothingIsMissing() { + + final WorldStateStorage worldStateStorage = + new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + + final MerklePatriciaTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorage, 15); + + final RangeStorageEntriesCollector collector = + RangeStorageEntriesCollector.createCollector( + Hash.ZERO, RangeManager.MAX_RANGE, 15, Integer.MAX_VALUE); + final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); + final TreeMap accounts = + (TreeMap) + accountStateTrie.entriesFrom( + root -> + RangeStorageEntriesCollector.collectEntries( + collector, visitor, root, Hash.ZERO)); + + final WorldStateProofProvider worldStateProofProvider = + new WorldStateProofProvider(worldStateStorage); + + // generate the proof + final List proofs = + worldStateProofProvider.getAccountProofRelatedNodes( + Hash.wrap(accountStateTrie.getRootHash()), Hash.ZERO); + proofs.addAll( + worldStateProofProvider.getAccountProofRelatedNodes( + Hash.wrap(accountStateTrie.getRootHash()), accounts.lastKey())); + + final Optional newBeginElementInRange = + RangeManager.findNewBeginElementInRange( + accountStateTrie.getRootHash(), proofs, accounts, RangeManager.MAX_RANGE); + + Assertions.assertThat(newBeginElementInRange).isEmpty(); + } +} diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/AbstractSnapMessageData.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/AbstractSnapMessageData.java index 2d8cb19895..fd6ceb700d 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/AbstractSnapMessageData.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/AbstractSnapMessageData.java @@ -23,23 +23,27 @@ import java.util.AbstractMap; import java.util.Map; import java.util.Optional; +import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; public abstract class AbstractSnapMessageData extends AbstractMessageData { - private Optional overrideStateRoot; + @VisibleForTesting + public static final BigInteger SIZE_REQUEST = BigInteger.valueOf(524288); // 512 * 1024 + + private Optional rootHash; public AbstractSnapMessageData(final Bytes data) { super(data); - overrideStateRoot = Optional.empty(); + rootHash = Optional.empty(); } - public Optional getOverrideStateRoot() { - return overrideStateRoot; + public Optional getRootHash() { + return rootHash; } - public void setOverrideStateRoot(final Optional overrideStateRoot) { - this.overrideStateRoot = overrideStateRoot; + public void setRootHash(final Optional rootHash) { + this.rootHash = rootHash; } @Override diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java index 06ce5082c6..baf36deee6 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java @@ -26,6 +26,7 @@ import java.util.function.Supplier; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.bytes.MutableBytes; +import org.apache.tuweni.bytes.MutableBytes32; import org.immutables.value.Value; public class InnerNodeDiscoveryManager extends StoredNodeFactory { @@ -132,6 +133,22 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { return path; } + public static Bytes32 decodePath(final Bytes bytes) { + final MutableBytes32 decoded = MutableBytes32.create(); + final MutableBytes path = MutableBytes.create(Bytes32.SIZE * 2); + path.set(0, bytes); + int decodedPos = 0; + for (int pathPos = 0; pathPos < path.size() - 1; pathPos += 2, decodedPos += 1) { + final byte high = path.get(pathPos); + final byte low = path.get(pathPos + 1); + if ((high & 0xf0) != 0 || (low & 0xf0) != 0) { + throw new IllegalArgumentException("Invalid path: contains elements larger than a nibble"); + } + decoded.set(decodedPos, (byte) (high << 4 | (low & 0xff))); + } + return decoded; + } + @Value.Immutable public interface InnerNode { Bytes location(); diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/MerkleTrieException.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/MerkleTrieException.java index d671f75dbe..3ad6d766e2 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/MerkleTrieException.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/MerkleTrieException.java @@ -14,17 +14,37 @@ */ package org.hyperledger.besu.ethereum.trie; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + /** * This exception is thrown when there is an issue retrieving or decoding values from {@link * MerkleStorage}. */ public class MerkleTrieException extends RuntimeException { + private Bytes32 hash; + private Bytes location; + public MerkleTrieException(final String message) { super(message); } + public MerkleTrieException(final String message, final Bytes32 hash, final Bytes location) { + super(message); + this.hash = hash; + this.location = location; + } + public MerkleTrieException(final String message, final Exception cause) { super(message, cause); } + + public Bytes32 getHash() { + return hash; + } + + public Bytes getLocation() { + return location; + } } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNode.java index 5bf15e5a75..1c3b490bd3 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNode.java @@ -123,7 +123,9 @@ class StoredNode implements Node { "Unable to load trie node value for hash " + hash + " location " - + location)); + + location, + hash, + location)); } return loaded;