mirror of https://github.com/hyperledger/besu
Karim Taam
11 months ago
commit
ccdb980692
@ -0,0 +1,200 @@ |
|||||||
|
/* |
||||||
|
* Copyright contributors to Hyperledger Besu. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||||
|
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||||
|
* specific language governing permissions and limitations under the License. |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: Apache-2.0 |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.hyperledger.besu.tests.acceptance.dsl; |
||||||
|
|
||||||
|
import static java.nio.charset.StandardCharsets.UTF_8; |
||||||
|
import static org.assertj.core.api.Assertions.assertThat; |
||||||
|
|
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.account.Accounts; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.blockchain.Blockchain; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.admin.AdminConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.bft.BftConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.clique.CliqueConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.eth.EthConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.login.LoginConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.net.NetConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.perm.PermissioningConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.priv.PrivConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.process.ExitedWithCode; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.txpool.TxPoolConditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.condition.web3.Web3Conditions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.contract.ContractVerifier; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.node.Node; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.Cluster; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.BesuNodeFactory; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.permissioning.PermissionedNodeBuilder; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.account.AccountTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.admin.AdminTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.bft.BftTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.clique.CliqueTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.contract.ContractTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.miner.MinerTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.net.NetTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.perm.PermissioningTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.privacy.PrivacyTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.txpool.TxPoolTransactions; |
||||||
|
import org.hyperledger.besu.tests.acceptance.dsl.transaction.web3.Web3Transactions; |
||||||
|
|
||||||
|
import java.io.BufferedReader; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.InputStreamReader; |
||||||
|
import java.math.BigInteger; |
||||||
|
import java.util.concurrent.ExecutorService; |
||||||
|
import java.util.concurrent.Executors; |
||||||
|
|
||||||
|
import org.apache.logging.log4j.ThreadContext; |
||||||
|
import org.junit.jupiter.api.AfterEach; |
||||||
|
import org.junit.jupiter.api.BeforeEach; |
||||||
|
import org.junit.jupiter.api.TestInfo; |
||||||
|
import org.junit.jupiter.api.extension.ExtendWith; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** |
||||||
|
* Superclass for acceptance tests. For now (transition to junit5 is ongoing) this class supports |
||||||
|
* junit5 format. Once the transition is complete, this class can be removed and recombined with |
||||||
|
* AcceptanceTestBase (original). |
||||||
|
*/ |
||||||
|
@ExtendWith(AcceptanceTestBaseTestWatcher.class) |
||||||
|
public class AcceptanceTestBaseJunit5 { |
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(AcceptanceTestBaseJunit5.class); |
||||||
|
|
||||||
|
protected final Accounts accounts; |
||||||
|
protected final AccountTransactions accountTransactions; |
||||||
|
protected final AdminConditions admin; |
||||||
|
protected final AdminTransactions adminTransactions; |
||||||
|
protected final Blockchain blockchain; |
||||||
|
protected final CliqueConditions clique; |
||||||
|
protected final CliqueTransactions cliqueTransactions; |
||||||
|
protected final Cluster cluster; |
||||||
|
protected final ContractVerifier contractVerifier; |
||||||
|
protected final ContractTransactions contractTransactions; |
||||||
|
protected final EthConditions eth; |
||||||
|
protected final EthTransactions ethTransactions; |
||||||
|
protected final BftTransactions bftTransactions; |
||||||
|
protected final BftConditions bft; |
||||||
|
protected final LoginConditions login; |
||||||
|
protected final NetConditions net; |
||||||
|
protected final BesuNodeFactory besu; |
||||||
|
protected final PermissioningConditions perm; |
||||||
|
protected final PermissionedNodeBuilder permissionedNodeBuilder; |
||||||
|
protected final PermissioningTransactions permissioningTransactions; |
||||||
|
protected final MinerTransactions minerTransactions; |
||||||
|
protected final Web3Conditions web3; |
||||||
|
protected final PrivConditions priv; |
||||||
|
protected final PrivacyTransactions privacyTransactions; |
||||||
|
protected final TxPoolConditions txPoolConditions; |
||||||
|
protected final TxPoolTransactions txPoolTransactions; |
||||||
|
protected final ExitedWithCode exitedSuccessfully; |
||||||
|
|
||||||
|
private final ExecutorService outputProcessorExecutor = Executors.newCachedThreadPool(); |
||||||
|
|
||||||
|
protected AcceptanceTestBaseJunit5() { |
||||||
|
ethTransactions = new EthTransactions(); |
||||||
|
accounts = new Accounts(ethTransactions); |
||||||
|
adminTransactions = new AdminTransactions(); |
||||||
|
cliqueTransactions = new CliqueTransactions(); |
||||||
|
bftTransactions = new BftTransactions(); |
||||||
|
accountTransactions = new AccountTransactions(accounts); |
||||||
|
permissioningTransactions = new PermissioningTransactions(); |
||||||
|
privacyTransactions = new PrivacyTransactions(); |
||||||
|
contractTransactions = new ContractTransactions(); |
||||||
|
minerTransactions = new MinerTransactions(); |
||||||
|
|
||||||
|
blockchain = new Blockchain(ethTransactions); |
||||||
|
clique = new CliqueConditions(ethTransactions, cliqueTransactions); |
||||||
|
eth = new EthConditions(ethTransactions); |
||||||
|
bft = new BftConditions(bftTransactions); |
||||||
|
login = new LoginConditions(); |
||||||
|
net = new NetConditions(new NetTransactions()); |
||||||
|
cluster = new Cluster(net); |
||||||
|
perm = new PermissioningConditions(permissioningTransactions); |
||||||
|
priv = new PrivConditions(privacyTransactions); |
||||||
|
admin = new AdminConditions(adminTransactions); |
||||||
|
web3 = new Web3Conditions(new Web3Transactions()); |
||||||
|
besu = new BesuNodeFactory(); |
||||||
|
txPoolTransactions = new TxPoolTransactions(); |
||||||
|
txPoolConditions = new TxPoolConditions(txPoolTransactions); |
||||||
|
contractVerifier = new ContractVerifier(accounts.getPrimaryBenefactor()); |
||||||
|
permissionedNodeBuilder = new PermissionedNodeBuilder(); |
||||||
|
exitedSuccessfully = new ExitedWithCode(0); |
||||||
|
} |
||||||
|
|
||||||
|
@BeforeEach |
||||||
|
public void setUp(final TestInfo testInfo) { |
||||||
|
// log4j is configured to create a file per test
|
||||||
|
// build/acceptanceTestLogs/${ctx:class}.${ctx:test}.log
|
||||||
|
ThreadContext.put("class", this.getClass().getSimpleName()); |
||||||
|
ThreadContext.put("test", testInfo.getTestMethod().get().getName()); |
||||||
|
} |
||||||
|
|
||||||
|
@AfterEach |
||||||
|
public void tearDownAcceptanceTestBase() { |
||||||
|
reportMemory(); |
||||||
|
cluster.close(); |
||||||
|
} |
||||||
|
|
||||||
|
public void reportMemory() { |
||||||
|
String os = System.getProperty("os.name"); |
||||||
|
String[] command = null; |
||||||
|
if (os.contains("Linux")) { |
||||||
|
command = new String[] {"/usr/bin/top", "-n", "1", "-o", "%MEM", "-b", "-c", "-w", "180"}; |
||||||
|
} |
||||||
|
if (os.contains("Mac")) { |
||||||
|
command = new String[] {"/usr/bin/top", "-l", "1", "-o", "mem", "-n", "20"}; |
||||||
|
} |
||||||
|
if (command != null) { |
||||||
|
LOG.info("Memory usage at end of test:"); |
||||||
|
final ProcessBuilder processBuilder = |
||||||
|
new ProcessBuilder(command) |
||||||
|
.redirectErrorStream(true) |
||||||
|
.redirectInput(ProcessBuilder.Redirect.INHERIT); |
||||||
|
try { |
||||||
|
final Process memInfoProcess = processBuilder.start(); |
||||||
|
outputProcessorExecutor.execute(() -> printOutput(memInfoProcess)); |
||||||
|
memInfoProcess.waitFor(); |
||||||
|
LOG.debug("Memory info process exited with code {}", memInfoProcess.exitValue()); |
||||||
|
} catch (final Exception e) { |
||||||
|
LOG.warn("Error running memory information process", e); |
||||||
|
} |
||||||
|
} else { |
||||||
|
LOG.info("Don't know how to report memory for OS {}", os); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private void printOutput(final Process process) { |
||||||
|
try (final BufferedReader in = |
||||||
|
new BufferedReader(new InputStreamReader(process.getInputStream(), UTF_8))) { |
||||||
|
String line = in.readLine(); |
||||||
|
while (line != null) { |
||||||
|
LOG.info(line); |
||||||
|
line = in.readLine(); |
||||||
|
} |
||||||
|
} catch (final IOException e) { |
||||||
|
LOG.warn("Failed to read output from memory information process: ", e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
protected void waitForBlockHeight(final Node node, final long blockchainHeight) { |
||||||
|
WaitUtils.waitFor( |
||||||
|
120, |
||||||
|
() -> |
||||||
|
assertThat(node.execute(ethTransactions.blockNumber())) |
||||||
|
.isGreaterThanOrEqualTo(BigInteger.valueOf(blockchainHeight))); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,58 @@ |
|||||||
|
/* |
||||||
|
* Copyright contributors to Hyperledger Besu. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||||
|
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||||
|
* specific language governing permissions and limitations under the License. |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: Apache-2.0 |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.hyperledger.besu.tests.acceptance.dsl; |
||||||
|
|
||||||
|
import java.io.File; |
||||||
|
|
||||||
|
import org.junit.jupiter.api.extension.ExtensionContext; |
||||||
|
import org.junit.jupiter.api.extension.TestWatcher; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
public class AcceptanceTestBaseTestWatcher implements TestWatcher { |
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(AcceptanceTestBaseTestWatcher.class); |
||||||
|
|
||||||
|
@Override |
||||||
|
public void testFailed(final ExtensionContext extensionContext, final Throwable e) { |
||||||
|
// add the result at the end of the log, so it is self-sufficient
|
||||||
|
LOG.error( |
||||||
|
"=========================================================================================="); |
||||||
|
LOG.error("Test failed. Reported Throwable at the point of failure:", e); |
||||||
|
LOG.error(e.getMessage()); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public void testSuccessful(final ExtensionContext extensionContext) { |
||||||
|
// if so configured, delete logs of successful tests
|
||||||
|
if (!Boolean.getBoolean("acctests.keepLogsOfPassingTests")) { |
||||||
|
try { |
||||||
|
// log4j is configured to create a file per test
|
||||||
|
// build/acceptanceTestLogs/${ctx:class}.${ctx:test}.log
|
||||||
|
String pathname = |
||||||
|
"build/acceptanceTestLogs/" |
||||||
|
+ extensionContext.getTestClass().get().getSimpleName() |
||||||
|
+ "." |
||||||
|
+ extensionContext.getTestMethod().get().getName() |
||||||
|
+ ".log"; |
||||||
|
LOG.info("Test successful, deleting log at {}", pathname); |
||||||
|
final File file = new File(pathname); |
||||||
|
file.delete(); |
||||||
|
} catch (final Exception e) { |
||||||
|
LOG.error("could not delete test file", e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,361 @@ |
|||||||
|
/* |
||||||
|
* Copyright contributors to Hyperledger Besu. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||||
|
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||||
|
* specific language governing permissions and limitations under the License. |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: Apache-2.0 |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.hyperledger.besu.cli.subcommands.storage; |
||||||
|
|
||||||
|
import static com.google.common.base.Preconditions.checkArgument; |
||||||
|
import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; |
||||||
|
|
||||||
|
import org.hyperledger.besu.datatypes.Hash; |
||||||
|
import org.hyperledger.besu.ethereum.chain.Blockchain; |
||||||
|
import org.hyperledger.besu.ethereum.chain.MutableBlockchain; |
||||||
|
import org.hyperledger.besu.ethereum.core.BlockHeader; |
||||||
|
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||||
|
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; |
||||||
|
|
||||||
|
import java.io.File; |
||||||
|
import java.io.FileInputStream; |
||||||
|
import java.io.FileOutputStream; |
||||||
|
import java.io.IOException; |
||||||
|
import java.io.ObjectInputStream; |
||||||
|
import java.io.ObjectOutputStream; |
||||||
|
import java.io.PrintWriter; |
||||||
|
import java.nio.file.Path; |
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.IdentityHashMap; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Optional; |
||||||
|
import java.util.concurrent.atomic.AtomicInteger; |
||||||
|
|
||||||
|
import org.apache.tuweni.bytes.Bytes32; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** Helper class for counting and pruning trie logs */ |
||||||
|
public class TrieLogHelper { |
||||||
|
private static final String TRIE_LOG_FILE = "trieLogsToRetain"; |
||||||
|
private static final long BATCH_SIZE = 20_000; |
||||||
|
private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000; |
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); |
||||||
|
|
||||||
|
static void prune( |
||||||
|
final DataStorageConfiguration config, |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||||
|
final MutableBlockchain blockchain, |
||||||
|
final Path dataDirectoryPath) { |
||||||
|
final String batchFileNameBase = |
||||||
|
dataDirectoryPath.resolve(DATABASE_PATH).resolve(TRIE_LOG_FILE).toString(); |
||||||
|
|
||||||
|
validatePruneConfiguration(config); |
||||||
|
|
||||||
|
final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); |
||||||
|
|
||||||
|
final long chainHeight = blockchain.getChainHeadBlockNumber(); |
||||||
|
|
||||||
|
final long lastBlockNumberToRetainTrieLogsFor = chainHeight - layersToRetain + 1; |
||||||
|
|
||||||
|
if (!validPruneRequirements(blockchain, chainHeight, lastBlockNumberToRetainTrieLogsFor)) { |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
final long numberOfBatches = calculateNumberofBatches(layersToRetain); |
||||||
|
|
||||||
|
processTrieLogBatches( |
||||||
|
rootWorldStateStorage, |
||||||
|
blockchain, |
||||||
|
chainHeight, |
||||||
|
lastBlockNumberToRetainTrieLogsFor, |
||||||
|
numberOfBatches, |
||||||
|
batchFileNameBase); |
||||||
|
|
||||||
|
if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { |
||||||
|
deleteFiles(batchFileNameBase, numberOfBatches); |
||||||
|
LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); |
||||||
|
} else { |
||||||
|
LOG.error("Prune failed. Re-run the subcommand to load the trie logs from file."); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static void processTrieLogBatches( |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||||
|
final MutableBlockchain blockchain, |
||||||
|
final long chainHeight, |
||||||
|
final long lastBlockNumberToRetainTrieLogsFor, |
||||||
|
final long numberOfBatches, |
||||||
|
final String batchFileNameBase) { |
||||||
|
|
||||||
|
for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { |
||||||
|
|
||||||
|
final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); |
||||||
|
|
||||||
|
final long lastBlockOfBatch = |
||||||
|
Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); |
||||||
|
|
||||||
|
final List<Hash> trieLogKeys = |
||||||
|
getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch); |
||||||
|
|
||||||
|
saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys); |
||||||
|
} |
||||||
|
|
||||||
|
LOG.info("Clear trie logs..."); |
||||||
|
rootWorldStateStorage.clearTrieLog(); |
||||||
|
|
||||||
|
for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { |
||||||
|
restoreTrieLogBatches(rootWorldStateStorage, batchNumber, batchFileNameBase); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static void saveTrieLogBatches( |
||||||
|
final String batchFileNameBase, |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||||
|
final long batchNumber, |
||||||
|
final List<Hash> trieLogKeys) { |
||||||
|
|
||||||
|
LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); |
||||||
|
|
||||||
|
try { |
||||||
|
saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); |
||||||
|
} catch (IOException e) { |
||||||
|
LOG.error("Error saving trie logs to file: {}", e.getMessage()); |
||||||
|
throw new RuntimeException(e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static void restoreTrieLogBatches( |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||||
|
final long batchNumber, |
||||||
|
final String batchFileNameBase) { |
||||||
|
|
||||||
|
try { |
||||||
|
LOG.info("Restoring trie logs retained from batch {}...", batchNumber); |
||||||
|
recreateTrieLogs(rootWorldStateStorage, batchNumber, batchFileNameBase); |
||||||
|
} catch (IOException e) { |
||||||
|
LOG.error("Error recreating trie logs from batch {}: {}", batchNumber, e.getMessage()); |
||||||
|
throw new RuntimeException(e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { |
||||||
|
|
||||||
|
LOG.info("Deleting files..."); |
||||||
|
|
||||||
|
for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { |
||||||
|
File file = new File(batchFileNameBase + "-" + batchNumber); |
||||||
|
if (file.exists()) { |
||||||
|
file.delete(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static List<Hash> getTrieLogKeysForBlocks( |
||||||
|
final MutableBlockchain blockchain, |
||||||
|
final long firstBlockOfBatch, |
||||||
|
final long lastBlockOfBatch) { |
||||||
|
final List<Hash> trieLogKeys = new ArrayList<>(); |
||||||
|
for (long i = firstBlockOfBatch; i >= lastBlockOfBatch; i--) { |
||||||
|
final Optional<BlockHeader> header = blockchain.getBlockHeader(i); |
||||||
|
header.ifPresentOrElse( |
||||||
|
blockHeader -> trieLogKeys.add(blockHeader.getHash()), |
||||||
|
() -> LOG.error("Error retrieving block")); |
||||||
|
} |
||||||
|
return trieLogKeys; |
||||||
|
} |
||||||
|
|
||||||
|
private static long calculateNumberofBatches(final long layersToRetain) { |
||||||
|
return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1); |
||||||
|
} |
||||||
|
|
||||||
|
private static boolean validPruneRequirements( |
||||||
|
final MutableBlockchain blockchain, |
||||||
|
final long chainHeight, |
||||||
|
final long lastBlockNumberToRetainTrieLogsFor) { |
||||||
|
if (lastBlockNumberToRetainTrieLogsFor < 0) { |
||||||
|
throw new IllegalArgumentException( |
||||||
|
"Trying to retain more trie logs than chain length (" |
||||||
|
+ chainHeight |
||||||
|
+ "), skipping pruning"); |
||||||
|
} |
||||||
|
|
||||||
|
final Optional<Hash> finalizedBlockHash = blockchain.getFinalized(); |
||||||
|
|
||||||
|
if (finalizedBlockHash.isEmpty()) { |
||||||
|
throw new RuntimeException("No finalized block present, can't safely run trie log prune"); |
||||||
|
} else { |
||||||
|
final Hash finalizedHash = finalizedBlockHash.get(); |
||||||
|
final Optional<BlockHeader> finalizedBlockHeader = blockchain.getBlockHeader(finalizedHash); |
||||||
|
if (finalizedBlockHeader.isPresent() |
||||||
|
&& finalizedBlockHeader.get().getNumber() < lastBlockNumberToRetainTrieLogsFor) { |
||||||
|
throw new IllegalArgumentException( |
||||||
|
"Trying to prune more layers than the finalized block height, skipping pruning"); |
||||||
|
} |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
private static void recreateTrieLogs( |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||||
|
final long batchNumber, |
||||||
|
final String batchFileNameBase) |
||||||
|
throws IOException { |
||||||
|
// process in chunk to avoid OOM
|
||||||
|
|
||||||
|
IdentityHashMap<byte[], byte[]> trieLogsToRetain = |
||||||
|
readTrieLogsFromFile(batchFileNameBase, batchNumber); |
||||||
|
final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; |
||||||
|
List<byte[]> keys = new ArrayList<>(trieLogsToRetain.keySet()); |
||||||
|
|
||||||
|
for (int startIndex = 0; startIndex < keys.size(); startIndex += chunkSize) { |
||||||
|
processTransactionChunk(startIndex, chunkSize, keys, trieLogsToRetain, rootWorldStateStorage); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static void processTransactionChunk( |
||||||
|
final int startIndex, |
||||||
|
final int chunkSize, |
||||||
|
final List<byte[]> keys, |
||||||
|
final IdentityHashMap<byte[], byte[]> trieLogsToRetain, |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { |
||||||
|
|
||||||
|
var updater = rootWorldStateStorage.updater(); |
||||||
|
int endIndex = Math.min(startIndex + chunkSize, keys.size()); |
||||||
|
|
||||||
|
for (int i = startIndex; i < endIndex; i++) { |
||||||
|
byte[] key = keys.get(i); |
||||||
|
byte[] value = trieLogsToRetain.get(key); |
||||||
|
updater.getTrieLogStorageTransaction().put(key, value); |
||||||
|
LOG.info("Key({}): {}", i, Bytes32.wrap(key).toShortHexString()); |
||||||
|
} |
||||||
|
|
||||||
|
updater.getTrieLogStorageTransaction().commit(); |
||||||
|
} |
||||||
|
|
||||||
|
private static void validatePruneConfiguration(final DataStorageConfiguration config) { |
||||||
|
checkArgument( |
||||||
|
config.getUnstable().getBonsaiTrieLogRetentionThreshold() |
||||||
|
>= config.getBonsaiMaxLayersToLoad(), |
||||||
|
String.format( |
||||||
|
"--Xbonsai-trie-log-retention-threshold minimum value is %d", |
||||||
|
config.getBonsaiMaxLayersToLoad())); |
||||||
|
checkArgument( |
||||||
|
config.getUnstable().getBonsaiTrieLogPruningLimit() > 0, |
||||||
|
String.format( |
||||||
|
"--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", |
||||||
|
config.getUnstable().getBonsaiTrieLogPruningLimit())); |
||||||
|
checkArgument( |
||||||
|
config.getUnstable().getBonsaiTrieLogPruningLimit() |
||||||
|
> config.getUnstable().getBonsaiTrieLogRetentionThreshold(), |
||||||
|
String.format( |
||||||
|
"--Xbonsai-trie-log-pruning-limit=%d must greater than --Xbonsai-trie-log-retention-threshold=%d", |
||||||
|
config.getUnstable().getBonsaiTrieLogPruningLimit(), |
||||||
|
config.getUnstable().getBonsaiTrieLogRetentionThreshold())); |
||||||
|
} |
||||||
|
|
||||||
|
private static void saveTrieLogsInFile( |
||||||
|
final List<Hash> trieLogsKeys, |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||||
|
final long batchNumber, |
||||||
|
final String batchFileNameBase) |
||||||
|
throws IOException { |
||||||
|
|
||||||
|
File file = new File(batchFileNameBase + "-" + batchNumber); |
||||||
|
if (file.exists()) { |
||||||
|
LOG.error("File already exists, skipping file creation"); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(file)) { |
||||||
|
ObjectOutputStream oos = new ObjectOutputStream(fos); |
||||||
|
oos.writeObject(getTrieLogs(trieLogsKeys, rootWorldStateStorage)); |
||||||
|
} catch (IOException e) { |
||||||
|
LOG.error(e.getMessage()); |
||||||
|
throw new RuntimeException(e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@SuppressWarnings("unchecked") |
||||||
|
private static IdentityHashMap<byte[], byte[]> readTrieLogsFromFile( |
||||||
|
final String batchFileNameBase, final long batchNumber) { |
||||||
|
|
||||||
|
IdentityHashMap<byte[], byte[]> trieLogs; |
||||||
|
try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); |
||||||
|
ObjectInputStream ois = new ObjectInputStream(fis)) { |
||||||
|
|
||||||
|
trieLogs = (IdentityHashMap<byte[], byte[]>) ois.readObject(); |
||||||
|
|
||||||
|
} catch (IOException | ClassNotFoundException e) { |
||||||
|
|
||||||
|
LOG.error(e.getMessage()); |
||||||
|
throw new RuntimeException(e); |
||||||
|
} |
||||||
|
|
||||||
|
return trieLogs; |
||||||
|
} |
||||||
|
|
||||||
|
private static IdentityHashMap<byte[], byte[]> getTrieLogs( |
||||||
|
final List<Hash> trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { |
||||||
|
IdentityHashMap<byte[], byte[]> trieLogsToRetain = new IdentityHashMap<>(); |
||||||
|
|
||||||
|
LOG.info("Obtaining trielogs from db, this may take a few minutes..."); |
||||||
|
trieLogKeys.forEach( |
||||||
|
hash -> |
||||||
|
rootWorldStateStorage |
||||||
|
.getTrieLog(hash) |
||||||
|
.ifPresent(trieLog -> trieLogsToRetain.put(hash.toArrayUnsafe(), trieLog))); |
||||||
|
return trieLogsToRetain; |
||||||
|
} |
||||||
|
|
||||||
|
static TrieLogCount getCount( |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||||
|
final int limit, |
||||||
|
final Blockchain blockchain) { |
||||||
|
final AtomicInteger total = new AtomicInteger(); |
||||||
|
final AtomicInteger canonicalCount = new AtomicInteger(); |
||||||
|
final AtomicInteger forkCount = new AtomicInteger(); |
||||||
|
final AtomicInteger orphanCount = new AtomicInteger(); |
||||||
|
rootWorldStateStorage |
||||||
|
.streamTrieLogKeys(limit) |
||||||
|
.map(Bytes32::wrap) |
||||||
|
.map(Hash::wrap) |
||||||
|
.forEach( |
||||||
|
hash -> { |
||||||
|
total.getAndIncrement(); |
||||||
|
blockchain |
||||||
|
.getBlockHeader(hash) |
||||||
|
.ifPresentOrElse( |
||||||
|
(header) -> { |
||||||
|
long number = header.getNumber(); |
||||||
|
final Optional<BlockHeader> headerByNumber = |
||||||
|
blockchain.getBlockHeader(number); |
||||||
|
if (headerByNumber.isPresent() |
||||||
|
&& headerByNumber.get().getHash().equals(hash)) { |
||||||
|
canonicalCount.getAndIncrement(); |
||||||
|
} else { |
||||||
|
forkCount.getAndIncrement(); |
||||||
|
} |
||||||
|
}, |
||||||
|
orphanCount::getAndIncrement); |
||||||
|
}); |
||||||
|
|
||||||
|
return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); |
||||||
|
} |
||||||
|
|
||||||
|
static void printCount(final PrintWriter out, final TrieLogCount count) { |
||||||
|
out.printf( |
||||||
|
"trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", |
||||||
|
count.total, count.canonicalCount, count.forkCount, count.orphanCount); |
||||||
|
} |
||||||
|
|
||||||
|
record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} |
||||||
|
} |
||||||
@ -0,0 +1,147 @@ |
|||||||
|
/* |
||||||
|
* Copyright Hyperledger Besu Contributors. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||||
|
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||||
|
* specific language governing permissions and limitations under the License. |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: Apache-2.0 |
||||||
|
*/ |
||||||
|
package org.hyperledger.besu.cli.subcommands.storage; |
||||||
|
|
||||||
|
import static com.google.common.base.Preconditions.checkArgument; |
||||||
|
import static com.google.common.base.Preconditions.checkNotNull; |
||||||
|
|
||||||
|
import org.hyperledger.besu.cli.util.VersionProvider; |
||||||
|
import org.hyperledger.besu.controller.BesuController; |
||||||
|
import org.hyperledger.besu.ethereum.chain.MutableBlockchain; |
||||||
|
import org.hyperledger.besu.ethereum.storage.StorageProvider; |
||||||
|
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||||
|
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; |
||||||
|
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; |
||||||
|
import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; |
||||||
|
|
||||||
|
import java.io.PrintWriter; |
||||||
|
import java.nio.file.Path; |
||||||
|
import java.nio.file.Paths; |
||||||
|
|
||||||
|
import org.apache.logging.log4j.Level; |
||||||
|
import org.apache.logging.log4j.core.config.Configurator; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import picocli.CommandLine; |
||||||
|
import picocli.CommandLine.Command; |
||||||
|
import picocli.CommandLine.ParentCommand; |
||||||
|
|
||||||
|
/** The Trie Log subcommand. */ |
||||||
|
@Command( |
||||||
|
name = "x-trie-log", |
||||||
|
description = "Manipulate trie logs", |
||||||
|
mixinStandardHelpOptions = true, |
||||||
|
versionProvider = VersionProvider.class, |
||||||
|
subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class}) |
||||||
|
public class TrieLogSubCommand implements Runnable { |
||||||
|
|
||||||
|
@SuppressWarnings("UnusedVariable") |
||||||
|
@ParentCommand |
||||||
|
private static StorageSubCommand parentCommand; |
||||||
|
|
||||||
|
@SuppressWarnings("unused") |
||||||
|
@CommandLine.Spec |
||||||
|
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
|
||||||
|
|
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
final PrintWriter out = spec.commandLine().getOut(); |
||||||
|
spec.commandLine().usage(out); |
||||||
|
} |
||||||
|
|
||||||
|
private static BesuController createBesuController() { |
||||||
|
return parentCommand.parentCommand.buildController(); |
||||||
|
} |
||||||
|
|
||||||
|
@Command( |
||||||
|
name = "count", |
||||||
|
description = "This command counts all the trie logs", |
||||||
|
mixinStandardHelpOptions = true, |
||||||
|
versionProvider = VersionProvider.class) |
||||||
|
static class CountTrieLog implements Runnable { |
||||||
|
|
||||||
|
@SuppressWarnings("unused") |
||||||
|
@ParentCommand |
||||||
|
private TrieLogSubCommand parentCommand; |
||||||
|
|
||||||
|
@SuppressWarnings("unused") |
||||||
|
@CommandLine.Spec |
||||||
|
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
|
||||||
|
|
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
TrieLogContext context = getTrieLogContext(); |
||||||
|
|
||||||
|
final PrintWriter out = spec.commandLine().getOut(); |
||||||
|
|
||||||
|
out.println("Counting trie logs..."); |
||||||
|
TrieLogHelper.printCount( |
||||||
|
out, |
||||||
|
TrieLogHelper.getCount( |
||||||
|
context.rootWorldStateStorage, Integer.MAX_VALUE, context.blockchain)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Command( |
||||||
|
name = "prune", |
||||||
|
description = |
||||||
|
"This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", |
||||||
|
mixinStandardHelpOptions = true, |
||||||
|
versionProvider = VersionProvider.class) |
||||||
|
static class PruneTrieLog implements Runnable { |
||||||
|
|
||||||
|
@SuppressWarnings("unused") |
||||||
|
@ParentCommand |
||||||
|
private TrieLogSubCommand parentCommand; |
||||||
|
|
||||||
|
@SuppressWarnings("unused") |
||||||
|
@CommandLine.Spec |
||||||
|
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
|
||||||
|
|
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
TrieLogContext context = getTrieLogContext(); |
||||||
|
final Path dataDirectoryPath = |
||||||
|
Paths.get( |
||||||
|
TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); |
||||||
|
TrieLogHelper.prune( |
||||||
|
context.config(), |
||||||
|
context.rootWorldStateStorage(), |
||||||
|
context.blockchain(), |
||||||
|
dataDirectoryPath); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
record TrieLogContext( |
||||||
|
DataStorageConfiguration config, |
||||||
|
BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||||
|
MutableBlockchain blockchain) {} |
||||||
|
|
||||||
|
private static TrieLogContext getTrieLogContext() { |
||||||
|
Configurator.setLevel(LoggerFactory.getLogger(TrieLogPruner.class).getName(), Level.DEBUG); |
||||||
|
checkNotNull(parentCommand); |
||||||
|
BesuController besuController = createBesuController(); |
||||||
|
final DataStorageConfiguration config = besuController.getDataStorageConfiguration(); |
||||||
|
checkArgument( |
||||||
|
DataStorageFormat.BONSAI.equals(config.getDataStorageFormat()), |
||||||
|
"Subcommand only works with data-storage-format=BONSAI"); |
||||||
|
|
||||||
|
final StorageProvider storageProvider = besuController.getStorageProvider(); |
||||||
|
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage = |
||||||
|
(BonsaiWorldStateKeyValueStorage) |
||||||
|
storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI); |
||||||
|
final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain(); |
||||||
|
return new TrieLogContext(config, rootWorldStateStorage, blockchain); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,34 @@ |
|||||||
|
/* |
||||||
|
* Copyright Hyperledger Besu contributors. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||||
|
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||||
|
* specific language governing permissions and limitations under the License. |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: Apache-2.0 |
||||||
|
* |
||||||
|
*/ |
||||||
|
package org.hyperledger.besu; |
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat; |
||||||
|
|
||||||
|
import org.hyperledger.besu.ethereum.forkid.ForkId; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
import org.apache.tuweni.bytes.Bytes; |
||||||
|
import org.junit.jupiter.api.Test; |
||||||
|
|
||||||
|
public class RawForkIdTest { |
||||||
|
@Test |
||||||
|
public void testFromRaw() { |
||||||
|
final ForkId forkId = new ForkId(Bytes.ofUnsignedInt(0xfe3366e7L), 1735371L); |
||||||
|
final List<List<Bytes>> forkIdAsBytesList = List.of(forkId.getForkIdAsBytesList()); |
||||||
|
assertThat(ForkId.fromRawForkId(forkIdAsBytesList).get()).isEqualTo(forkId); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,265 @@ |
|||||||
|
/* |
||||||
|
* Copyright contributors to Hyperledger Besu. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||||
|
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||||
|
* specific language governing permissions and limitations under the License. |
||||||
|
* |
||||||
|
* SPDX-License-Identifier: Apache-2.0 |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.hyperledger.besu.cli.subcommands.storage; |
||||||
|
|
||||||
|
import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; |
||||||
|
import static org.junit.jupiter.api.Assertions.assertArrayEquals; |
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals; |
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows; |
||||||
|
import static org.mockito.ArgumentMatchers.any; |
||||||
|
import static org.mockito.Mockito.when; |
||||||
|
|
||||||
|
import org.hyperledger.besu.datatypes.Hash; |
||||||
|
import org.hyperledger.besu.ethereum.chain.MutableBlockchain; |
||||||
|
import org.hyperledger.besu.ethereum.core.BlockHeader; |
||||||
|
import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; |
||||||
|
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; |
||||||
|
import org.hyperledger.besu.ethereum.storage.StorageProvider; |
||||||
|
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||||
|
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; |
||||||
|
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; |
||||||
|
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.nio.file.Files; |
||||||
|
import java.nio.file.Path; |
||||||
|
import java.util.Optional; |
||||||
|
|
||||||
|
import org.apache.tuweni.bytes.Bytes; |
||||||
|
import org.junit.jupiter.api.AfterEach; |
||||||
|
import org.junit.jupiter.api.BeforeAll; |
||||||
|
import org.junit.jupiter.api.BeforeEach; |
||||||
|
import org.junit.jupiter.api.Test; |
||||||
|
import org.junit.jupiter.api.extension.ExtendWith; |
||||||
|
import org.junit.jupiter.api.io.TempDir; |
||||||
|
import org.mockito.Mock; |
||||||
|
import org.mockito.junit.jupiter.MockitoExtension; |
||||||
|
|
||||||
|
@ExtendWith(MockitoExtension.class) |
||||||
|
class TrieLogHelperTest { |
||||||
|
|
||||||
|
private static final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); |
||||||
|
private static BonsaiWorldStateKeyValueStorage inMemoryWorldState; |
||||||
|
|
||||||
|
@Mock private MutableBlockchain blockchain; |
||||||
|
|
||||||
|
@TempDir static Path dataDir; |
||||||
|
|
||||||
|
Path test; |
||||||
|
static BlockHeader blockHeader1; |
||||||
|
static BlockHeader blockHeader2; |
||||||
|
static BlockHeader blockHeader3; |
||||||
|
static BlockHeader blockHeader4; |
||||||
|
static BlockHeader blockHeader5; |
||||||
|
|
||||||
|
@BeforeAll |
||||||
|
public static void setup() throws IOException { |
||||||
|
|
||||||
|
blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); |
||||||
|
blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); |
||||||
|
blockHeader3 = new BlockHeaderTestFixture().number(3).buildHeader(); |
||||||
|
blockHeader4 = new BlockHeaderTestFixture().number(4).buildHeader(); |
||||||
|
blockHeader5 = new BlockHeaderTestFixture().number(5).buildHeader(); |
||||||
|
|
||||||
|
inMemoryWorldState = |
||||||
|
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); |
||||||
|
|
||||||
|
var updater = inMemoryWorldState.updater(); |
||||||
|
updater |
||||||
|
.getTrieLogStorageTransaction() |
||||||
|
.put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe()); |
||||||
|
updater |
||||||
|
.getTrieLogStorageTransaction() |
||||||
|
.put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe()); |
||||||
|
updater |
||||||
|
.getTrieLogStorageTransaction() |
||||||
|
.put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe()); |
||||||
|
updater |
||||||
|
.getTrieLogStorageTransaction() |
||||||
|
.put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe()); |
||||||
|
updater |
||||||
|
.getTrieLogStorageTransaction() |
||||||
|
.put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); |
||||||
|
updater.getTrieLogStorageTransaction().commit(); |
||||||
|
} |
||||||
|
|
||||||
|
@BeforeEach |
||||||
|
void createDirectory() throws IOException { |
||||||
|
Files.createDirectories(dataDir.resolve("database")); |
||||||
|
} |
||||||
|
|
||||||
|
@AfterEach |
||||||
|
void deleteDirectory() throws IOException { |
||||||
|
Files.deleteIfExists(dataDir.resolve("database")); |
||||||
|
} |
||||||
|
|
||||||
|
void mockBlockchainBase() { |
||||||
|
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); |
||||||
|
when(blockchain.getFinalized()).thenReturn(Optional.of(blockHeader3.getBlockHash())); |
||||||
|
when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(blockHeader3)); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void prune() { |
||||||
|
|
||||||
|
DataStorageConfiguration dataStorageConfiguration = |
||||||
|
ImmutableDataStorageConfiguration.builder() |
||||||
|
.dataStorageFormat(BONSAI) |
||||||
|
.bonsaiMaxLayersToLoad(2L) |
||||||
|
.unstable( |
||||||
|
ImmutableDataStorageConfiguration.Unstable.builder() |
||||||
|
.bonsaiTrieLogRetentionThreshold(3) |
||||||
|
.build() |
||||||
|
.withBonsaiTrieLogRetentionThreshold(3)) |
||||||
|
.build(); |
||||||
|
|
||||||
|
mockBlockchainBase(); |
||||||
|
when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5)); |
||||||
|
when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4)); |
||||||
|
when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); |
||||||
|
|
||||||
|
// assert trie logs that will be pruned exist before prune call
|
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x01").toArrayUnsafe()); |
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x02").toArrayUnsafe()); |
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x03").toArrayUnsafe()); |
||||||
|
|
||||||
|
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); |
||||||
|
|
||||||
|
// assert pruned trie logs are not in the DB
|
||||||
|
assertEquals(inMemoryWorldState.getTrieLog(blockHeader1.getHash()), Optional.empty()); |
||||||
|
assertEquals(inMemoryWorldState.getTrieLog(blockHeader2.getHash()), Optional.empty()); |
||||||
|
|
||||||
|
// assert retained trie logs are in the DB
|
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x03").toArrayUnsafe()); |
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x04").toArrayUnsafe()); |
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x05").toArrayUnsafe()); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void cantPruneIfNoFinalizedIsFound() { |
||||||
|
DataStorageConfiguration dataStorageConfiguration = |
||||||
|
ImmutableDataStorageConfiguration.builder() |
||||||
|
.dataStorageFormat(BONSAI) |
||||||
|
.bonsaiMaxLayersToLoad(2L) |
||||||
|
.unstable( |
||||||
|
ImmutableDataStorageConfiguration.Unstable.builder() |
||||||
|
.bonsaiTrieLogRetentionThreshold(2) |
||||||
|
.build() |
||||||
|
.withBonsaiTrieLogRetentionThreshold(2)) |
||||||
|
.build(); |
||||||
|
|
||||||
|
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); |
||||||
|
when(blockchain.getFinalized()).thenReturn(Optional.empty()); |
||||||
|
|
||||||
|
assertThrows( |
||||||
|
RuntimeException.class, |
||||||
|
() -> |
||||||
|
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { |
||||||
|
DataStorageConfiguration dataStorageConfiguration = |
||||||
|
ImmutableDataStorageConfiguration.builder() |
||||||
|
.dataStorageFormat(BONSAI) |
||||||
|
.bonsaiMaxLayersToLoad(2L) |
||||||
|
.unstable( |
||||||
|
ImmutableDataStorageConfiguration.Unstable.builder() |
||||||
|
.bonsaiTrieLogRetentionThreshold(10) |
||||||
|
.build() |
||||||
|
.withBonsaiTrieLogRetentionThreshold(10)) |
||||||
|
.build(); |
||||||
|
|
||||||
|
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); |
||||||
|
|
||||||
|
assertThrows( |
||||||
|
IllegalArgumentException.class, |
||||||
|
() -> |
||||||
|
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void cantPruneIfUserRequiredFurtherThanFinalized() { |
||||||
|
|
||||||
|
DataStorageConfiguration dataStorageConfiguration = |
||||||
|
ImmutableDataStorageConfiguration.builder() |
||||||
|
.dataStorageFormat(BONSAI) |
||||||
|
.bonsaiMaxLayersToLoad(2L) |
||||||
|
.unstable( |
||||||
|
ImmutableDataStorageConfiguration.Unstable.builder() |
||||||
|
.bonsaiTrieLogRetentionThreshold(2) |
||||||
|
.build() |
||||||
|
.withBonsaiTrieLogRetentionThreshold(2)) |
||||||
|
.build(); |
||||||
|
|
||||||
|
mockBlockchainBase(); |
||||||
|
|
||||||
|
assertThrows( |
||||||
|
IllegalArgumentException.class, |
||||||
|
() -> |
||||||
|
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { |
||||||
|
Files.delete(dataDir.resolve("database")); |
||||||
|
|
||||||
|
DataStorageConfiguration dataStorageConfiguration = |
||||||
|
ImmutableDataStorageConfiguration.builder() |
||||||
|
.dataStorageFormat(BONSAI) |
||||||
|
.bonsaiMaxLayersToLoad(2L) |
||||||
|
.unstable( |
||||||
|
ImmutableDataStorageConfiguration.Unstable.builder() |
||||||
|
.bonsaiTrieLogRetentionThreshold(2) |
||||||
|
.build() |
||||||
|
.withBonsaiTrieLogRetentionThreshold(2)) |
||||||
|
.build(); |
||||||
|
|
||||||
|
assertThrows( |
||||||
|
RuntimeException.class, |
||||||
|
() -> |
||||||
|
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); |
||||||
|
|
||||||
|
// assert all trie logs are still in the DB
|
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x01").toArrayUnsafe()); |
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x02").toArrayUnsafe()); |
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x03").toArrayUnsafe()); |
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x04").toArrayUnsafe()); |
||||||
|
assertArrayEquals( |
||||||
|
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), |
||||||
|
Bytes.fromHexString("0x05").toArrayUnsafe()); |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue