mirror of https://github.com/hyperledger/besu
Add --X-trie-log subcommand (#6303)
* Add x-trie-log subcommand for one-off trie log backlog prune Signed-off-by: Simon Dudley <simon.dudley@consensys.net> Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com> --------- Signed-off-by: Simon Dudley <simon.dudley@consensys.net> Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com> Co-authored-by: Simon Dudley <simon.dudley@consensys.net>pull/6372/head
Gabriel Fukushima
11 months ago
committed by
GitHub
parent
428177f514
commit
e51e042906
@ -0,0 +1,361 @@ |
||||
/* |
||||
* Copyright contributors to Hyperledger Besu. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
|
||||
package org.hyperledger.besu.cli.subcommands.storage; |
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument; |
||||
import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.chain.Blockchain; |
||||
import org.hyperledger.besu.ethereum.chain.MutableBlockchain; |
||||
import org.hyperledger.besu.ethereum.core.BlockHeader; |
||||
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; |
||||
|
||||
import java.io.File; |
||||
import java.io.FileInputStream; |
||||
import java.io.FileOutputStream; |
||||
import java.io.IOException; |
||||
import java.io.ObjectInputStream; |
||||
import java.io.ObjectOutputStream; |
||||
import java.io.PrintWriter; |
||||
import java.nio.file.Path; |
||||
import java.util.ArrayList; |
||||
import java.util.IdentityHashMap; |
||||
import java.util.List; |
||||
import java.util.Optional; |
||||
import java.util.concurrent.atomic.AtomicInteger; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes32; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
/** Helper class for counting and pruning trie logs */ |
||||
public class TrieLogHelper { |
||||
private static final String TRIE_LOG_FILE = "trieLogsToRetain"; |
||||
private static final long BATCH_SIZE = 20_000; |
||||
private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000; |
||||
private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); |
||||
|
||||
static void prune( |
||||
final DataStorageConfiguration config, |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||
final MutableBlockchain blockchain, |
||||
final Path dataDirectoryPath) { |
||||
final String batchFileNameBase = |
||||
dataDirectoryPath.resolve(DATABASE_PATH).resolve(TRIE_LOG_FILE).toString(); |
||||
|
||||
validatePruneConfiguration(config); |
||||
|
||||
final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); |
||||
|
||||
final long chainHeight = blockchain.getChainHeadBlockNumber(); |
||||
|
||||
final long lastBlockNumberToRetainTrieLogsFor = chainHeight - layersToRetain + 1; |
||||
|
||||
if (!validPruneRequirements(blockchain, chainHeight, lastBlockNumberToRetainTrieLogsFor)) { |
||||
return; |
||||
} |
||||
|
||||
final long numberOfBatches = calculateNumberofBatches(layersToRetain); |
||||
|
||||
processTrieLogBatches( |
||||
rootWorldStateStorage, |
||||
blockchain, |
||||
chainHeight, |
||||
lastBlockNumberToRetainTrieLogsFor, |
||||
numberOfBatches, |
||||
batchFileNameBase); |
||||
|
||||
if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { |
||||
deleteFiles(batchFileNameBase, numberOfBatches); |
||||
LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); |
||||
} else { |
||||
LOG.error("Prune failed. Re-run the subcommand to load the trie logs from file."); |
||||
} |
||||
} |
||||
|
||||
private static void processTrieLogBatches( |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||
final MutableBlockchain blockchain, |
||||
final long chainHeight, |
||||
final long lastBlockNumberToRetainTrieLogsFor, |
||||
final long numberOfBatches, |
||||
final String batchFileNameBase) { |
||||
|
||||
for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { |
||||
|
||||
final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); |
||||
|
||||
final long lastBlockOfBatch = |
||||
Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); |
||||
|
||||
final List<Hash> trieLogKeys = |
||||
getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch); |
||||
|
||||
saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys); |
||||
} |
||||
|
||||
LOG.info("Clear trie logs..."); |
||||
rootWorldStateStorage.clearTrieLog(); |
||||
|
||||
for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { |
||||
restoreTrieLogBatches(rootWorldStateStorage, batchNumber, batchFileNameBase); |
||||
} |
||||
} |
||||
|
||||
private static void saveTrieLogBatches( |
||||
final String batchFileNameBase, |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||
final long batchNumber, |
||||
final List<Hash> trieLogKeys) { |
||||
|
||||
LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); |
||||
|
||||
try { |
||||
saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); |
||||
} catch (IOException e) { |
||||
LOG.error("Error saving trie logs to file: {}", e.getMessage()); |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
private static void restoreTrieLogBatches( |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||
final long batchNumber, |
||||
final String batchFileNameBase) { |
||||
|
||||
try { |
||||
LOG.info("Restoring trie logs retained from batch {}...", batchNumber); |
||||
recreateTrieLogs(rootWorldStateStorage, batchNumber, batchFileNameBase); |
||||
} catch (IOException e) { |
||||
LOG.error("Error recreating trie logs from batch {}: {}", batchNumber, e.getMessage()); |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
private static void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { |
||||
|
||||
LOG.info("Deleting files..."); |
||||
|
||||
for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { |
||||
File file = new File(batchFileNameBase + "-" + batchNumber); |
||||
if (file.exists()) { |
||||
file.delete(); |
||||
} |
||||
} |
||||
} |
||||
|
||||
private static List<Hash> getTrieLogKeysForBlocks( |
||||
final MutableBlockchain blockchain, |
||||
final long firstBlockOfBatch, |
||||
final long lastBlockOfBatch) { |
||||
final List<Hash> trieLogKeys = new ArrayList<>(); |
||||
for (long i = firstBlockOfBatch; i >= lastBlockOfBatch; i--) { |
||||
final Optional<BlockHeader> header = blockchain.getBlockHeader(i); |
||||
header.ifPresentOrElse( |
||||
blockHeader -> trieLogKeys.add(blockHeader.getHash()), |
||||
() -> LOG.error("Error retrieving block")); |
||||
} |
||||
return trieLogKeys; |
||||
} |
||||
|
||||
private static long calculateNumberofBatches(final long layersToRetain) { |
||||
return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1); |
||||
} |
||||
|
||||
private static boolean validPruneRequirements( |
||||
final MutableBlockchain blockchain, |
||||
final long chainHeight, |
||||
final long lastBlockNumberToRetainTrieLogsFor) { |
||||
if (lastBlockNumberToRetainTrieLogsFor < 0) { |
||||
throw new IllegalArgumentException( |
||||
"Trying to retain more trie logs than chain length (" |
||||
+ chainHeight |
||||
+ "), skipping pruning"); |
||||
} |
||||
|
||||
final Optional<Hash> finalizedBlockHash = blockchain.getFinalized(); |
||||
|
||||
if (finalizedBlockHash.isEmpty()) { |
||||
throw new RuntimeException("No finalized block present, can't safely run trie log prune"); |
||||
} else { |
||||
final Hash finalizedHash = finalizedBlockHash.get(); |
||||
final Optional<BlockHeader> finalizedBlockHeader = blockchain.getBlockHeader(finalizedHash); |
||||
if (finalizedBlockHeader.isPresent() |
||||
&& finalizedBlockHeader.get().getNumber() < lastBlockNumberToRetainTrieLogsFor) { |
||||
throw new IllegalArgumentException( |
||||
"Trying to prune more layers than the finalized block height, skipping pruning"); |
||||
} |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
private static void recreateTrieLogs( |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||
final long batchNumber, |
||||
final String batchFileNameBase) |
||||
throws IOException { |
||||
// process in chunk to avoid OOM
|
||||
|
||||
IdentityHashMap<byte[], byte[]> trieLogsToRetain = |
||||
readTrieLogsFromFile(batchFileNameBase, batchNumber); |
||||
final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; |
||||
List<byte[]> keys = new ArrayList<>(trieLogsToRetain.keySet()); |
||||
|
||||
for (int startIndex = 0; startIndex < keys.size(); startIndex += chunkSize) { |
||||
processTransactionChunk(startIndex, chunkSize, keys, trieLogsToRetain, rootWorldStateStorage); |
||||
} |
||||
} |
||||
|
||||
private static void processTransactionChunk( |
||||
final int startIndex, |
||||
final int chunkSize, |
||||
final List<byte[]> keys, |
||||
final IdentityHashMap<byte[], byte[]> trieLogsToRetain, |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { |
||||
|
||||
var updater = rootWorldStateStorage.updater(); |
||||
int endIndex = Math.min(startIndex + chunkSize, keys.size()); |
||||
|
||||
for (int i = startIndex; i < endIndex; i++) { |
||||
byte[] key = keys.get(i); |
||||
byte[] value = trieLogsToRetain.get(key); |
||||
updater.getTrieLogStorageTransaction().put(key, value); |
||||
LOG.info("Key({}): {}", i, Bytes32.wrap(key).toShortHexString()); |
||||
} |
||||
|
||||
updater.getTrieLogStorageTransaction().commit(); |
||||
} |
||||
|
||||
private static void validatePruneConfiguration(final DataStorageConfiguration config) { |
||||
checkArgument( |
||||
config.getUnstable().getBonsaiTrieLogRetentionThreshold() |
||||
>= config.getBonsaiMaxLayersToLoad(), |
||||
String.format( |
||||
"--Xbonsai-trie-log-retention-threshold minimum value is %d", |
||||
config.getBonsaiMaxLayersToLoad())); |
||||
checkArgument( |
||||
config.getUnstable().getBonsaiTrieLogPruningLimit() > 0, |
||||
String.format( |
||||
"--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", |
||||
config.getUnstable().getBonsaiTrieLogPruningLimit())); |
||||
checkArgument( |
||||
config.getUnstable().getBonsaiTrieLogPruningLimit() |
||||
> config.getUnstable().getBonsaiTrieLogRetentionThreshold(), |
||||
String.format( |
||||
"--Xbonsai-trie-log-pruning-limit=%d must greater than --Xbonsai-trie-log-retention-threshold=%d", |
||||
config.getUnstable().getBonsaiTrieLogPruningLimit(), |
||||
config.getUnstable().getBonsaiTrieLogRetentionThreshold())); |
||||
} |
||||
|
||||
private static void saveTrieLogsInFile( |
||||
final List<Hash> trieLogsKeys, |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||
final long batchNumber, |
||||
final String batchFileNameBase) |
||||
throws IOException { |
||||
|
||||
File file = new File(batchFileNameBase + "-" + batchNumber); |
||||
if (file.exists()) { |
||||
LOG.error("File already exists, skipping file creation"); |
||||
return; |
||||
} |
||||
|
||||
try (FileOutputStream fos = new FileOutputStream(file)) { |
||||
ObjectOutputStream oos = new ObjectOutputStream(fos); |
||||
oos.writeObject(getTrieLogs(trieLogsKeys, rootWorldStateStorage)); |
||||
} catch (IOException e) { |
||||
LOG.error(e.getMessage()); |
||||
throw new RuntimeException(e); |
||||
} |
||||
} |
||||
|
||||
@SuppressWarnings("unchecked") |
||||
private static IdentityHashMap<byte[], byte[]> readTrieLogsFromFile( |
||||
final String batchFileNameBase, final long batchNumber) { |
||||
|
||||
IdentityHashMap<byte[], byte[]> trieLogs; |
||||
try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); |
||||
ObjectInputStream ois = new ObjectInputStream(fis)) { |
||||
|
||||
trieLogs = (IdentityHashMap<byte[], byte[]>) ois.readObject(); |
||||
|
||||
} catch (IOException | ClassNotFoundException e) { |
||||
|
||||
LOG.error(e.getMessage()); |
||||
throw new RuntimeException(e); |
||||
} |
||||
|
||||
return trieLogs; |
||||
} |
||||
|
||||
private static IdentityHashMap<byte[], byte[]> getTrieLogs( |
||||
final List<Hash> trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { |
||||
IdentityHashMap<byte[], byte[]> trieLogsToRetain = new IdentityHashMap<>(); |
||||
|
||||
LOG.info("Obtaining trielogs from db, this may take a few minutes..."); |
||||
trieLogKeys.forEach( |
||||
hash -> |
||||
rootWorldStateStorage |
||||
.getTrieLog(hash) |
||||
.ifPresent(trieLog -> trieLogsToRetain.put(hash.toArrayUnsafe(), trieLog))); |
||||
return trieLogsToRetain; |
||||
} |
||||
|
||||
static TrieLogCount getCount( |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||
final int limit, |
||||
final Blockchain blockchain) { |
||||
final AtomicInteger total = new AtomicInteger(); |
||||
final AtomicInteger canonicalCount = new AtomicInteger(); |
||||
final AtomicInteger forkCount = new AtomicInteger(); |
||||
final AtomicInteger orphanCount = new AtomicInteger(); |
||||
rootWorldStateStorage |
||||
.streamTrieLogKeys(limit) |
||||
.map(Bytes32::wrap) |
||||
.map(Hash::wrap) |
||||
.forEach( |
||||
hash -> { |
||||
total.getAndIncrement(); |
||||
blockchain |
||||
.getBlockHeader(hash) |
||||
.ifPresentOrElse( |
||||
(header) -> { |
||||
long number = header.getNumber(); |
||||
final Optional<BlockHeader> headerByNumber = |
||||
blockchain.getBlockHeader(number); |
||||
if (headerByNumber.isPresent() |
||||
&& headerByNumber.get().getHash().equals(hash)) { |
||||
canonicalCount.getAndIncrement(); |
||||
} else { |
||||
forkCount.getAndIncrement(); |
||||
} |
||||
}, |
||||
orphanCount::getAndIncrement); |
||||
}); |
||||
|
||||
return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); |
||||
} |
||||
|
||||
static void printCount(final PrintWriter out, final TrieLogCount count) { |
||||
out.printf( |
||||
"trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", |
||||
count.total, count.canonicalCount, count.forkCount, count.orphanCount); |
||||
} |
||||
|
||||
record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} |
||||
} |
||||
@ -0,0 +1,147 @@ |
||||
/* |
||||
* Copyright Hyperledger Besu Contributors. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
package org.hyperledger.besu.cli.subcommands.storage; |
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument; |
||||
import static com.google.common.base.Preconditions.checkNotNull; |
||||
|
||||
import org.hyperledger.besu.cli.util.VersionProvider; |
||||
import org.hyperledger.besu.controller.BesuController; |
||||
import org.hyperledger.besu.ethereum.chain.MutableBlockchain; |
||||
import org.hyperledger.besu.ethereum.storage.StorageProvider; |
||||
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; |
||||
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; |
||||
import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; |
||||
|
||||
import java.io.PrintWriter; |
||||
import java.nio.file.Path; |
||||
import java.nio.file.Paths; |
||||
|
||||
import org.apache.logging.log4j.Level; |
||||
import org.apache.logging.log4j.core.config.Configurator; |
||||
import org.slf4j.LoggerFactory; |
||||
import picocli.CommandLine; |
||||
import picocli.CommandLine.Command; |
||||
import picocli.CommandLine.ParentCommand; |
||||
|
||||
/** The Trie Log subcommand. */ |
||||
@Command( |
||||
name = "x-trie-log", |
||||
description = "Manipulate trie logs", |
||||
mixinStandardHelpOptions = true, |
||||
versionProvider = VersionProvider.class, |
||||
subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class}) |
||||
public class TrieLogSubCommand implements Runnable { |
||||
|
||||
@SuppressWarnings("UnusedVariable") |
||||
@ParentCommand |
||||
private static StorageSubCommand parentCommand; |
||||
|
||||
@SuppressWarnings("unused") |
||||
@CommandLine.Spec |
||||
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
|
||||
|
||||
@Override |
||||
public void run() { |
||||
final PrintWriter out = spec.commandLine().getOut(); |
||||
spec.commandLine().usage(out); |
||||
} |
||||
|
||||
private static BesuController createBesuController() { |
||||
return parentCommand.parentCommand.buildController(); |
||||
} |
||||
|
||||
@Command( |
||||
name = "count", |
||||
description = "This command counts all the trie logs", |
||||
mixinStandardHelpOptions = true, |
||||
versionProvider = VersionProvider.class) |
||||
static class CountTrieLog implements Runnable { |
||||
|
||||
@SuppressWarnings("unused") |
||||
@ParentCommand |
||||
private TrieLogSubCommand parentCommand; |
||||
|
||||
@SuppressWarnings("unused") |
||||
@CommandLine.Spec |
||||
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
|
||||
|
||||
@Override |
||||
public void run() { |
||||
TrieLogContext context = getTrieLogContext(); |
||||
|
||||
final PrintWriter out = spec.commandLine().getOut(); |
||||
|
||||
out.println("Counting trie logs..."); |
||||
TrieLogHelper.printCount( |
||||
out, |
||||
TrieLogHelper.getCount( |
||||
context.rootWorldStateStorage, Integer.MAX_VALUE, context.blockchain)); |
||||
} |
||||
} |
||||
|
||||
@Command( |
||||
name = "prune", |
||||
description = |
||||
"This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", |
||||
mixinStandardHelpOptions = true, |
||||
versionProvider = VersionProvider.class) |
||||
static class PruneTrieLog implements Runnable { |
||||
|
||||
@SuppressWarnings("unused") |
||||
@ParentCommand |
||||
private TrieLogSubCommand parentCommand; |
||||
|
||||
@SuppressWarnings("unused") |
||||
@CommandLine.Spec |
||||
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
|
||||
|
||||
@Override |
||||
public void run() { |
||||
TrieLogContext context = getTrieLogContext(); |
||||
final Path dataDirectoryPath = |
||||
Paths.get( |
||||
TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); |
||||
TrieLogHelper.prune( |
||||
context.config(), |
||||
context.rootWorldStateStorage(), |
||||
context.blockchain(), |
||||
dataDirectoryPath); |
||||
} |
||||
} |
||||
|
||||
record TrieLogContext( |
||||
DataStorageConfiguration config, |
||||
BonsaiWorldStateKeyValueStorage rootWorldStateStorage, |
||||
MutableBlockchain blockchain) {} |
||||
|
||||
private static TrieLogContext getTrieLogContext() { |
||||
Configurator.setLevel(LoggerFactory.getLogger(TrieLogPruner.class).getName(), Level.DEBUG); |
||||
checkNotNull(parentCommand); |
||||
BesuController besuController = createBesuController(); |
||||
final DataStorageConfiguration config = besuController.getDataStorageConfiguration(); |
||||
checkArgument( |
||||
DataStorageFormat.BONSAI.equals(config.getDataStorageFormat()), |
||||
"Subcommand only works with data-storage-format=BONSAI"); |
||||
|
||||
final StorageProvider storageProvider = besuController.getStorageProvider(); |
||||
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage = |
||||
(BonsaiWorldStateKeyValueStorage) |
||||
storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI); |
||||
final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain(); |
||||
return new TrieLogContext(config, rootWorldStateStorage, blockchain); |
||||
} |
||||
} |
||||
@ -0,0 +1,265 @@ |
||||
/* |
||||
* Copyright contributors to Hyperledger Besu. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on |
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the |
||||
* specific language governing permissions and limitations under the License. |
||||
* |
||||
* SPDX-License-Identifier: Apache-2.0 |
||||
*/ |
||||
|
||||
package org.hyperledger.besu.cli.subcommands.storage; |
||||
|
||||
import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; |
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals; |
||||
import static org.junit.jupiter.api.Assertions.assertEquals; |
||||
import static org.junit.jupiter.api.Assertions.assertThrows; |
||||
import static org.mockito.ArgumentMatchers.any; |
||||
import static org.mockito.Mockito.when; |
||||
|
||||
import org.hyperledger.besu.datatypes.Hash; |
||||
import org.hyperledger.besu.ethereum.chain.MutableBlockchain; |
||||
import org.hyperledger.besu.ethereum.core.BlockHeader; |
||||
import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; |
||||
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; |
||||
import org.hyperledger.besu.ethereum.storage.StorageProvider; |
||||
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; |
||||
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; |
||||
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; |
||||
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; |
||||
|
||||
import java.io.IOException; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
import java.util.Optional; |
||||
|
||||
import org.apache.tuweni.bytes.Bytes; |
||||
import org.junit.jupiter.api.AfterEach; |
||||
import org.junit.jupiter.api.BeforeAll; |
||||
import org.junit.jupiter.api.BeforeEach; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.extension.ExtendWith; |
||||
import org.junit.jupiter.api.io.TempDir; |
||||
import org.mockito.Mock; |
||||
import org.mockito.junit.jupiter.MockitoExtension; |
||||
|
||||
@ExtendWith(MockitoExtension.class) |
||||
class TrieLogHelperTest { |
||||
|
||||
private static final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); |
||||
private static BonsaiWorldStateKeyValueStorage inMemoryWorldState; |
||||
|
||||
@Mock private MutableBlockchain blockchain; |
||||
|
||||
@TempDir static Path dataDir; |
||||
|
||||
Path test; |
||||
static BlockHeader blockHeader1; |
||||
static BlockHeader blockHeader2; |
||||
static BlockHeader blockHeader3; |
||||
static BlockHeader blockHeader4; |
||||
static BlockHeader blockHeader5; |
||||
|
||||
@BeforeAll |
||||
public static void setup() throws IOException { |
||||
|
||||
blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); |
||||
blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); |
||||
blockHeader3 = new BlockHeaderTestFixture().number(3).buildHeader(); |
||||
blockHeader4 = new BlockHeaderTestFixture().number(4).buildHeader(); |
||||
blockHeader5 = new BlockHeaderTestFixture().number(5).buildHeader(); |
||||
|
||||
inMemoryWorldState = |
||||
new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); |
||||
|
||||
var updater = inMemoryWorldState.updater(); |
||||
updater |
||||
.getTrieLogStorageTransaction() |
||||
.put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe()); |
||||
updater |
||||
.getTrieLogStorageTransaction() |
||||
.put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe()); |
||||
updater |
||||
.getTrieLogStorageTransaction() |
||||
.put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe()); |
||||
updater |
||||
.getTrieLogStorageTransaction() |
||||
.put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe()); |
||||
updater |
||||
.getTrieLogStorageTransaction() |
||||
.put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); |
||||
updater.getTrieLogStorageTransaction().commit(); |
||||
} |
||||
|
||||
@BeforeEach |
||||
void createDirectory() throws IOException { |
||||
Files.createDirectories(dataDir.resolve("database")); |
||||
} |
||||
|
||||
@AfterEach |
||||
void deleteDirectory() throws IOException { |
||||
Files.deleteIfExists(dataDir.resolve("database")); |
||||
} |
||||
|
||||
void mockBlockchainBase() { |
||||
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); |
||||
when(blockchain.getFinalized()).thenReturn(Optional.of(blockHeader3.getBlockHash())); |
||||
when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(blockHeader3)); |
||||
} |
||||
|
||||
@Test |
||||
public void prune() { |
||||
|
||||
DataStorageConfiguration dataStorageConfiguration = |
||||
ImmutableDataStorageConfiguration.builder() |
||||
.dataStorageFormat(BONSAI) |
||||
.bonsaiMaxLayersToLoad(2L) |
||||
.unstable( |
||||
ImmutableDataStorageConfiguration.Unstable.builder() |
||||
.bonsaiTrieLogRetentionThreshold(3) |
||||
.build() |
||||
.withBonsaiTrieLogRetentionThreshold(3)) |
||||
.build(); |
||||
|
||||
mockBlockchainBase(); |
||||
when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5)); |
||||
when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4)); |
||||
when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); |
||||
|
||||
// assert trie logs that will be pruned exist before prune call
|
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), |
||||
Bytes.fromHexString("0x01").toArrayUnsafe()); |
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), |
||||
Bytes.fromHexString("0x02").toArrayUnsafe()); |
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), |
||||
Bytes.fromHexString("0x03").toArrayUnsafe()); |
||||
|
||||
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); |
||||
|
||||
// assert pruned trie logs are not in the DB
|
||||
assertEquals(inMemoryWorldState.getTrieLog(blockHeader1.getHash()), Optional.empty()); |
||||
assertEquals(inMemoryWorldState.getTrieLog(blockHeader2.getHash()), Optional.empty()); |
||||
|
||||
// assert retained trie logs are in the DB
|
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), |
||||
Bytes.fromHexString("0x03").toArrayUnsafe()); |
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), |
||||
Bytes.fromHexString("0x04").toArrayUnsafe()); |
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), |
||||
Bytes.fromHexString("0x05").toArrayUnsafe()); |
||||
} |
||||
|
||||
@Test |
||||
public void cantPruneIfNoFinalizedIsFound() { |
||||
DataStorageConfiguration dataStorageConfiguration = |
||||
ImmutableDataStorageConfiguration.builder() |
||||
.dataStorageFormat(BONSAI) |
||||
.bonsaiMaxLayersToLoad(2L) |
||||
.unstable( |
||||
ImmutableDataStorageConfiguration.Unstable.builder() |
||||
.bonsaiTrieLogRetentionThreshold(2) |
||||
.build() |
||||
.withBonsaiTrieLogRetentionThreshold(2)) |
||||
.build(); |
||||
|
||||
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); |
||||
when(blockchain.getFinalized()).thenReturn(Optional.empty()); |
||||
|
||||
assertThrows( |
||||
RuntimeException.class, |
||||
() -> |
||||
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); |
||||
} |
||||
|
||||
@Test |
||||
public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { |
||||
DataStorageConfiguration dataStorageConfiguration = |
||||
ImmutableDataStorageConfiguration.builder() |
||||
.dataStorageFormat(BONSAI) |
||||
.bonsaiMaxLayersToLoad(2L) |
||||
.unstable( |
||||
ImmutableDataStorageConfiguration.Unstable.builder() |
||||
.bonsaiTrieLogRetentionThreshold(10) |
||||
.build() |
||||
.withBonsaiTrieLogRetentionThreshold(10)) |
||||
.build(); |
||||
|
||||
when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); |
||||
|
||||
assertThrows( |
||||
IllegalArgumentException.class, |
||||
() -> |
||||
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); |
||||
} |
||||
|
||||
@Test |
||||
public void cantPruneIfUserRequiredFurtherThanFinalized() { |
||||
|
||||
DataStorageConfiguration dataStorageConfiguration = |
||||
ImmutableDataStorageConfiguration.builder() |
||||
.dataStorageFormat(BONSAI) |
||||
.bonsaiMaxLayersToLoad(2L) |
||||
.unstable( |
||||
ImmutableDataStorageConfiguration.Unstable.builder() |
||||
.bonsaiTrieLogRetentionThreshold(2) |
||||
.build() |
||||
.withBonsaiTrieLogRetentionThreshold(2)) |
||||
.build(); |
||||
|
||||
mockBlockchainBase(); |
||||
|
||||
assertThrows( |
||||
IllegalArgumentException.class, |
||||
() -> |
||||
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); |
||||
} |
||||
|
||||
@Test |
||||
public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { |
||||
Files.delete(dataDir.resolve("database")); |
||||
|
||||
DataStorageConfiguration dataStorageConfiguration = |
||||
ImmutableDataStorageConfiguration.builder() |
||||
.dataStorageFormat(BONSAI) |
||||
.bonsaiMaxLayersToLoad(2L) |
||||
.unstable( |
||||
ImmutableDataStorageConfiguration.Unstable.builder() |
||||
.bonsaiTrieLogRetentionThreshold(2) |
||||
.build() |
||||
.withBonsaiTrieLogRetentionThreshold(2)) |
||||
.build(); |
||||
|
||||
assertThrows( |
||||
RuntimeException.class, |
||||
() -> |
||||
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); |
||||
|
||||
// assert all trie logs are still in the DB
|
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), |
||||
Bytes.fromHexString("0x01").toArrayUnsafe()); |
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), |
||||
Bytes.fromHexString("0x02").toArrayUnsafe()); |
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), |
||||
Bytes.fromHexString("0x03").toArrayUnsafe()); |
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), |
||||
Bytes.fromHexString("0x04").toArrayUnsafe()); |
||||
assertArrayEquals( |
||||
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), |
||||
Bytes.fromHexString("0x05").toArrayUnsafe()); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue