CipherLabs
/
besu
mirror of https://github.com/hyperledger/besu
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Import export trie log (#6363)

Browse Source 
* Import and export trie log subcommands
* change option name and fix descriptions

Signed-off-by: Gabriel Fukushima <gabrielfukushima@gmail.com>
Co-authored-by: Jason Frame <jason.frame@consensys.net>
pull/6445/head
Gabriel Fukushima 10 months ago committed by GitHub
parent 24718e391a
commit cfea3ab2fd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 349 additions and 83 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 11
      besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
  2. 103
      besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java
  3. 106
      besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java
  4. 20
      besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java
  5. 192
      besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java

11
besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java
Unescape View File

@ -62,23 +62,28 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
private final DataStorageOptions.Unstable unstableOptions = new Unstable();
static class Unstable {
private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED =
"--Xbonsai-limit-trie-logs-enabled";
private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD =
"--Xbonsai-trie-logs-retention-threshold";
private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit";
@CommandLine.Option(
hidden = true,
names = {"--Xbonsai-trie-log-pruning-enabled"},
names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED},
description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})")
private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
@CommandLine.Option(
hidden = true,
names = {"--Xbonsai-trie-log-retention-threshold"},
names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD},
description =
"The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})")
private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
@CommandLine.Option(
hidden = true,
names = {"--Xbonsai-trie-log-pruning-limit"},
names = {BONSAI_TRIE_LOG_PRUNING_LIMIT},
description =
"The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;

103
besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java
Unescape View File

@ -22,7 +22,11 @@ import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import java.io.File;
@ -32,6 +36,7 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.IdentityHashMap;
@ -39,6 +44,7 @@ import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -97,16 +103,15 @@ public class TrieLogHelper {
final String batchFileNameBase) {
for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) {
final String batchFileName = batchFileNameBase + "-" + batchNumber;
final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE);
final long lastBlockOfBatch =
Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor);
final List<Hash> trieLogKeys =
getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch);
saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys);
LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber);
saveTrieLogBatches(batchFileName, rootWorldStateStorage, trieLogKeys);
}
LOG.info("Clear trie logs...");
@ -118,15 +123,12 @@ public class TrieLogHelper {
}
private static void saveTrieLogBatches(
final String batchFileNameBase,
final String batchFileName,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final long batchNumber,
final List<Hash> trieLogKeys) {
LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber);
try {
saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase);
saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchFileName);
} catch (IOException e) {
LOG.error("Error saving trie logs to file: {}", e.getMessage());
throw new RuntimeException(e);
@ -210,9 +212,8 @@ public class TrieLogHelper {
final String batchFileNameBase)
throws IOException {
// process in chunk to avoid OOM
IdentityHashMap<byte[], byte[]> trieLogsToRetain =
readTrieLogsFromFile(batchFileNameBase, batchNumber);
final String batchFileName = batchFileNameBase + "-" + batchNumber;
IdentityHashMap<byte[], byte[]> trieLogsToRetain = readTrieLogsFromFile(batchFileName);
final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION;
List<byte[]> keys = new ArrayList<>(trieLogsToRetain.keySet());
@ -265,11 +266,10 @@ public class TrieLogHelper {
private static void saveTrieLogsInFile(
final List<Hash> trieLogsKeys,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final long batchNumber,
final String batchFileNameBase)
final String batchFileName)
throws IOException {
File file = new File(batchFileNameBase + "-" + batchNumber);
File file = new File(batchFileName);
if (file.exists()) {
LOG.error("File already exists, skipping file creation");
return;
@ -285,17 +285,14 @@ public class TrieLogHelper {
}
@SuppressWarnings("unchecked")
private static IdentityHashMap<byte[], byte[]> readTrieLogsFromFile(
final String batchFileNameBase, final long batchNumber) {
static IdentityHashMap<byte[], byte[]> readTrieLogsFromFile(final String batchFileName) {
IdentityHashMap<byte[], byte[]> trieLogs;
try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber);
try (FileInputStream fis = new FileInputStream(batchFileName);
ObjectInputStream ois = new ObjectInputStream(fis)) {
trieLogs = (IdentityHashMap<byte[], byte[]>) ois.readObject();
} catch (IOException | ClassNotFoundException e) {
LOG.error(e.getMessage());
throw new RuntimeException(e);
}
@ -303,6 +300,52 @@ public class TrieLogHelper {
return trieLogs;
}
private static void saveTrieLogsAsRlpInFile(
final List<Hash> trieLogsKeys,
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final String batchFileName) {
File file = new File(batchFileName);
if (file.exists()) {
LOG.error("File already exists, skipping file creation");
return;
}
final IdentityHashMap<byte[], byte[]> trieLogs =
getTrieLogs(trieLogsKeys, rootWorldStateStorage);
final Bytes rlp =
RLP.encode(
o ->
o.writeList(
trieLogs.entrySet(), (val, out) -> out.writeRaw(Bytes.wrap(val.getValue()))));
try {
Files.write(file.toPath(), rlp.toArrayUnsafe());
} catch (IOException e) {
LOG.error(e.getMessage());
throw new RuntimeException(e);
}
}
static IdentityHashMap<byte[], byte[]> readTrieLogsAsRlpFromFile(final String batchFileName) {
try {
final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName)));
final BytesValueRLPInput input = new BytesValueRLPInput(file, false);
input.enterList();
final IdentityHashMap<byte[], byte[]> trieLogs = new IdentityHashMap<>();
while (!input.isEndOfCurrentList()) {
final Bytes trieLogBytes = input.currentListAsBytes();
TrieLogLayer trieLogLayer =
TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(trieLogBytes), false));
trieLogs.put(trieLogLayer.getBlockHash().toArrayUnsafe(), trieLogBytes.toArrayUnsafe());
}
input.leaveList();
return trieLogs;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static IdentityHashMap<byte[], byte[]> getTrieLogs(
final List<Hash> trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) {
IdentityHashMap<byte[], byte[]> trieLogsToRetain = new IdentityHashMap<>();
@ -357,5 +400,25 @@ public class TrieLogHelper {
count.total, count.canonicalCount, count.forkCount, count.orphanCount);
}
static void importTrieLog(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) {
var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString());
var updater = rootWorldStateStorage.updater();
trieLog.forEach((key, value) -> updater.getTrieLogStorageTransaction().put(key, value));
updater.getTrieLogStorageTransaction().commit();
}
static void exportTrieLog(
final BonsaiWorldStateKeyValueStorage rootWorldStateStorage,
final List<Hash> trieLogHash,
final Path directoryPath)
throws IOException {
final String trieLogFile = directoryPath.toString();
saveTrieLogsAsRlpInFile(trieLogHash, rootWorldStateStorage, trieLogFile);
}
record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {}
}

106
besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java
Unescape View File

@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
import org.hyperledger.besu.cli.util.VersionProvider;
import org.hyperledger.besu.controller.BesuController;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
@ -26,9 +27,11 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.config.Configurator;
@ -43,7 +46,12 @@ import picocli.CommandLine.ParentCommand;
description = "Manipulate trie logs",
mixinStandardHelpOptions = true,
versionProvider = VersionProvider.class,
subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class})
subcommands = {
TrieLogSubCommand.CountTrieLog.class,
TrieLogSubCommand.PruneTrieLog.class,
TrieLogSubCommand.ExportTrieLog.class,
TrieLogSubCommand.ImportTrieLog.class
})
public class TrieLogSubCommand implements Runnable {
@SuppressWarnings("UnusedVariable")
@ -123,6 +131,102 @@ public class TrieLogSubCommand implements Runnable {
}
}
@Command(
name = "export",
description = "This command exports the trie log of a determined block to a binary file",
mixinStandardHelpOptions = true,
versionProvider = VersionProvider.class)
static class ExportTrieLog implements Runnable {
@SuppressWarnings("unused")
@ParentCommand
private TrieLogSubCommand parentCommand;
@SuppressWarnings("unused")
@CommandLine.Spec
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
@CommandLine.Option(
names = "--trie-log-block-hash",
description =
"Comma separated list of hashes from the blocks you want to export the trie logs of",
split = " {0,1}, {0,1}",
arity = "1..*")
private List<String> trieLogBlockHashList;
@CommandLine.Option(
names = "--trie-log-file-path",
description = "The file you want to export the trie logs to",
arity = "1..1")
private Path trieLogFilePath = null;
@Override
public void run() {
if (trieLogFilePath == null) {
trieLogFilePath =
Paths.get(
TrieLogSubCommand.parentCommand
.parentCommand
.dataDir()
.resolve("trie-logs.bin")
.toAbsolutePath()
.toString());
}
TrieLogContext context = getTrieLogContext();
final List<Hash> listOfBlockHashes =
trieLogBlockHashList.stream().map(Hash::fromHexString).toList();
try {
TrieLogHelper.exportTrieLog(
context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
@Command(
name = "import",
description = "This command imports a trie log exported by another besu node",
mixinStandardHelpOptions = true,
versionProvider = VersionProvider.class)
static class ImportTrieLog implements Runnable {
@SuppressWarnings("unused")
@ParentCommand
private TrieLogSubCommand parentCommand;
@SuppressWarnings("unused")
@CommandLine.Spec
private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
@CommandLine.Option(
names = "--trie-log-file-path",
description = "The file you want to import the trie logs from",
arity = "1..1")
private Path trieLogFilePath = null;
@Override
public void run() {
if (trieLogFilePath == null) {
trieLogFilePath =
Paths.get(
TrieLogSubCommand.parentCommand
.parentCommand
.dataDir()
.resolve("trie-logs.bin")
.toAbsolutePath()
.toString());
}
TrieLogContext context = getTrieLogContext();
TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath);
}
}
record TrieLogContext(
DataStorageConfiguration config,
BonsaiWorldStateKeyValueStorage rootWorldStateStorage,

20
besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java
Unescape View File

@ -34,8 +34,8 @@ public class DataStorageOptionsTest
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit())
.isEqualTo(1),
"--Xbonsai-trie-log-pruning-enabled",
"--Xbonsai-trie-log-pruning-limit",
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-pruning-limit",
"1");
}
@ -43,8 +43,8 @@ public class DataStorageOptionsTest
public void bonsaiTrieLogPruningLimitShouldBePositive() {
internalTestFailure(
"--Xbonsai-trie-log-pruning-limit=0 must be greater than 0",
"--Xbonsai-trie-log-pruning-enabled",
"--Xbonsai-trie-log-pruning-limit",
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-pruning-limit",
"0");
}
@ -54,8 +54,8 @@ public class DataStorageOptionsTest
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold())
.isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1),
"--Xbonsai-trie-log-pruning-enabled",
"--Xbonsai-trie-log-retention-threshold",
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-retention-threshold",
"513");
}
@ -65,8 +65,8 @@ public class DataStorageOptionsTest
dataStorageConfiguration ->
assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold())
.isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD),
"--Xbonsai-trie-log-pruning-enabled",
"--Xbonsai-trie-log-retention-threshold",
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-retention-threshold",
"512");
}
@ -74,8 +74,8 @@ public class DataStorageOptionsTest
public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() {
internalTestFailure(
"--Xbonsai-trie-log-retention-threshold minimum value is 512",
"--Xbonsai-trie-log-pruning-enabled",
"--Xbonsai-trie-log-retention-threshold",
"--Xbonsai-limit-trie-logs-enabled",
"--Xbonsai-trie-logs-retention-threshold",
"511");
}

192
besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java
Unescape View File

@ -15,6 +15,7 @@
package org.hyperledger.besu.cli.subcommands.storage;
import static java.util.Collections.singletonList;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -27,8 +28,11 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture;
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
@ -36,11 +40,12 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.tuweni.bytes.Bytes;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@ -56,17 +61,14 @@ class TrieLogHelperTest {
@Mock private MutableBlockchain blockchain;
@TempDir static Path dataDir;
Path test;
static BlockHeader blockHeader1;
static BlockHeader blockHeader2;
static BlockHeader blockHeader3;
static BlockHeader blockHeader4;
static BlockHeader blockHeader5;
@BeforeAll
public static void setup() throws IOException {
@BeforeEach
public void setup() throws IOException {
blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader();
blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader();
@ -78,33 +80,33 @@ class TrieLogHelperTest {
new BonsaiWorldStateKeyValueStorage(
storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG);
createTrieLog(blockHeader1);
var updater = inMemoryWorldState.updater();
updater
.getTrieLogStorageTransaction()
.put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe());
.put(blockHeader1.getHash().toArrayUnsafe(), createTrieLog(blockHeader1));
updater
.getTrieLogStorageTransaction()
.put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe());
.put(blockHeader2.getHash().toArrayUnsafe(), createTrieLog(blockHeader2));
updater
.getTrieLogStorageTransaction()
.put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe());
.put(blockHeader3.getHash().toArrayUnsafe(), createTrieLog(blockHeader3));
updater
.getTrieLogStorageTransaction()
.put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe());
.put(blockHeader4.getHash().toArrayUnsafe(), createTrieLog(blockHeader4));
updater
.getTrieLogStorageTransaction()
.put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe());
.put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5));
updater.getTrieLogStorageTransaction().commit();
}
@BeforeEach
void createDirectory() throws IOException {
Files.createDirectories(dataDir.resolve("database"));
}
@AfterEach
void deleteDirectory() throws IOException {
Files.deleteIfExists(dataDir.resolve("database"));
private static byte[] createTrieLog(final BlockHeader blockHeader) {
TrieLogLayer trieLogLayer = new TrieLogLayer();
trieLogLayer.setBlockHash(blockHeader.getBlockHash());
final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput();
TrieLogFactoryImpl.writeTo(trieLogLayer, rlpLog);
return rlpLog.encoded().toArrayUnsafe();
}
void mockBlockchainBase() {
@ -114,7 +116,8 @@ class TrieLogHelperTest {
}
@Test
public void prune() {
public void prune(final @TempDir Path dataDir) throws IOException {
Files.createDirectories(dataDir.resolve("database"));
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
@ -134,14 +137,11 @@ class TrieLogHelperTest {
// assert trie logs that will be pruned exist before prune call
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(),
Bytes.fromHexString("0x01").toArrayUnsafe());
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(),
Bytes.fromHexString("0x02").toArrayUnsafe());
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(),
Bytes.fromHexString("0x03").toArrayUnsafe());
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3));
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir);
@ -151,18 +151,15 @@ class TrieLogHelperTest {
// assert retained trie logs are in the DB
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(),
Bytes.fromHexString("0x03").toArrayUnsafe());
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(),
Bytes.fromHexString("0x04").toArrayUnsafe());
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(),
Bytes.fromHexString("0x05").toArrayUnsafe());
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5));
}
@Test
public void cantPruneIfNoFinalizedIsFound() {
public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
@ -184,7 +181,7 @@ class TrieLogHelperTest {
}
@Test
public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() {
public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDir Path dataDir) {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
@ -205,7 +202,7 @@ class TrieLogHelperTest {
}
@Test
public void cantPruneIfUserRequiredFurtherThanFinalized() {
public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
@ -227,8 +224,7 @@ class TrieLogHelperTest {
}
@Test
public void exceptionWhileSavingFileStopsPruneProcess() throws IOException {
Files.delete(dataDir.resolve("database"));
public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDir) {
DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
@ -244,23 +240,121 @@ class TrieLogHelperTest {
assertThrows(
RuntimeException.class,
() ->
TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir));
TrieLogHelper.prune(
dataStorageConfiguration,
inMemoryWorldState,
blockchain,
dataDir.resolve("unknownPath")));
// assert all trie logs are still in the DB
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(),
Bytes.fromHexString("0x01").toArrayUnsafe());
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4));
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5));
}
@Test
public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException {
TrieLogHelper.exportTrieLog(
inMemoryWorldState,
singletonList(blockHeader1.getHash()),
dataDir.resolve("trie-log-dump"));
var trieLog =
TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString())
.entrySet()
.stream()
.findFirst()
.get();
assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe());
assertArrayEquals(
trieLog.getValue(), inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
}
@Test
public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException {
TrieLogHelper.exportTrieLog(
inMemoryWorldState,
List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()),
dataDir.resolve("trie-log-dump"));
var trieLogs =
TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString())
.entrySet()
.stream()
.collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue));
assertArrayEquals(
trieLogs.get(blockHeader1.getHash()),
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(),
Bytes.fromHexString("0x02").toArrayUnsafe());
trieLogs.get(blockHeader2.getHash()),
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get());
assertArrayEquals(
trieLogs.get(blockHeader3.getHash()),
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get());
}
@Test
public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException {
StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider();
BonsaiWorldStateKeyValueStorage inMemoryWorldState2 =
new BonsaiWorldStateKeyValueStorage(
tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG);
TrieLogHelper.exportTrieLog(
inMemoryWorldState,
singletonList(blockHeader1.getHash()),
dataDir.resolve("trie-log-dump"));
var trieLog =
TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString());
var updater = inMemoryWorldState2.updater();
trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v));
updater.getTrieLogStorageTransaction().commit();
assertArrayEquals(
inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(),
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
}
@Test
public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException {
StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider();
BonsaiWorldStateKeyValueStorage inMemoryWorldState2 =
new BonsaiWorldStateKeyValueStorage(
tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG);
TrieLogHelper.exportTrieLog(
inMemoryWorldState,
List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()),
dataDir.resolve("trie-log-dump"));
var trieLog =
TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString());
var updater = inMemoryWorldState2.updater();
trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v));
updater.getTrieLogStorageTransaction().commit();
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(),
Bytes.fromHexString("0x03").toArrayUnsafe());
inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(),
inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get());
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(),
Bytes.fromHexString("0x04").toArrayUnsafe());
inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get(),
inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get());
assertArrayEquals(
inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(),
Bytes.fromHexString("0x05").toArrayUnsafe());
inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get(),
inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get());
}
}

Write Preview
Loading…
Cancel
Save