diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java b/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java index b7588f48b6..69b5730eed 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java @@ -18,7 +18,6 @@ import tech.pegasys.pantheon.cli.PantheonCommand; import tech.pegasys.pantheon.cli.PantheonControllerBuilder; import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration.Builder; import tech.pegasys.pantheon.util.BlockImporter; -import tech.pegasys.pantheon.util.BlockchainImporter; import picocli.CommandLine.RunLast; @@ -31,7 +30,6 @@ public final class Pantheon { final PantheonCommand pantheonCommand = new PantheonCommand( new BlockImporter(), - new BlockchainImporter(), new RunnerBuilder(), new PantheonControllerBuilder(), new Builder()); diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/ImportBlockchainSubCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/ImportBlockchainSubCommand.java deleted file mode 100644 index 475fd059af..0000000000 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/ImportBlockchainSubCommand.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.cli; - -import static com.google.common.base.Preconditions.checkNotNull; - -import tech.pegasys.pantheon.util.BlockImporter; - -import java.io.IOException; -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.ExecutionException; -import picocli.CommandLine.Parameters; -import picocli.CommandLine.ParentCommand; - -@Command( - name = "import-blockchain", - description = - "This command imports blocks from a file into the database. WARNING: This import is ALPHA and does not include comprehensive validations yet.", - mixinStandardHelpOptions = true -) -class ImportBlockchainSubCommand implements Runnable { - - private static final Logger LOG = LogManager.getLogger(); - - @ParentCommand - private PantheonCommand parentCommand; // Picocli injects reference to parent command - - @Parameters(arity = "1..1", paramLabel = "PATH", description = "File containing blocks to import") - private final Path blocksImportPath = null; - - @CommandLine.Option( - names = {"--skip-header-validation"}, - description = "WARNING: Set only if the import file is pre-validated." - ) - private final Boolean isSkipHeaderValidation = false; - - @CommandLine.Option( - names = {"--metrics-interval-sec"}, - description = "seconds between logging progress metrics.", - defaultValue = "30" - ) - private final Integer metricsIntervalSec = 30; - - @CommandLine.Option( - names = {"--account-commit-interval"}, - description = "commit account state every n accounts.", - defaultValue = "100000" - ) - private final Integer accountCommitInterval = 100_000; - - @CommandLine.Option( - names = {"--skip-blocks"}, - description = "skip processing the blocks in the import file", - defaultValue = "false" - ) - private final Boolean isSkipBlocks = Boolean.FALSE; - - @CommandLine.Option( - names = {"--skip-accounts"}, - description = "skip processing the accounts in the import file", - defaultValue = "false" - ) - private final Boolean isSkipAccounts = Boolean.FALSE; - - @CommandLine.Option( - names = {"--start-of-world-state"}, - description = - "file offset for the starting byte of the world state. Only relevant in combination with --skip-blocks." - ) - private final Long worldStateOffset = null; - - public ImportBlockchainSubCommand() {} - - @Override - public void run() { - LOG.info("Runs import sub command with blocksImportPath : {}", blocksImportPath); - - checkNotNull(parentCommand); - - checkNotNull(isSkipHeaderValidation); - checkNotNull(isSkipBlocks); - checkNotNull(isSkipAccounts); - - try { - final BlockImporter.ImportResult result = - parentCommand.blockchainImporter.importBlockchain( - blocksImportPath, - parentCommand.buildController(), - isSkipHeaderValidation, - metricsIntervalSec, - accountCommitInterval, - isSkipBlocks, - isSkipAccounts, - worldStateOffset); - System.out.println(result); - } catch (final IOException e) { - throw new ExecutionException( - new CommandLine(this), - String.format("Unable to import blocks from $1%s", blocksImportPath)); - } - } -} diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java index f2b55e6435..101e0795c3 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java @@ -33,7 +33,6 @@ import tech.pegasys.pantheon.ethereum.jsonrpc.websocket.WebSocketConfiguration; import tech.pegasys.pantheon.ethereum.p2p.peers.DefaultPeer; import tech.pegasys.pantheon.ethereum.util.InvalidConfigurationException; import tech.pegasys.pantheon.util.BlockImporter; -import tech.pegasys.pantheon.util.BlockchainImporter; import tech.pegasys.pantheon.util.bytes.BytesValue; import java.io.File; @@ -120,7 +119,6 @@ public class PantheonCommand implements Runnable { } private final BlockImporter blockImporter; - final BlockchainImporter blockchainImporter; private final PantheonControllerBuilder controllerBuilder; private final Builder synchronizerConfigurationBuilder; @@ -361,12 +359,10 @@ public class PantheonCommand implements Runnable { public PantheonCommand( final BlockImporter blockImporter, - final BlockchainImporter blockchainImporter, final RunnerBuilder runnerBuilder, final PantheonControllerBuilder controllerBuilder, final Builder synchronizerConfigurationBuilder) { this.blockImporter = blockImporter; - this.blockchainImporter = blockchainImporter; this.runnerBuilder = runnerBuilder; this.controllerBuilder = controllerBuilder; this.synchronizerConfigurationBuilder = synchronizerConfigurationBuilder; @@ -380,9 +376,7 @@ public class PantheonCommand implements Runnable { final CommandLine commandLine = new CommandLine(this); final ImportSubCommand importSubCommand = new ImportSubCommand(blockImporter); - final ImportBlockchainSubCommand importBlockchainSubCommand = new ImportBlockchainSubCommand(); commandLine.addSubcommand("import", importSubCommand); - commandLine.addSubcommand("import-blockchain", importBlockchainSubCommand); commandLine.addSubcommand("export-pub-key", new ExportPublicKeySubCommand()); commandLine.registerConverter(Address.class, Address::fromHexString); diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockchainImporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockchainImporter.java deleted file mode 100644 index 005ed11ab0..0000000000 --- a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockchainImporter.java +++ /dev/null @@ -1,576 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.util; - -import static com.google.common.base.Preconditions.checkNotNull; -import static java.lang.String.format; - -import tech.pegasys.pantheon.controller.PantheonController; -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.GenesisConfig; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Address; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.BlockBody; -import tech.pegasys.pantheon.ethereum.core.BlockHeader; -import tech.pegasys.pantheon.ethereum.core.Hash; -import tech.pegasys.pantheon.ethereum.core.MutableAccount; -import tech.pegasys.pantheon.ethereum.core.MutableWorldState; -import tech.pegasys.pantheon.ethereum.core.Transaction; -import tech.pegasys.pantheon.ethereum.core.TransactionReceipt; -import tech.pegasys.pantheon.ethereum.core.Wei; -import tech.pegasys.pantheon.ethereum.core.WorldUpdater; -import tech.pegasys.pantheon.ethereum.mainnet.BlockHeaderValidator; -import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSpec; -import tech.pegasys.pantheon.ethereum.mainnet.ScheduleBasedBlockHashFunction; -import tech.pegasys.pantheon.ethereum.rlp.FileRLPInput; -import tech.pegasys.pantheon.ethereum.rlp.RLPInput; -import tech.pegasys.pantheon.util.bytes.BytesValue; -import tech.pegasys.pantheon.util.uint.UInt256; - -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.util.Strings; - -/** - * Pantheon Blockchain Import Util. - * - *

Expected File RLP Format: - * - *

Snapshot: BlockhainSnapshot || WorldStateSnapshot - * - *

BlockchainSnapshot: N || BlockWithReceipts[0] || BlockWithReceipts[1] || ... || - * BlockWithReceipts[N] - * - *

BlockWithReceipts[n]: Block[n] || Receipts[n] - * - *

Block[n]: Header[n] || Transactions[n] || OmmerHeaders[n] - * - *

Transactions[n]: [ Transaction[0] || Transaction[1] || ... || Transaction[T] ] - * OmmerHeaders[n]: [ OmmerHeader[0] || OmmerHeader[1] || ... || OmmerHeader[O] ] Receipts[n]: [ - * Receipt[0] || Receipt[1] || ... || Receipt[T] ] - * - *

WorldStateSnapshot: AccountSnapshot[0] || AccountSnapshot[1] || ... || AccountSnapshot[A] - * AccountSnapshot[a]: AccountAddress[a] || AccountState[a] || AccountCode[a] || - * AccountStorageSnapshot[a] AccountStorageSnapshot[a]: AccountStorageEntry[0] || - * AccountStorageEntry[1] || ... || AccountStorageEntry[E] AccountStorageEntry[e]: - * AccountStorageKey[e] || AccountStorageValue[e] - * - *

N = number of blocks T = number of transactions in block O = number of ommers in block A = - * number of accounts in world state E = number of storage entries in the account || = concatenation - */ -public class BlockchainImporter extends BlockImporter { - private static final Logger LOG = LogManager.getLogger(); - private static final Logger METRICS_LOG = LogManager.getLogger(LOG.getName() + "-metrics"); - - Boolean isSkipHeaderValidation = false; - - /** - * Imports blockchain from file as concatenated RLP sections - * - * @param the consensus context type - * @param dataFilePath Path to the file containing the dataFilePath - * @param pantheonController the PantheonController that defines blockchain behavior - * @param isSkipHeaderValidation if true, header validation is skipped. This must only be used - * when the source data is fully trusted / guaranteed to be correct. - * @param metricsIntervalSec seconds between logging progress metrics - * @param accountCommitInterval commit account state every n accounts - * @param isSkipBlocks true if blocks in the import file should be skipped over. - * @param isSkipAccounts true if accounts in the import file should be skipped over. - * @param worldStateOffset file offset for the starting byte of the world state. Only relevant in - * combination with isSkipBlocks - * @return the import result - * @throws IOException On Failure - */ - public BlockImporter.ImportResult importBlockchain( - final Path dataFilePath, - final PantheonController pantheonController, - final boolean isSkipHeaderValidation, - final int metricsIntervalSec, - final int accountCommitInterval, - final boolean isSkipBlocks, - final boolean isSkipAccounts, - final Long worldStateOffset) - throws IOException { - checkNotNull(dataFilePath); - checkNotNull(pantheonController); - this.isSkipHeaderValidation = isSkipHeaderValidation; - final long startTime = System.currentTimeMillis(); - - checkNotNull(dataFilePath); - try (final FileChannel file = FileChannel.open(dataFilePath, StandardOpenOption.READ)) { - final FileRLPInput rlp = new FileRLPInput(file, true); - LOG.info("Import started."); - - final BlockchainImporter.ImportResult blockImportResults; - blockImportResults = - importBlockchain( - pantheonController, rlp, isSkipBlocks, metricsIntervalSec, worldStateOffset); - - if (!isSkipAccounts) { - final Hash worldStateRootHash; - worldStateRootHash = - importWorldState(pantheonController, rlp, metricsIntervalSec, accountCommitInterval); - validateWorldStateRootHash(pantheonController, worldStateRootHash); - } - - final long totalRunningSec = (System.currentTimeMillis() - startTime) / 1000; - final double totallRunningHours = totalRunningSec / (60.0 * 60); - - final String message = - format( - "Import finished in %,d seconds (%,1.2f hours).", - totalRunningSec, totallRunningHours); - METRICS_LOG.info(message); - - return blockImportResults; - } catch (final Exception e) { - final String message = format("Unable to import from file '%s'", dataFilePath.toString()); - throw new RuntimeException(message, e); - } finally { - pantheonController.close(); - } - } - - /** - * Imports the blockchain section of the file - * - * @param the consensus context type - * @param pantheonController the PantheonController that defines blockchain behavior - * @param rlp RLP Input File - * @param isSkipBlocks true if blocks in the import file should be skipped over. - * @param metricsIntervalSec seconds between logging progress metrics - * @param worldStateOffset file offset for the starting byte of the world state. Only relevant in - * combination with isSkipBlocks - * @return the import result - */ - private BlockImporter.ImportResult importBlockchain( - final PantheonController pantheonController, - final FileRLPInput rlp, - final Boolean isSkipBlocks, - final int metricsIntervalSec, - final Long worldStateOffset) { - final ProtocolSchedule protocolSchedule = pantheonController.getProtocolSchedule(); - final ProtocolContext context = pantheonController.getProtocolContext(); - final GenesisConfig genesis = pantheonController.getGenesisConfig(); - checkNotNull(isSkipBlocks); - final BlockHeader genesisHeader = genesis.getBlock().getHeader(); - - final long startTime = System.currentTimeMillis(); - long lapStartTime = startTime; - final long metricsIntervalMS = - 1_000L * metricsIntervalSec; // Use Millis here to make math easier - long nextMetricsTime = startTime + metricsIntervalMS; - long itemStartingOffset = 0; - final String logAction = isSkipBlocks ? "Skipped" : "Imported"; - - final long totalBlockCount = rlp.readLongScalar(); - - LOG.info( - format( - "Import file contains %,d blocks, starting at file offset %,d.", - totalBlockCount, rlp.currentOffset())); - - if (isSkipBlocks && worldStateOffset != null) { - // Skip blocks. Offset was given, so we don't even have to parse through the blocks - logFinalMetrics("Skipped", "block", startTime, Math.toIntExact(totalBlockCount)); - rlp.setTo(worldStateOffset); - return new BlockchainImporter.ImportResult(UInt256.ZERO, 0); - } - - final Function headerReader = - rlp2 -> BlockHeader.readFrom(rlp2, ScheduleBasedBlockHashFunction.create(protocolSchedule)); - - BlockHeader previousHeader = genesis.getBlock().getHeader(); - int blockCount = 0; - int lapCount = 0; - BlockHeader header = null; - BlockBody body = null; - List receipts = null; - try { - while (blockCount < totalBlockCount) { - header = null; // Reset so that if an error occurs, we log the correct data. - body = null; - receipts = null; - - blockCount++; - lapCount++; - itemStartingOffset = rlp.currentOffset(); - - if (isSkipBlocks && !(blockCount == totalBlockCount - 1)) { - // Skip block, unless it is the last one. If it's the last one, it will get parsed - // printed into the log, but not stored. This is for ops & dev debug purposes. - rlp.skipNext(); - rlp.skipNext(); - } else { - rlp.enterList(true); - header = headerReader.apply(rlp); - body = new BlockBody(rlp.readList(Transaction::readFrom), rlp.readList(headerReader)); - rlp.leaveList(); - receipts = rlp.readList(TransactionReceipt::readFrom); - if (header.getNumber() == genesisHeader.getNumber()) { - // Don't import genesis block - previousHeader = header; - continue; - } - final ProtocolSpec protocolSpec = - protocolSchedule.getByBlockNumber(header.getNumber()); - - if (!isSkipHeaderValidation) { - // Validate headers here because we already have the previous block, and can avoid - // an unnecessary lookup compared to doing the validation in the BlockImporter below. - final BlockHeaderValidator blockHeaderValidator = - protocolSpec.getBlockHeaderValidator(); - final boolean validHeader = - blockHeaderValidator.validateHeader( - header, previousHeader, context, HeaderValidationMode.FULL); - if (!validHeader) { - final String message = - format( - "Invalid header block number %,d at file position %,d", - header.getNumber(), itemStartingOffset); - throw new IllegalStateException(message); - } - } - - if (blockCount == 1) { - // Log the first block for ops & dev debug purposes. - LOG.info( - format( - "First Block, file offset=%,d\nHeader=%s\n\nBody=%s\n\n", - itemStartingOffset, header, body)); - } - - if (blockCount == totalBlockCount) { - // Log the last block for ops & dev debug purposes. - LOG.info( - format( - "Last Block, file offset=%,d\nHeader=%s\n\nBody=%s\n\n", - itemStartingOffset, header, body)); - } - - if (LOG.isTraceEnabled()) { - final String receiptsStr = - receipts == null ? null : Strings.join(receipts.iterator(), ','); - LOG.trace( - format( - "About to import block from file offset %,d with header=%s, body=%s, receipts=%s", - itemStartingOffset, header, body, receiptsStr)); - } - - final tech.pegasys.pantheon.ethereum.core.BlockImporter blockImporter; - blockImporter = protocolSpec.getBlockImporter(); - - if (!isSkipBlocks) { - // Do not validate headers here. They were already validated above, since we already - // have the previous block on-hand, we avoid the extra lookup BlockImporter would do - final boolean blockImported = - blockImporter.fastImportBlock( - context, new Block(header, body), receipts, HeaderValidationMode.NONE); - if (!blockImported) { - final String message = - format( - "Invalid header block number %,d at file position %,d", - header.getNumber(), itemStartingOffset); - throw new IllegalStateException(message); - } - } - } - - if (System.currentTimeMillis() >= nextMetricsTime) { - logLapMetrics(logAction, "block", startTime, blockCount, lapStartTime, lapCount); - lapCount = 0; - lapStartTime = System.currentTimeMillis(); - nextMetricsTime = lapStartTime + metricsIntervalMS; - } - previousHeader = header; - } - } catch (final RuntimeException e) { - final String receiptsStr = receipts == null ? null : Strings.join(receipts.iterator(), ','); - final String message = - format( - "Error importing block from file offset %,d with header=%s, body=%s, receipts=%s", - itemStartingOffset, header, body, receiptsStr); - throw new RuntimeException(message, e); - } - - logFinalMetrics(logAction, "block", startTime, blockCount); - return new BlockchainImporter.ImportResult( - context.getBlockchain().getChainHead().getTotalDifficulty(), blockCount); - } - - /** - * Imports the worldstate section of the file - * - * @param pantheonController the PantheonController that defines blockchain behavior - * @param rlp RLP Input File - * @param metricsIntervalSec seconds between logging progress metrics - * @param accountCommitInterval commit account state every n accounts - * @param the consensus context type - * @return root hash of the world state - */ - private Hash importWorldState( - final PantheonController pantheonController, - final FileRLPInput rlp, - final int metricsIntervalSec, - final int accountCommitInterval) { - final ProtocolContext context = pantheonController.getProtocolContext(); - final MutableWorldState worldState = context.getWorldStateArchive().getMutable(); - WorldUpdater worldStateUpdater = worldState.updater(); - - final long startTime = System.currentTimeMillis(); - long lapStartTime = startTime; - final long metricsIntervalMS = - 1_000L * metricsIntervalSec; // Use Millis here to make math easier - long nextMetricsTime = startTime + metricsIntervalMS; - long itemStartingOffset = 0; - - int count = 0; - int lapCount = 0; - Address address = null; - MutableAccount account = null; - Long nonce = null; - Wei balance = null; - Hash storageRoot = null; - Hash codeHash = null; - - LOG.info(format("Starting Account Import at file offset %,d", rlp.currentOffset())); - LOG.info("Initial world state root hash: {}", worldState.rootHash()); - try { - while (!rlp.isDone() && rlp.nextSize() == 20) { - address = null; // reset to null here so we can log useful info if error occurs - account = null; - nonce = null; - balance = null; - storageRoot = null; - codeHash = null; - - count++; - lapCount++; - itemStartingOffset = rlp.currentOffset(); - - address = Address.readFrom(rlp); - - rlp.enterList(true); - nonce = rlp.readLongScalar(); - balance = rlp.readUInt256Scalar(Wei::wrap); - storageRoot = Hash.wrap(rlp.readBytes32()); - codeHash = Hash.wrap(rlp.readBytes32()); - rlp.leaveList(); - - if (LOG.isTraceEnabled()) { - LOG.trace( - format( - "About to import account from file offset %,d with address=%s, nonce=%s, balance=%s", - itemStartingOffset, address, nonce, balance)); - } - - account = worldStateUpdater.createAccount(address, nonce, balance); - - final BytesValue code = rlp.readBytesValue(); - account.setCode(code); - - // hash code and compare to codehash - verifyCodeHash(address, code, codeHash); - - while (!rlp.isDone() && rlp.nextSize() == 32) { - // Read an Account Storage Entry. We know the key is 32 bytes, vs 20 bytes if we started - // with the next Account - final UInt256 key = rlp.readUInt256Scalar(); - final UInt256 value = rlp.readUInt256Scalar(); - account.setStorageValue(key, value); - } - - // Add verification for each account's storage root hash here, if debugging state root - // becomes a problem - // Functionally, the single check at the end is enough, but if that fails, it doesn't give - // any - // indication about which account started the mismatch. That check can go here if it - // becomes necessary. - - if (count == 1) { - // Log the first account for ops & dev debug purposes. - LOG.info( - format( - "Importing first account number %d at file offset %,d. address=%s, account=%s, account nonce=%s, account balance=%s, account storage root=%s, account code hash=%s", - count, - itemStartingOffset, - address, - account, - nonce, - balance, - storageRoot, - codeHash)); - } - - if (count % accountCommitInterval == 0) { - worldStateUpdater.commit(); - - worldStateUpdater = - worldState.updater(); // Get a new updater, so the old one can GC itself. - } - if (count % accountCommitInterval == 0) { - worldState.persist(); - } - if (System.currentTimeMillis() >= nextMetricsTime) { - logLapMetrics("Imported", "account", startTime, count, lapStartTime, lapCount); - lapCount = 0; - lapStartTime = System.currentTimeMillis(); - nextMetricsTime = lapStartTime + metricsIntervalMS; - } - } - - // Log the last account for ops & dev debug purposes. - LOG.info( - format( - "Importing last account number %d at file offset %,d. address=%s, account=%s, account nonce=%s, account balance=%s, account storage root=%s, account code hash=%s", - count, itemStartingOffset, address, account, nonce, balance, storageRoot, codeHash)); - - // Do a final commit & persist. - worldStateUpdater.commit(); - worldState.persist(); - } catch (final Exception e) { - final String message = - format( - "Error importing account number %d at file offset %,d. address=%s, account=%s, account nonce=%s, account balance=%s, account storage root=%s, account code hash=%s", - count, itemStartingOffset, address, account, nonce, balance, storageRoot, codeHash); - throw new RuntimeException(message, e); - } - logFinalMetrics("Imported", "account", startTime, count); - LOG.info("Final world state root hash: {}", worldState.rootHash()); - return worldState.rootHash(); - } - - /** - * Verifies the account code against it's stated hash - * - * @param address Address of the account being checked - * @param code Code to be verified - * @param codeHashFromState stated hash of the code to verify - */ - private void verifyCodeHash( - final Address address, final BytesValue code, final Hash codeHashFromState) { - final Hash myHash = Hash.hash(code); - if (!myHash.equals(codeHashFromState)) { - final String message = - format( - "Code hash does not match for account %s. Expected %s, but got %s for code %s", - address, codeHashFromState, myHash, code); - throw new RuntimeException(message); - } - } - - /** - * Verifies the calculated world state's root hash against the stated value in the blockain's head - * block - * - * @param pantheonController the PantheonController that defines blockchain behavior - * @param worldStateRootHash calculated world state's root hash - * @param the consensus context type - */ - private void validateWorldStateRootHash( - final PantheonController pantheonController, final Hash worldStateRootHash) { - final ProtocolContext context = pantheonController.getProtocolContext(); - final MutableBlockchain blockchain = context.getBlockchain(); - final Optional header = blockchain.getBlockHeader(blockchain.getChainHeadHash()); - - if (!header.isPresent()) { - final String message = - "Can not get header for blockchain head, using hash " + blockchain.getChainHeadHash(); - throw new IllegalStateException(message); - } - final Hash blockStorageHash = header.get().getStateRoot(); - if (!blockStorageHash.equals(worldStateRootHash)) { - final String message = - format( - "Invalid block: state root mismatch (expected=%s, actual=%s)", - blockStorageHash, worldStateRootHash); - throw new RuntimeException(message); - } - } - - /** - * logs progress metrics for each 'lap' of execution. The total stats are also logged. Laps are - * defined as number of seconds between logging progress metrics. - * - * @param action Action being performed on itemName. (eg "Imported", "Skipped") - * @param itemName Item being tracked. (eg "account", "block"). identifier for what is being - * tracked - * @param startTime timestamp for when the overall process started - * @param totalItemCount total items processed - * @param lapStartTime timestamp for when the current lap processing started - * @param lapItemCount items processed this lap - */ - private void logLapMetrics( - final String action, - final String itemName, - final long startTime, - final int totalItemCount, - final long lapStartTime, - final int lapItemCount) { - final long curTime = System.currentTimeMillis(); - long lapRunningSec = (curTime - lapStartTime) / 1000; - long totalRunningSec = (curTime - startTime) / 1000; - lapRunningSec = lapRunningSec > 0 ? lapRunningSec : 1; // Set min time to 1 sec. - totalRunningSec = totalRunningSec > 0 ? totalRunningSec : 1; // Set min time to 1 sec. - final long lapItemPerSec = lapItemCount / lapRunningSec; - final long totalItemPerSec = totalItemCount / totalRunningSec; - final String message = - format( - "%s %,7d %ss in %3d seconds (%,5d %ss/sec). Totals: %,7d %ss in %3d seconds (%,5d %ss/sec).", - action, - lapItemCount, - itemName, - lapRunningSec, - lapItemPerSec, - itemName, - totalItemCount, - itemName, - totalRunningSec, - totalItemPerSec, - itemName); - METRICS_LOG.info(message); - } - - /** - * logs the final metrics for this process. - * - * @param action Action being performed on itemName. (eg "Imported", "Skipped") - * @param itemName Item being tracked. (eg "account", "block"). identifier for what is being - * tracked - * @param startTime timestamp for when the overall process started - * @param totalItemCount total items processed - */ - private void logFinalMetrics( - final String action, final String itemName, final long startTime, final int totalItemCount) { - final long curTime = System.currentTimeMillis(); - long totalRunningSec = (curTime - startTime) / 1000; - totalRunningSec = totalRunningSec > 0 ? totalRunningSec : 1; // Set min time to 1 sec. - final long totalItemPerSec = totalItemCount / totalRunningSec; - final String message = - format( - "%s %,d %ss in %,d seconds (%,d %ss/sec).", - action, totalItemCount, itemName, totalRunningSec, totalItemPerSec, itemName); - METRICS_LOG.info(message); - } -} diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java index cf84247907..6b401b0aa9 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java @@ -91,7 +91,7 @@ public abstract class CommandTestAbstract { final PantheonCommand pantheonCommand = new PantheonCommand( - mockBlockImporter, null, mockRunnerBuilder, mockControllerBuilder, mockSyncConfBuilder); + mockBlockImporter, mockRunnerBuilder, mockControllerBuilder, mockSyncConfBuilder); // parse using Ansi.OFF to be able to assert on non formatted output results pantheonCommand.parse( diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockchainImporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockchainImporterTest.java deleted file mode 100644 index c53e110fa3..0000000000 --- a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockchainImporterTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.util; - -import static org.assertj.core.api.Assertions.assertThat; -import static tech.pegasys.pantheon.controller.KeyPairUtil.loadKeyPair; - -import tech.pegasys.pantheon.controller.MainnetPantheonController; -import tech.pegasys.pantheon.controller.PantheonController; -import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair; -import tech.pegasys.pantheon.ethereum.chain.GenesisConfig; -import tech.pegasys.pantheon.ethereum.core.MiningParameters; -import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.mainnet.MainnetProtocolSchedule; -import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; -import tech.pegasys.pantheon.util.uint.UInt256; - -import java.io.File; -import java.net.URL; -import java.nio.file.Path; - -import com.google.common.base.Charsets; -import com.google.common.io.Resources; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -/** Tests for {@link BlockchainImporter}. */ -public final class BlockchainImporterTest { - - @Rule public final TemporaryFolder folder = new TemporaryFolder(); - - BlockchainImporter blockImporter = new BlockchainImporter(); - - @Test - public void importBlocksWithoutValidation() throws Exception { - blockImportTest(true); - } - - @Test - public void importBlocksWithValidation() throws Exception { - blockImportTest(false); - } - - private void blockImportTest(final boolean skipValidation) throws Exception { - final URL importFileURL = - getClass().getClassLoader().getResource("tech/pegasys/pantheon/util/blockchain-import.bin"); - assertThat(importFileURL).isNotNull(); - - final Path source = new File(importFileURL.toURI()).toPath(); - final Path target = folder.newFolder().toPath(); - - final URL genesisJsonUrl = - getClass() - .getClassLoader() - .getResource("tech/pegasys/pantheon/util/blockchain-import-genesis-file.json"); - assertThat(genesisJsonUrl).isNotNull(); - final String genesisJson = Resources.toString(genesisJsonUrl, Charsets.UTF_8); - final KeyPair keyPair = loadKeyPair(target); - - final ProtocolSchedule protocolSchedule = MainnetProtocolSchedule.create(); - final MiningParameters miningParams = new MiningParametersTestBuilder().enabled(false).build(); - final GenesisConfig genesisConfig = GenesisConfig.fromJson(genesisJson, protocolSchedule); - final PantheonController ctrl = - MainnetPantheonController.init( - target, - genesisConfig, - SynchronizerConfiguration.builder().build(), - miningParams, - keyPair); - final BlockchainImporter.ImportResult result = - blockImporter.importBlockchain(source, ctrl, skipValidation, 1, 1, false, false, null); - System.out.println(source); - System.out.println(target); - - System.out.println(result); - assertThat(result.count).isEqualTo(33); - assertThat(result.td).isEqualTo(UInt256.of(4357120)); - } -}