From f9e50ae8c6a8815f6da9ae942dd69c235399a557 Mon Sep 17 00:00:00 2001 From: mbaxter Date: Wed, 14 Aug 2019 12:56:24 -0400 Subject: [PATCH] [PIE-1810] Update export subcommand to export blocks in rlp format (#1852) Signed-off-by: Adrian Sutton --- .../java/tech/pegasys/pantheon/Pantheon.java | 12 +- .../pantheon/chainexport/BlockExporter.java | 94 +++++++ .../chainexport/RlpBlockExporter.java | 35 +++ ...inImporter.java => JsonBlockImporter.java} | 11 +- .../RlpBlockImporter.java} | 11 +- .../chainimport/{ => internal}/BlockData.java | 4 +- .../chainimport/{ => internal}/ChainData.java | 2 +- .../{ => internal}/TransactionData.java | 2 +- .../pegasys/pantheon/cli/PantheonCommand.java | 31 ++- .../subcommands/blocks/BlockExportFormat.java | 17 ++ ...lockFormat.java => BlockImportFormat.java} | 2 +- .../subcommands/blocks/BlocksSubCommand.java | 181 ++++++++---- .../pegasys/pantheon/util/BlockExporter.java | 88 ------ .../chainexport/RlpBlockExporterTest.java | 245 ++++++++++++++++ ...erTest.java => JsonBlockImporterTest.java} | 22 +- .../RlpBlockImporterTest.java} | 15 +- .../pantheon/cli/CommandTestAbstract.java | 36 +-- .../blocks/BlocksSubCommandTest.java | 261 ++++++++++++------ .../pantheon/util/BlockExporterTest.java | 111 -------- 19 files changed, 772 insertions(+), 408 deletions(-) create mode 100644 pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java create mode 100644 pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java rename pantheon/src/main/java/tech/pegasys/pantheon/chainimport/{ChainImporter.java => JsonBlockImporter.java} (96%) rename pantheon/src/main/java/tech/pegasys/pantheon/{util/BlockImporter.java => chainimport/RlpBlockImporter.java} (96%) rename pantheon/src/main/java/tech/pegasys/pantheon/chainimport/{ => internal}/BlockData.java (96%) rename pantheon/src/main/java/tech/pegasys/pantheon/chainimport/{ => internal}/ChainData.java (94%) rename pantheon/src/main/java/tech/pegasys/pantheon/chainimport/{ => internal}/TransactionData.java (98%) create mode 100644 pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java rename pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/{BlockFormat.java => BlockImportFormat.java} (95%) delete mode 100644 pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java create mode 100644 pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java rename pantheon/src/test/java/tech/pegasys/pantheon/chainimport/{ChainImporterTest.java => JsonBlockImporterTest.java} (95%) rename pantheon/src/test/java/tech/pegasys/pantheon/{util/BlockImporterTest.java => chainimport/RlpBlockImporterTest.java} (91%) delete mode 100644 pantheon/src/test/java/tech/pegasys/pantheon/util/BlockExporterTest.java diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java b/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java index 638de717c2..fe0e549ee9 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java @@ -14,12 +14,12 @@ package tech.pegasys.pantheon; import static org.apache.logging.log4j.LogManager.getLogger; -import tech.pegasys.pantheon.chainimport.ChainImporter; +import tech.pegasys.pantheon.chainexport.RlpBlockExporter; +import tech.pegasys.pantheon.chainimport.JsonBlockImporter; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.PantheonCommand; import tech.pegasys.pantheon.controller.PantheonController; import tech.pegasys.pantheon.services.PantheonPluginContextImpl; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import org.apache.logging.log4j.Logger; import picocli.CommandLine.RunLast; @@ -36,9 +36,9 @@ public final class Pantheon { final PantheonCommand pantheonCommand = new PantheonCommand( logger, - new BlockImporter(), - new BlockExporter(), - ChainImporter::new, + new RlpBlockImporter(), + JsonBlockImporter::new, + RlpBlockExporter::new, new RunnerBuilder(), new PantheonController.Builder(), new PantheonPluginContextImpl(), diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java new file mode 100644 index 0000000000..3281343862 --- /dev/null +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java @@ -0,0 +1,94 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.chainexport; + +import static com.google.common.base.Preconditions.checkArgument; + +import tech.pegasys.pantheon.ethereum.chain.Blockchain; +import tech.pegasys.pantheon.ethereum.core.Block; +import tech.pegasys.pantheon.ethereum.core.BlockHeader; +import tech.pegasys.pantheon.ethereum.core.Hash; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Optional; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** Pantheon Block Export Util. */ +public abstract class BlockExporter { + private static final Logger LOG = LogManager.getLogger(); + private final Blockchain blockchain; + + protected BlockExporter(final Blockchain blockchain) { + this.blockchain = blockchain; + } + + /** + * Export blocks that are stored in Pantheon's block storage. + * + * @param outputFile the path at which to save the exported block data + * @param maybeStartBlock the starting index of the block list to export (inclusive) + * @param maybeEndBlock the ending index of the block list to export (exclusive), if not specified + * a single block will be export + * @throws IOException if an I/O error occurs while writing data to disk + */ + public void exportBlocks( + final File outputFile, + final Optional maybeStartBlock, + final Optional maybeEndBlock) + throws IOException { + + // Get range to export + final long startBlock = maybeStartBlock.orElse(BlockHeader.GENESIS_BLOCK_NUMBER); + final long endBlock = maybeEndBlock.orElse(blockchain.getChainHeadBlockNumber() + 1L); + checkArgument(startBlock >= 0 && endBlock >= 0, "Start and end blocks must be greater than 0."); + checkArgument(startBlock < endBlock, "Start block must be less than end block"); + + // Append to file if a range is specified + final boolean append = maybeStartBlock.isPresent(); + FileOutputStream outputStream = new FileOutputStream(outputFile, append); + + LOG.info( + "Exporting blocks [{},{}) to file {} (appending: {})", + startBlock, + endBlock, + outputFile.toString(), + Boolean.toString(append)); + + long blockNumber = 0L; + for (long i = startBlock; i < endBlock; i++) { + Optional blockHash = blockchain.getBlockHashByNumber(i); + if (blockHash.isEmpty()) { + LOG.warn("Unable to export blocks [{} - {}). Blocks not found.", i, endBlock); + break; + } + + final Block block = blockchain.getBlockByHash(blockHash.get()); + blockNumber = block.getHeader().getNumber(); + if (blockNumber % 100 == 0) { + LOG.info("Export at block {}", blockNumber); + } + + exportBlock(outputStream, block); + } + + outputStream.close(); + LOG.info("Export complete at block {}", blockNumber); + } + + protected abstract void exportBlock(final FileOutputStream outputStream, final Block block) + throws IOException; +} diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java new file mode 100644 index 0000000000..63b348f7a6 --- /dev/null +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.chainexport; + +import tech.pegasys.pantheon.ethereum.chain.Blockchain; +import tech.pegasys.pantheon.ethereum.core.Block; +import tech.pegasys.pantheon.ethereum.rlp.RLP; +import tech.pegasys.pantheon.util.bytes.BytesValue; + +import java.io.FileOutputStream; +import java.io.IOException; + +public class RlpBlockExporter extends BlockExporter { + + public RlpBlockExporter(final Blockchain blockchain) { + super(blockchain); + } + + @Override + protected void exportBlock(final FileOutputStream outputStream, final Block block) + throws IOException { + final BytesValue rlp = RLP.encode(block::writeTo); + outputStream.write(rlp.getArrayUnsafe()); + } +} diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainImporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/JsonBlockImporter.java similarity index 96% rename from pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainImporter.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/JsonBlockImporter.java index 6f1088f0cb..9aabb91729 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainImporter.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/JsonBlockImporter.java @@ -12,6 +12,8 @@ */ package tech.pegasys.pantheon.chainimport; +import tech.pegasys.pantheon.chainimport.internal.BlockData; +import tech.pegasys.pantheon.chainimport.internal.ChainData; import tech.pegasys.pantheon.config.GenesisConfigOptions; import tech.pegasys.pantheon.controller.PantheonController; import tech.pegasys.pantheon.ethereum.blockcreation.MiningCoordinator; @@ -39,13 +41,18 @@ import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -public class ChainImporter { +/** + * Tool for importing blocks with transactions from human-readable json. + * + * @param The consensus algorithm context + */ +public class JsonBlockImporter { private static final Logger LOG = LogManager.getLogger(); private final ObjectMapper mapper; private final PantheonController controller; - public ChainImporter(final PantheonController controller) { + public JsonBlockImporter(final PantheonController controller) { this.controller = controller; mapper = new ObjectMapper(); // Jdk8Module allows us to easily parse {@code Optional} values from json diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockImporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/RlpBlockImporter.java similarity index 96% rename from pantheon/src/main/java/tech/pegasys/pantheon/util/BlockImporter.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/RlpBlockImporter.java index 0b3d476999..a9a7cca3a4 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockImporter.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/RlpBlockImporter.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.util; +package tech.pegasys.pantheon.chainimport; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.logging.log4j.LogManager.getLogger; @@ -41,8 +41,8 @@ import java.util.concurrent.Semaphore; import com.google.common.base.MoreObjects; import org.apache.logging.log4j.Logger; -/** Pantheon Block Import Util. */ -public class BlockImporter { +/** Tool for importing rlp-encoded block data from files. */ +public class RlpBlockImporter { private static final Logger LOG = getLogger(); private final Semaphore blockBacklog = new Semaphore(2); @@ -60,7 +60,7 @@ public class BlockImporter { * @return the import result * @throws IOException On Failure */ - public BlockImporter.ImportResult importBlockchain( + public RlpBlockImporter.ImportResult importBlockchain( final Path blocks, final PantheonController pantheonController) throws IOException { final ProtocolSchedule protocolSchedule = pantheonController.getProtocolSchedule(); final ProtocolContext context = pantheonController.getProtocolContext(); @@ -125,7 +125,8 @@ public class BlockImporter { if (previousBlockFuture != null) { previousBlockFuture.join(); } - return new BlockImporter.ImportResult(blockchain.getChainHead().getTotalDifficulty(), count); + return new RlpBlockImporter.ImportResult( + blockchain.getChainHead().getTotalDifficulty(), count); } finally { validationExecutor.shutdownNow(); try { diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/BlockData.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/BlockData.java similarity index 96% rename from pantheon/src/main/java/tech/pegasys/pantheon/chainimport/BlockData.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/BlockData.java index 06b20562d3..ef8743aadd 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/BlockData.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/BlockData.java @@ -10,9 +10,9 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.chainimport; +package tech.pegasys.pantheon.chainimport.internal; -import tech.pegasys.pantheon.chainimport.TransactionData.NonceProvider; +import tech.pegasys.pantheon.chainimport.internal.TransactionData.NonceProvider; import tech.pegasys.pantheon.ethereum.core.Account; import tech.pegasys.pantheon.ethereum.core.Address; import tech.pegasys.pantheon.ethereum.core.Hash; diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainData.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/ChainData.java similarity index 94% rename from pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainData.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/ChainData.java index 7c3127fadc..6b7cd38fca 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/ChainData.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/ChainData.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.chainimport; +package tech.pegasys.pantheon.chainimport.internal; import java.util.List; diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/TransactionData.java b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/TransactionData.java similarity index 98% rename from pantheon/src/main/java/tech/pegasys/pantheon/chainimport/TransactionData.java rename to pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/TransactionData.java index fb133ea481..cbcf02e221 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/TransactionData.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/TransactionData.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.chainimport; +package tech.pegasys.pantheon.chainimport.internal; import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair; import tech.pegasys.pantheon.crypto.SECP256K1.PrivateKey; diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java index 9d63456bd2..743424f621 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java @@ -31,6 +31,7 @@ import static tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration.DEFA import tech.pegasys.pantheon.PantheonInfo; import tech.pegasys.pantheon.Runner; import tech.pegasys.pantheon.RunnerBuilder; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.config.EthNetworkConfig; import tech.pegasys.pantheon.cli.config.NetworkName; import tech.pegasys.pantheon.cli.converter.MetricCategoryConverter; @@ -51,7 +52,8 @@ import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand; import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand.KeyLoader; import tech.pegasys.pantheon.cli.subcommands.RetestethSubCommand; import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand; -import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ChainImporterFactory; +import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.JsonBlockImporterFactory; +import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.RlpBlockExporterFactory; import tech.pegasys.pantheon.cli.subcommands.operator.OperatorSubCommand; import tech.pegasys.pantheon.cli.subcommands.rlp.RLPSubCommand; import tech.pegasys.pantheon.cli.util.ConfigOptionSearchAndRunHandler; @@ -93,8 +95,6 @@ import tech.pegasys.pantheon.services.PantheonEventsImpl; import tech.pegasys.pantheon.services.PantheonPluginContextImpl; import tech.pegasys.pantheon.services.PicoCLIOptionsImpl; import tech.pegasys.pantheon.services.kvstore.RocksDbConfiguration; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import tech.pegasys.pantheon.util.PermissioningConfigurationValidator; import tech.pegasys.pantheon.util.bytes.BytesValue; import tech.pegasys.pantheon.util.number.Fraction; @@ -153,12 +153,12 @@ import picocli.CommandLine.ParameterException; public class PantheonCommand implements DefaultCommandValues, Runnable { private final Logger logger; - private final ChainImporterFactory chainImporterFactory; private CommandLine commandLine; - private final BlockImporter blockImporter; - private final BlockExporter blockExporter; + private final RlpBlockImporter rlpBlockImporter; + private final JsonBlockImporterFactory jsonBlockImporterFactory; + private final RlpBlockExporterFactory rlpBlockExporterFactory; final NetworkingOptions networkingOptions = NetworkingOptions.create(); final SynchronizerOptions synchronizerOptions = SynchronizerOptions.create(); @@ -633,17 +633,17 @@ public class PantheonCommand implements DefaultCommandValues, Runnable { public PantheonCommand( final Logger logger, - final BlockImporter blockImporter, - final BlockExporter blockExporter, - final ChainImporterFactory chainImporterFactory, + final RlpBlockImporter rlpBlockImporter, + final JsonBlockImporterFactory jsonBlockImporterFactory, + final RlpBlockExporterFactory rlpBlockExporterFactory, final RunnerBuilder runnerBuilder, final PantheonController.Builder controllerBuilderFactory, final PantheonPluginContextImpl pantheonPluginContext, final Map environment) { this.logger = logger; - this.blockImporter = blockImporter; - this.blockExporter = blockExporter; - this.chainImporterFactory = chainImporterFactory; + this.rlpBlockImporter = rlpBlockImporter; + this.rlpBlockExporterFactory = rlpBlockExporterFactory; + this.jsonBlockImporterFactory = jsonBlockImporterFactory; this.runnerBuilder = runnerBuilder; this.controllerBuilderFactory = controllerBuilderFactory; this.pantheonPluginContext = pantheonPluginContext; @@ -690,7 +690,10 @@ public class PantheonCommand implements DefaultCommandValues, Runnable { commandLine.addSubcommand( BlocksSubCommand.COMMAND_NAME, new BlocksSubCommand( - blockImporter, blockExporter, chainImporterFactory, resultHandler.out())); + rlpBlockImporter, + jsonBlockImporterFactory, + rlpBlockExporterFactory, + resultHandler.out())); commandLine.addSubcommand( PublicKeySubCommand.COMMAND_NAME, new PublicKeySubCommand(resultHandler.out(), getKeyLoader())); @@ -1363,7 +1366,7 @@ public class PantheonCommand implements DefaultCommandValues, Runnable { } } - private Path dataDir() { + public Path dataDir() { if (isFullInstantiation()) { return standaloneCommands.dataPath.toAbsolutePath(); } else if (isDocker) { diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java new file mode 100644 index 0000000000..2fda6eb6bf --- /dev/null +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java @@ -0,0 +1,17 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.cli.subcommands.blocks; + +public enum BlockExportFormat { + RLP +} diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockFormat.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockImportFormat.java similarity index 95% rename from pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockFormat.java rename to pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockImportFormat.java index e7207986c9..45a6575ab1 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockFormat.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockImportFormat.java @@ -12,7 +12,7 @@ */ package tech.pegasys.pantheon.cli.subcommands.blocks; -public enum BlockFormat { +public enum BlockImportFormat { RLP, JSON } diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java index 13d48baac3..685214381f 100644 --- a/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java +++ b/pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java @@ -13,34 +13,33 @@ package tech.pegasys.pantheon.cli.subcommands.blocks; import static com.google.common.base.Preconditions.checkNotNull; -import static java.nio.charset.StandardCharsets.UTF_8; import static tech.pegasys.pantheon.cli.DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP; import static tech.pegasys.pantheon.cli.DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP; import static tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.COMMAND_NAME; -import tech.pegasys.pantheon.chainimport.ChainImporter; +import tech.pegasys.pantheon.chainexport.RlpBlockExporter; +import tech.pegasys.pantheon.chainimport.JsonBlockImporter; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.PantheonCommand; import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ExportSubCommand; import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ImportSubCommand; import tech.pegasys.pantheon.controller.PantheonController; +import tech.pegasys.pantheon.ethereum.ProtocolContext; +import tech.pegasys.pantheon.ethereum.chain.Blockchain; import tech.pegasys.pantheon.ethereum.core.Address; -import tech.pegasys.pantheon.ethereum.core.Block; import tech.pegasys.pantheon.ethereum.core.MiningParameters; import tech.pegasys.pantheon.ethereum.core.Wei; import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration; import tech.pegasys.pantheon.metrics.prometheus.MetricsService; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import tech.pegasys.pantheon.util.bytes.BytesValue; -import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; +import java.nio.file.Paths; import java.util.Optional; import io.vertx.core.Vertx; @@ -75,20 +74,21 @@ public class BlocksSubCommand implements Runnable { @Spec private CommandSpec spec; // Picocli injects reference to command spec - private final BlockImporter blockImporter; - private final BlockExporter blockExporter; - private final ChainImporterFactory chainImporterFactory; + private final RlpBlockImporter rlpBlockImporter; + private final JsonBlockImporterFactory jsonBlockImporterFactory; + + private final RlpBlockExporterFactory rlpBlockExporterFactory; private final PrintStream out; public BlocksSubCommand( - final BlockImporter blockImporter, - final BlockExporter blockExporter, - final ChainImporterFactory chainImporterFactory, + final RlpBlockImporter rlpBlockImporter, + final JsonBlockImporterFactory jsonBlockImporterFactory, + final RlpBlockExporterFactory rlpBlockExporterFactory, final PrintStream out) { - this.blockImporter = blockImporter; - this.blockExporter = blockExporter; - this.chainImporterFactory = chainImporterFactory; + this.rlpBlockImporter = rlpBlockImporter; + this.rlpBlockExporterFactory = rlpBlockExporterFactory; + this.jsonBlockImporterFactory = jsonBlockImporterFactory; this.out = out; } @@ -125,7 +125,7 @@ public class BlocksSubCommand implements Runnable { description = "The type of data to be imported, possible values are: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", arity = "1..1") - private final BlockFormat format = BlockFormat.RLP; + private final BlockImportFormat format = BlockImportFormat.RLP; @SuppressWarnings("unused") @Spec @@ -136,7 +136,8 @@ public class BlocksSubCommand implements Runnable { LOG.info("Import {} block data from {}", format, blocksImportFile); checkCommand(parentCommand); - checkNotNull(parentCommand.blockImporter); + checkNotNull(parentCommand.rlpBlockImporter); + checkNotNull(parentCommand.jsonBlockImporterFactory); Optional metricsService = initMetrics(parentCommand); @@ -199,14 +200,14 @@ public class BlocksSubCommand implements Runnable { private void importJsonBlocks(final PantheonController controller, final Path path) throws IOException { - ChainImporter importer = parentCommand.chainImporterFactory.get(controller); + JsonBlockImporter importer = parentCommand.jsonBlockImporterFactory.get(controller); final String jsonData = Files.readString(path); importer.importChain(jsonData); } private void importRlpBlocks(final PantheonController controller, final Path path) throws IOException { - parentCommand.blockImporter.importBlockchain(path, controller); + parentCommand.rlpBlockImporter.importBlockchain(path, controller); } } @@ -226,7 +227,6 @@ public class BlocksSubCommand implements Runnable { @Option( names = "--start-block", - required = true, paramLabel = MANDATORY_LONG_FORMAT_HELP, description = "the starting index of the block list to export (inclusive)", arity = "1..1") @@ -241,71 +241,125 @@ public class BlocksSubCommand implements Runnable { arity = "1..1") private final Long endBlock = null; + @Option( + names = "--format", + hidden = true, + description = + "The format to export, possible values are: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", + arity = "1..1") + private final BlockExportFormat format = BlockExportFormat.RLP; + @Option( names = "--to", + required = true, paramLabel = MANDATORY_FILE_FORMAT_HELP, description = "File to write the block list instead of standard output", arity = "1..1") private File blocksExportFile = null; + @SuppressWarnings("unused") + @Spec + private CommandSpec spec; + @Override public void run() { - LOG.info("Runs export sub command"); + LOG.info("Export {} block data to file {}", format, blocksExportFile.toPath()); checkCommand(this, startBlock, endBlock); + final Optional metricsService = initMetrics(parentCommand); - Optional metricsService = initMetrics(parentCommand); - + final PantheonController controller = createPantheonController(); try { - - final BlockExporter.ExportResult exportResult = - parentCommand.blockExporter.exportBlockchain( - parentCommand.parentCommand.buildController(), startBlock, endBlock); - - outputBlock(exportResult.blocks); - - if (exportResult.blocks.isEmpty()) { - throw new ExecutionException(new CommandLine(this), "No block found at the given index"); - } else if (!exportResult.allBlocksAreFound) { - throw new ExecutionException( - new CommandLine(this), - "Partial export due to inability to recover all requested blocks"); + switch (format) { + case RLP: + exportRlpFormat(controller); + break; + default: + throw new ParameterException( + spec.commandLine(), "Unsupported format: " + format.toString()); } - + } catch (IOException e) { + throw new ExecutionException( + spec.commandLine(), "An error occurred while exporting blocks.", e); } finally { metricsService.ifPresent(MetricsService::stop); } } - private static void checkCommand( + private PantheonController createPantheonController() { + return parentCommand.parentCommand.buildController(); + } + + private void exportRlpFormat(final PantheonController controller) throws IOException { + final ProtocolContext context = controller.getProtocolContext(); + RlpBlockExporter exporter = + parentCommand.rlpBlockExporterFactory.get(context.getBlockchain()); + exporter.exportBlocks(blocksExportFile, getStartBlock(), getEndBlock()); + } + + private void checkCommand( final ExportSubCommand exportSubCommand, final Long startBlock, final Long endBlock) { checkNotNull(exportSubCommand.parentCommand); - checkNotNull(exportSubCommand.parentCommand.blockExporter); - checkNotNull(startBlock); - if (startBlock < 0) { - throw new CommandLine.ParameterException( - new CommandLine(exportSubCommand), - "--start-block must be greater than or equal to zero"); + + final Optional maybeStartBlock = getStartBlock(); + final Optional maybeEndBlock = getEndBlock(); + + maybeStartBlock + .filter(blockNum -> blockNum < 0) + .ifPresent( + (blockNum) -> { + throw new CommandLine.ParameterException( + spec.commandLine(), + "Parameter --start-block (" + + blockNum + + ") must be greater than or equal to zero."); + }); + + maybeEndBlock + .filter(blockNum -> blockNum < 0) + .ifPresent( + (blockNum) -> { + throw new CommandLine.ParameterException( + spec.commandLine(), + "Parameter --end-block (" + + blockNum + + ") must be greater than or equal to zero."); + }); + + if (maybeStartBlock.isPresent() && maybeEndBlock.isPresent()) { + if (endBlock <= startBlock) { + throw new CommandLine.ParameterException( + spec.commandLine(), + "Parameter --end-block (" + + endBlock + + ") must be greater start block (" + + startBlock + + ")."); + } } - if (endBlock != null && startBlock >= endBlock) { + + // Error if data directory is empty + Path databasePath = + Paths.get( + parentCommand.parentCommand.dataDir().toAbsolutePath().toString(), + PantheonController.DATABASE_PATH); + File databaseDirectory = new File(databasePath.toString()); + if (!databaseDirectory.isDirectory() || databaseDirectory.list().length == 0) { + // Empty data directory, nothing to export throw new CommandLine.ParameterException( - new CommandLine(exportSubCommand), "--end-block must be greater than --start-block"); + spec.commandLine(), + "Chain is empty. Unable to export blocks from specified data directory: " + + databaseDirectory.toString()); } } - private void outputBlock(final List blocks) { - if (blocksExportFile != null) { - final Path path = blocksExportFile.toPath(); - try (final BufferedWriter fileWriter = Files.newBufferedWriter(path, UTF_8)) { - fileWriter.write(blocks.toString()); - } catch (final IOException e) { - throw new ExecutionException( - new CommandLine(this), "An error occurred while trying to write the exported blocks"); - } - } else { - parentCommand.out.println(blocks.toString()); - } + private Optional getStartBlock() { + return Optional.ofNullable(startBlock); + } + + private Optional getEndBlock() { + return Optional.ofNullable(endBlock); } } @@ -326,7 +380,12 @@ public class BlocksSubCommand implements Runnable { } @FunctionalInterface - public interface ChainImporterFactory { - ChainImporter get(PantheonController controller); + public interface JsonBlockImporterFactory { + JsonBlockImporter get(PantheonController controller); + } + + @FunctionalInterface + public interface RlpBlockExporterFactory { + RlpBlockExporter get(Blockchain blockchain); } } diff --git a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java b/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java deleted file mode 100644 index d3300535af..0000000000 --- a/pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.util; - -import tech.pegasys.pantheon.controller.PantheonController; -import tech.pegasys.pantheon.ethereum.ProtocolContext; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.Hash; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import com.google.common.base.MoreObjects; - -/** Pantheon Block Export Util. */ -public class BlockExporter { - - /** - * Export blocks that are stored in Pantheon's block storage. - * - * @param pantheonController the PantheonController that defines blockchain behavior - * @param the consensus context type - * @param startBlock the starting index of the block list to export (inclusive) - * @param endBlock the ending index of the block list to export (exclusive), if not specified a - * single block will be export - * @return the export result - */ - public ExportResult exportBlockchain( - final PantheonController pantheonController, final Long startBlock, final Long endBlock) { - - final ProtocolContext context = pantheonController.getProtocolContext(); - final MutableBlockchain blockchain = context.getBlockchain(); - - final Long sanitizedEndBlock = sanitizedEndBlockIndex(startBlock, endBlock); - - final List blocks = new ArrayList<>(); - for (long currentBlockIndex = startBlock; - currentBlockIndex < sanitizedEndBlock; - currentBlockIndex += 1) { - Optional blockHashByNumber = blockchain.getBlockHashByNumber(currentBlockIndex); - blockHashByNumber.ifPresent(hash -> blocks.add(blockchain.getBlockByHash(hash))); - } - - final boolean allBlocksAreFound = blocks.size() == (sanitizedEndBlock - startBlock); - - return new ExportResult(blocks, allBlocksAreFound); - } - - private Long sanitizedEndBlockIndex(final Long startBlock, final Long endBlock) { - if (endBlock == null) { - return startBlock + 1; - } else { - return endBlock; - } - } - - public static final class ExportResult { - - public final List blocks; - - public final boolean allBlocksAreFound; - - ExportResult(final List blocks, final boolean allBlocksAreFound) { - this.blocks = blocks; - this.allBlocksAreFound = allBlocksAreFound; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("blocks", blocks) - .add("allBlocksAreFound", allBlocksAreFound) - .toString(); - } - } -} diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java new file mode 100644 index 0000000000..4ed40097cf --- /dev/null +++ b/pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java @@ -0,0 +1,245 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package tech.pegasys.pantheon.chainexport; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; +import tech.pegasys.pantheon.config.GenesisConfigFile; +import tech.pegasys.pantheon.controller.PantheonController; +import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair; +import tech.pegasys.pantheon.ethereum.chain.Blockchain; +import tech.pegasys.pantheon.ethereum.core.Block; +import tech.pegasys.pantheon.ethereum.core.BlockBody; +import tech.pegasys.pantheon.ethereum.core.BlockHeader; +import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider; +import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder; +import tech.pegasys.pantheon.ethereum.core.PrivacyParameters; +import tech.pegasys.pantheon.ethereum.eth.EthProtocolConfiguration; +import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; +import tech.pegasys.pantheon.ethereum.eth.transactions.TransactionPoolConfiguration; +import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; +import tech.pegasys.pantheon.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; +import tech.pegasys.pantheon.ethereum.util.RawBlockIterator; +import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; +import tech.pegasys.pantheon.testutil.BlockTestUtil; +import tech.pegasys.pantheon.testutil.TestClock; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** Tests for {@link BlockExporter}. */ +public final class RlpBlockExporterTest { + + @ClassRule public static final TemporaryFolder folder = new TemporaryFolder(); + private static Blockchain blockchain; + private static long chainHead; + private static ProtocolSchedule protocolSchedule; + + @BeforeClass + public static void setupBlockchain() throws IOException { + final PantheonController controller = createController(); + final Path blocks = folder.newFile("1000.blocks").toPath(); + BlockTestUtil.write1000Blocks(blocks); + blockchain = importBlocks(controller, blocks); + chainHead = blockchain.getChainHeadBlockNumber(); + protocolSchedule = controller.getProtocolSchedule(); + } + + private static Blockchain importBlocks( + final PantheonController controller, final Path blocksFile) throws IOException { + final RlpBlockImporter blockImporter = new RlpBlockImporter(); + + blockImporter.importBlockchain(blocksFile, controller); + return controller.getProtocolContext().getBlockchain(); + } + + private static PantheonController createController() throws IOException { + final Path dataDir = folder.newFolder().toPath(); + return new PantheonController.Builder() + .fromGenesisConfig(GenesisConfigFile.mainnet()) + .synchronizerConfiguration(SynchronizerConfiguration.builder().build()) + .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) + .storageProvider(new InMemoryStorageProvider()) + .networkId(1) + .miningParameters(new MiningParametersTestBuilder().enabled(false).build()) + .nodeKeys(KeyPair.generate()) + .metricsSystem(new NoOpMetricsSystem()) + .privacyParameters(PrivacyParameters.DEFAULT) + .dataDirectory(dataDir) + .clock(TestClock.fixed()) + .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) + .build(); + } + + @Test + public void exportBlocks_noBounds() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + exporter.exportBlocks(outputPath, Optional.empty(), Optional.empty()); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = 0; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(chainHead + 1L); + } + + @Test + public void exportBlocks_withLowerBound() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + final long lowerBound = 990; + exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.empty()); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = lowerBound; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(chainHead + 1L); + } + + @Test + public void exportBlocks_withUpperBound() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + final long upperBound = 10; + exporter.exportBlocks(outputPath, Optional.empty(), Optional.of(upperBound)); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = 0; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(upperBound); + } + + @Test + public void exportBlocks_withUpperAndLowerBounds() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + final long lowerBound = 5; + final long upperBound = 10; + exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.of(upperBound)); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = lowerBound; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(upperBound); + } + + @Test + public void exportBlocks_withRangeBeyondChainHead() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + final long lowerBound = chainHead - 10; + final long upperBound = chainHead + 10; + exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.of(upperBound)); + + // Iterate over blocks and check that they match expectations + RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath()); + long currentBlockNumber = lowerBound; + while (blockIterator.hasNext()) { + final Block actual = blockIterator.next(); + final Block expected = getBlock(blockchain, currentBlockNumber); + assertThat(actual).isEqualTo(expected); + currentBlockNumber++; + } + + // Check that we iterated to the end of the chain + assertThat(currentBlockNumber).isEqualTo(chainHead + 1L); + } + + @Test + public void exportBlocks_negativeStartNumber() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.of(-1L), Optional.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("greater than 0"); + } + + @Test + public void exportBlocks_negativeEndNumber() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.empty(), Optional.of(-1L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("greater than 0"); + } + + @Test + public void exportBlocks_outOfOrderBounds() throws IOException { + final File outputPath = folder.newFile(); + final RlpBlockExporter exporter = new RlpBlockExporter(blockchain); + + assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.of(10L), Optional.of(2L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Start block must be less than end block"); + } + + private RawBlockIterator getBlockIterator(final Path blocks) throws IOException { + return new RawBlockIterator( + blocks, + rlp -> + BlockHeader.readFrom(rlp, ScheduleBasedBlockHeaderFunctions.create(protocolSchedule))); + } + + private Block getBlock(final Blockchain blockchain, final long blockNumber) { + final BlockHeader header = blockchain.getBlockHeader(blockNumber).get(); + final BlockBody body = blockchain.getBlockBody(header.getHash()).get(); + return new Block(header, body); + } +} diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/ChainImporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/JsonBlockImporterTest.java similarity index 95% rename from pantheon/src/test/java/tech/pegasys/pantheon/chainimport/ChainImporterTest.java rename to pantheon/src/test/java/tech/pegasys/pantheon/chainimport/JsonBlockImporterTest.java index 595dd20302..0889499793 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/ChainImporterTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/JsonBlockImporterTest.java @@ -54,7 +54,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -public abstract class ChainImporterTest { +public abstract class JsonBlockImporterTest { @Rule public final TemporaryFolder folder = new TemporaryFolder(); @@ -62,14 +62,14 @@ public abstract class ChainImporterTest { protected final GenesisConfigFile genesisConfigFile; protected final boolean isEthash; - public ChainImporterTest(final String consensusEngine) throws IOException { + public JsonBlockImporterTest(final String consensusEngine) throws IOException { this.consensusEngine = consensusEngine; final String genesisData = getFileContents("genesis.json"); this.genesisConfigFile = GenesisConfigFile.fromConfig(genesisData); this.isEthash = genesisConfigFile.getConfigOptions().isEthHash(); } - public static class SingletonTests extends ChainImporterTest { + public static class SingletonTests extends JsonBlockImporterTest { public SingletonTests() throws IOException { super("unsupported"); } @@ -77,7 +77,7 @@ public abstract class ChainImporterTest { @Test public void importChain_unsupportedConsensusAlgorithm() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("clique", "blocks-import-valid.json"); @@ -90,7 +90,7 @@ public abstract class ChainImporterTest { } @RunWith(Parameterized.class) - public static class ParameterizedTests extends ChainImporterTest { + public static class ParameterizedTests extends JsonBlockImporterTest { public ParameterizedTests(final String consensusEngine) throws IOException { super(consensusEngine); @@ -105,7 +105,7 @@ public abstract class ChainImporterTest { @Test public void importChain_validJson_withBlockNumbers() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-valid.json"); importer.importChain(jsonData); @@ -196,7 +196,7 @@ public abstract class ChainImporterTest { @Test public void importChain_validJson_noBlockIdentifiers() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-valid-no-block-identifiers.json"); importer.importChain(jsonData); @@ -287,7 +287,7 @@ public abstract class ChainImporterTest { @Test public void importChain_validJson_withParentHashes() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); String jsonData = getFileContents("blocks-import-valid.json"); @@ -338,7 +338,7 @@ public abstract class ChainImporterTest { @Test public void importChain_invalidParent() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-invalid-bad-parent.json"); @@ -350,7 +350,7 @@ public abstract class ChainImporterTest { @Test public void importChain_invalidTransaction() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-invalid-bad-tx.json"); @@ -363,7 +363,7 @@ public abstract class ChainImporterTest { @Test public void importChain_specialFields() throws IOException { final PantheonController controller = createController(); - final ChainImporter importer = new ChainImporter<>(controller); + final JsonBlockImporter importer = new JsonBlockImporter<>(controller); final String jsonData = getFileContents("blocks-import-special-fields.json"); diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockImporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java similarity index 91% rename from pantheon/src/test/java/tech/pegasys/pantheon/util/BlockImporterTest.java rename to pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java index 56935f2891..d51282b431 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockImporterTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package tech.pegasys.pantheon.util; +package tech.pegasys.pantheon.chainimport; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; @@ -39,12 +39,12 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -/** Tests for {@link BlockImporter}. */ -public final class BlockImporterTest { +/** Tests for {@link RlpBlockImporter}. */ +public final class RlpBlockImporterTest { @Rule public final TemporaryFolder folder = new TemporaryFolder(); - BlockImporter blockImporter = new BlockImporter(); + private RlpBlockImporter rlpBlockImporter = new RlpBlockImporter(); @Test public void blockImport() throws IOException { @@ -66,8 +66,8 @@ public final class BlockImporterTest { .clock(TestClock.fixed()) .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) .build(); - final BlockImporter.ImportResult result = - blockImporter.importBlockchain(source, targetController); + final RlpBlockImporter.ImportResult result = + rlpBlockImporter.importBlockchain(source, targetController); // Don't count the Genesis block assertThat(result.count).isEqualTo(999); assertThat(result.td).isEqualTo(UInt256.of(21991996248790L)); @@ -105,7 +105,8 @@ public final class BlockImporterTest { .clock(TestClock.fixed()) .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) .build(); - final BlockImporter.ImportResult result = blockImporter.importBlockchain(source, controller); + final RlpBlockImporter.ImportResult result = + rlpBlockImporter.importBlockchain(source, controller); // Don't count the Genesis block assertThat(result.count).isEqualTo(958); diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java index 39b0edc6d7..f974cee15c 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java @@ -23,7 +23,9 @@ import static org.mockito.Mockito.when; import tech.pegasys.pantheon.Runner; import tech.pegasys.pantheon.RunnerBuilder; -import tech.pegasys.pantheon.chainimport.ChainImporter; +import tech.pegasys.pantheon.chainexport.RlpBlockExporter; +import tech.pegasys.pantheon.chainimport.JsonBlockImporter; +import tech.pegasys.pantheon.chainimport.RlpBlockImporter; import tech.pegasys.pantheon.cli.config.EthNetworkConfig; import tech.pegasys.pantheon.cli.options.EthProtocolOptions; import tech.pegasys.pantheon.cli.options.MetricsCLIOptions; @@ -32,7 +34,8 @@ import tech.pegasys.pantheon.cli.options.RocksDBOptions; import tech.pegasys.pantheon.cli.options.SynchronizerOptions; import tech.pegasys.pantheon.cli.options.TransactionPoolOptions; import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand.KeyLoader; -import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ChainImporterFactory; +import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.JsonBlockImporterFactory; +import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.RlpBlockExporterFactory; import tech.pegasys.pantheon.controller.PantheonController; import tech.pegasys.pantheon.controller.PantheonControllerBuilder; import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair; @@ -48,8 +51,6 @@ import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule; import tech.pegasys.pantheon.ethereum.permissioning.PermissioningConfiguration; import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration; import tech.pegasys.pantheon.services.PantheonPluginContextImpl; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import tech.pegasys.pantheon.util.bytes.BytesValue; import java.io.ByteArrayOutputStream; @@ -101,9 +102,9 @@ public abstract class CommandTestAbstract { @Mock protected ProtocolContext mockProtocolContext; @Mock protected BlockBroadcaster mockBlockBroadcaster; @Mock protected PantheonController mockController; - @Mock protected BlockImporter mockBlockImporter; - @Mock protected BlockExporter mockBlockExporter; - @Mock protected ChainImporter chainImporter; + @Mock protected RlpBlockExporter rlpBlockExporter; + @Mock protected JsonBlockImporter jsonBlockImporter; + @Mock protected RlpBlockImporter rlpBlockImporter; @Mock protected Logger mockLogger; @Mock protected PantheonPluginContextImpl mockPantheonPluginContext; @@ -215,8 +216,9 @@ public abstract class CommandTestAbstract { } @SuppressWarnings("unchecked") - private ChainImporter chainImporterFactory(final PantheonController controller) { - return (ChainImporter) chainImporter; + private JsonBlockImporter jsonBlockImporterFactory( + final PantheonController controller) { + return (JsonBlockImporter) jsonBlockImporter; } private TestPantheonCommand parseCommand( @@ -227,9 +229,9 @@ public abstract class CommandTestAbstract { final TestPantheonCommand pantheonCommand = new TestPantheonCommand( mockLogger, - mockBlockImporter, - mockBlockExporter, - this::chainImporterFactory, + rlpBlockImporter, + this::jsonBlockImporterFactory, + (blockchain) -> rlpBlockExporter, mockRunnerBuilder, mockControllerBuilderFactory, keyLoader, @@ -257,9 +259,9 @@ public abstract class CommandTestAbstract { TestPantheonCommand( final Logger mockLogger, - final BlockImporter mockBlockImporter, - final BlockExporter mockBlockExporter, - final ChainImporterFactory chainImporterFactory, + final RlpBlockImporter mockBlockImporter, + final JsonBlockImporterFactory jsonBlockImporterFactory, + final RlpBlockExporterFactory rlpBlockExporterFactory, final RunnerBuilder mockRunnerBuilder, final PantheonController.Builder controllerBuilderFactory, final KeyLoader keyLoader, @@ -268,8 +270,8 @@ public abstract class CommandTestAbstract { super( mockLogger, mockBlockImporter, - mockBlockExporter, - chainImporterFactory, + jsonBlockImporterFactory, + rlpBlockExporterFactory, mockRunnerBuilder, controllerBuilderFactory, pantheonPluginContext, diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java index 2c993bc109..6f745283e2 100644 --- a/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java +++ b/pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java @@ -14,36 +14,21 @@ package tech.pegasys.pantheon.cli.subcommands.blocks; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.contentOf; -import static org.assertj.core.util.Arrays.asList; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import tech.pegasys.pantheon.cli.CommandTestAbstract; -import tech.pegasys.pantheon.config.GenesisConfigFile; import tech.pegasys.pantheon.controller.PantheonController; -import tech.pegasys.pantheon.crypto.SECP256K1; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider; -import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder; -import tech.pegasys.pantheon.ethereum.core.PrivacyParameters; -import tech.pegasys.pantheon.ethereum.eth.EthProtocolConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.transactions.TransactionPoolConfiguration; -import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; -import tech.pegasys.pantheon.testutil.BlockTestUtil; -import tech.pegasys.pantheon.testutil.TestClock; -import tech.pegasys.pantheon.util.BlockExporter; -import tech.pegasys.pantheon.util.BlockImporter; import java.io.File; import java.io.IOException; import java.io.Writer; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; import org.junit.Rule; import org.junit.Test; @@ -87,9 +72,9 @@ public class BlocksSubCommandTest extends CommandTestAbstract { + System.lineSeparator(); private static final String EXPECTED_BLOCK_EXPORT_USAGE = - "Usage: pantheon blocks export [-hV] [--end-block=] --start-block=" + "Usage: pantheon blocks export [-hV] [--end-block=] [--start-block=]" + System.lineSeparator() - + " [--to=]" + + " --to=" + System.lineSeparator() + "This command export a specific block from storage" + System.lineSeparator() @@ -161,7 +146,7 @@ public class BlocksSubCommandTest extends CommandTestAbstract { parseCommand( BLOCK_SUBCOMMAND_NAME, BLOCK_IMPORT_SUBCOMMAND_NAME, "--from", fileToImport.getPath()); - verify(mockBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); + verify(rlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); assertThat(pathArgumentCaptor.getValue()).isEqualByComparingTo(fileToImport.toPath()); @@ -180,7 +165,7 @@ public class BlocksSubCommandTest extends CommandTestAbstract { "--from", fileToImport.getPath()); - verify(mockBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); + verify(rlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any()); assertThat(pathArgumentCaptor.getValue()).isEqualByComparingTo(fileToImport.toPath()); @@ -207,107 +192,221 @@ public class BlocksSubCommandTest extends CommandTestAbstract { assertThat(commandOutput.toString()).isEmpty(); assertThat(commandErrorOutput.toString()).isEmpty(); - verify(chainImporter, times(1)).importChain(stringArgumentCaptor.capture()); + verify(jsonBlockImporter, times(1)).importChain(stringArgumentCaptor.capture()); assertThat(stringArgumentCaptor.getValue()).isEqualTo(fileContent); } // Export sub-sub-command @Test - public void callingBlockExportSubCommandWithoutStartBlockMustDisplayErrorAndUsage() { - parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME); - final String expectedErrorOutputStart = "Missing required option '--start-block='"; + public void blocksExport_missingFileParam() throws IOException { + createDbDirectory(true); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME); + final String expectedErrorOutputStart = "Missing required option '--to='"; assertThat(commandOutput.toString()).isEmpty(); assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); } @Test - public void callingBlockExportSubCommandHelpMustDisplayUsage() { - parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--help"); - assertThat(commandOutput.toString()).startsWith(EXPECTED_BLOCK_EXPORT_USAGE); - assertThat(commandErrorOutput.toString()).isEmpty(); + public void blocksExport_noDbDirectory() throws IOException { + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath()); + final String expectedErrorOutputStart = + "Chain is empty. Unable to export blocks from specified data directory: " + + folder.getRoot().getAbsolutePath() + + "/" + + PantheonController.DATABASE_PATH; + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); } @Test - public void callingBlockExportSubCommandWithNegativeStartBlockMustDisplayErrorAndUsage() { - final String expectedErrorOutputStart = "--start-block must be greater than or equal to zero"; - parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=-1"); + public void blocksExport_emptyDbDirectory() throws IOException { + createDbDirectory(false); + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath()); + final String expectedErrorOutputStart = + "Chain is empty. Unable to export blocks from specified data directory: " + + folder.getRoot().getAbsolutePath() + + "/" + + PantheonController.DATABASE_PATH; assertThat(commandOutput.toString()).isEmpty(); assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); } @Test - public void - callingBlockExportSubCommandWithEndBlockGreaterThanStartBlockMustDisplayErrorMessage() { - final String expectedErrorOutputStart = "--end-block must be greater than --start-block"; + public void blocksExport_noStartOrEnd() throws IOException { + createDbDirectory(true); + final File outputFile = folder.newFile("blocks.bin"); parseCommand( - BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=2", "--end-block=1"); + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath()); assertThat(commandOutput.toString()).isEmpty(); - assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + assertThat(commandErrorOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.empty(), Optional.empty()); } @Test - public void callingBlockExportSubCommandWithEndBlockEqualToStartBlockMustDisplayErrorMessage() { - final String expectedErrorOutputStart = "--end-block must be greater than --start-block"; + public void blocksExport_withStartAndNoEnd() throws IOException { + createDbDirectory(true); + final File outputFile = folder.newFile("blocks.bin"); parseCommand( - BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=2", "--end-block=2"); + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--start-block=1"); assertThat(commandOutput.toString()).isEmpty(); - assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart); + assertThat(commandErrorOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.of(1L), Optional.empty()); + } + + @Test + public void blocksExport_withEndAndNoStart() throws IOException { + createDbDirectory(true); + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--end-block=10"); + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.empty(), Optional.of(10L)); } @Test - public void callingExportSubCommandWithFilePathMustWriteBlockInThisFile() throws Exception { + public void blocksExport_withStartAndEnd() throws IOException { + createDbDirectory(true); + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--start-block=1", + "--end-block=10"); + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).isEmpty(); - final long startBlock = 0L; - final long endBlock = 2L; - final PantheonController pantheonController = initPantheonController(); - final MutableBlockchain blockchain = pantheonController.getProtocolContext().getBlockchain(); - final File outputFile = File.createTempFile("export", "store"); + verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.of(1L), Optional.of(10L)); + } - mockBlockExporter = new BlockExporter(); + @Test + public void blocksExport_withOutOfOrderStartAndEnd() throws IOException { + createDbDirectory(true); + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--start-block=10", + "--end-block=1"); + assertThat(commandErrorOutput.toString()) + .contains("Parameter --end-block (1) must be greater start block (10)"); + assertThat(commandOutput.toString()).isEmpty(); - doReturn(pantheonController).when(mockControllerBuilder).build(); + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } + @Test + public void blocksExport_withEmptyRange() throws IOException { + createDbDirectory(true); + final File outputFile = folder.newFile("blocks.bin"); parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, - "--start-block=" + startBlock, - "--end-block=" + endBlock, - "--to=" + outputFile.getPath()); + "--to", + outputFile.getPath(), + "--start-block=10", + "--end-block=10"); + assertThat(commandErrorOutput.toString()) + .contains("Parameter --end-block (10) must be greater start block (10)"); + assertThat(commandOutput.toString()).isEmpty(); - final Block blockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } - final Block secondBlockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(endBlock - 1).get()); + @Test + public void blocksExport_withInvalidStart() throws IOException { + createDbDirectory(true); + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--start-block=-1"); + assertThat(commandErrorOutput.toString()) + .contains("Parameter --start-block (-1) must be greater than or equal to zero"); + assertThat(commandOutput.toString()).isEmpty(); - assertThat(contentOf(outputFile)) - .isEqualTo(asList(new Block[] {blockFromBlockchain, secondBlockFromBlockchain}).toString()); + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } + @Test + public void blocksExport_withInvalidEnd() throws IOException { + createDbDirectory(true); + final File outputFile = folder.newFile("blocks.bin"); + parseCommand( + "--data-path=" + folder.getRoot().getAbsolutePath(), + BLOCK_SUBCOMMAND_NAME, + BLOCK_EXPORT_SUBCOMMAND_NAME, + "--to", + outputFile.getPath(), + "--end-block=-1"); + assertThat(commandErrorOutput.toString()) + .contains("Parameter --end-block (-1) must be greater than or equal to zero"); assertThat(commandOutput.toString()).isEmpty(); + + verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any()); + } + + @Test + public void callingBlockExportSubCommandHelpMustDisplayUsage() { + parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--help"); + assertThat(commandOutput.toString()).startsWith(EXPECTED_BLOCK_EXPORT_USAGE); assertThat(commandErrorOutput.toString()).isEmpty(); } - private PantheonController initPantheonController() throws IOException { - final BlockImporter blockImporter = new BlockImporter(); - final Path dataDir = folder.newFolder().toPath(); - final Path source = dataDir.resolve("1000.blocks"); - BlockTestUtil.write1000Blocks(source); - final PantheonController targetController = - new PantheonController.Builder() - .fromGenesisConfig(GenesisConfigFile.mainnet()) - .synchronizerConfiguration(SynchronizerConfiguration.builder().build()) - .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) - .storageProvider(new InMemoryStorageProvider()) - .networkId(1) - .miningParameters(new MiningParametersTestBuilder().enabled(false).build()) - .nodeKeys(SECP256K1.KeyPair.generate()) - .metricsSystem(new NoOpMetricsSystem()) - .privacyParameters(PrivacyParameters.DEFAULT) - .dataDirectory(dataDir) - .clock(TestClock.fixed()) - .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) - .build(); - blockImporter.importBlockchain(source, targetController); - return targetController; + private void createDbDirectory(final boolean createDataFiles) throws IOException { + File dbDir = folder.newFolder(PantheonController.DATABASE_PATH); + if (createDataFiles) { + Path dataFilePath = Paths.get(dbDir.getAbsolutePath(), "0000001.sst"); + final boolean success = new File(dataFilePath.toString()).createNewFile(); + assertThat(success).isTrue(); + } } } diff --git a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockExporterTest.java b/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockExporterTest.java deleted file mode 100644 index 9f9afd870e..0000000000 --- a/pantheon/src/test/java/tech/pegasys/pantheon/util/BlockExporterTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package tech.pegasys.pantheon.util; - -import static org.assertj.core.api.Assertions.assertThat; - -import tech.pegasys.pantheon.config.GenesisConfigFile; -import tech.pegasys.pantheon.controller.PantheonController; -import tech.pegasys.pantheon.crypto.SECP256K1; -import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain; -import tech.pegasys.pantheon.ethereum.core.Block; -import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider; -import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder; -import tech.pegasys.pantheon.ethereum.core.PrivacyParameters; -import tech.pegasys.pantheon.ethereum.eth.EthProtocolConfiguration; -import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration; -import tech.pegasys.pantheon.ethereum.eth.transactions.TransactionPoolConfiguration; -import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem; -import tech.pegasys.pantheon.testutil.BlockTestUtil; -import tech.pegasys.pantheon.testutil.TestClock; - -import java.io.IOException; -import java.nio.file.Path; - -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -/** Tests for {@link BlockExporter}. */ -public final class BlockExporterTest { - - @ClassRule public static final TemporaryFolder folder = new TemporaryFolder(); - - private static PantheonController targetController; - - private final BlockExporter blockExporter = new BlockExporter(); - - @BeforeClass - public static void initPantheonController() throws IOException { - final BlockImporter blockImporter = new BlockImporter(); - final Path dataDir = folder.newFolder().toPath(); - final Path source = dataDir.resolve("1000.blocks"); - BlockTestUtil.write1000Blocks(source); - targetController = - new PantheonController.Builder() - .fromGenesisConfig(GenesisConfigFile.mainnet()) - .synchronizerConfiguration(SynchronizerConfiguration.builder().build()) - .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) - .storageProvider(new InMemoryStorageProvider()) - .networkId(1) - .miningParameters(new MiningParametersTestBuilder().enabled(false).build()) - .nodeKeys(SECP256K1.KeyPair.generate()) - .metricsSystem(new NoOpMetricsSystem()) - .privacyParameters(PrivacyParameters.DEFAULT) - .dataDirectory(dataDir) - .clock(TestClock.fixed()) - .transactionPoolConfiguration(TransactionPoolConfiguration.builder().build()) - .build(); - blockImporter.importBlockchain(source, targetController); - } - - @Test - public void callingBlockExporterWithOnlyStartBlockShouldReturnOneBlock() throws Exception { - - final long startBlock = 0L; - - final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain(); - - final Block blockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); - - BlockExporter.ExportResult exportResult = - blockExporter.exportBlockchain(targetController, startBlock, null); - - assertThat(exportResult.blocks).contains(blockFromBlockchain); - } - - @Test - public void callingBlockExporterWithStartBlockAndBlockShouldReturnSeveralBlocks() - throws Exception { - - final long startBlock = 0L; - final long endBlock = 1L; - - final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain(); - - final Block blockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get()); - - final Block secondBlockFromBlockchain = - blockchain.getBlockByHash(blockchain.getBlockHashByNumber(endBlock - 1).get()); - - BlockExporter.ExportResult exportResult = - blockExporter.exportBlockchain(targetController, startBlock, endBlock); - - assertThat(exportResult.blocks) - .contains(blockFromBlockchain) - .contains(secondBlockFromBlockchain); - } -}