[PIE-1810] Update export subcommand to export blocks in rlp format (#1852)

Signed-off-by: Adrian Sutton <adrian.sutton@consensys.net>
pull/2/head
mbaxter 5 years ago committed by GitHub
parent a5daeba71d
commit f9e50ae8c6
  1. 12
      pantheon/src/main/java/tech/pegasys/pantheon/Pantheon.java
  2. 94
      pantheon/src/main/java/tech/pegasys/pantheon/chainexport/BlockExporter.java
  3. 35
      pantheon/src/main/java/tech/pegasys/pantheon/chainexport/RlpBlockExporter.java
  4. 11
      pantheon/src/main/java/tech/pegasys/pantheon/chainimport/JsonBlockImporter.java
  5. 11
      pantheon/src/main/java/tech/pegasys/pantheon/chainimport/RlpBlockImporter.java
  6. 4
      pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/BlockData.java
  7. 2
      pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/ChainData.java
  8. 2
      pantheon/src/main/java/tech/pegasys/pantheon/chainimport/internal/TransactionData.java
  9. 31
      pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java
  10. 17
      pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockExportFormat.java
  11. 2
      pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlockImportFormat.java
  12. 181
      pantheon/src/main/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommand.java
  13. 88
      pantheon/src/main/java/tech/pegasys/pantheon/util/BlockExporter.java
  14. 245
      pantheon/src/test/java/tech/pegasys/pantheon/chainexport/RlpBlockExporterTest.java
  15. 22
      pantheon/src/test/java/tech/pegasys/pantheon/chainimport/JsonBlockImporterTest.java
  16. 15
      pantheon/src/test/java/tech/pegasys/pantheon/chainimport/RlpBlockImporterTest.java
  17. 36
      pantheon/src/test/java/tech/pegasys/pantheon/cli/CommandTestAbstract.java
  18. 261
      pantheon/src/test/java/tech/pegasys/pantheon/cli/subcommands/blocks/BlocksSubCommandTest.java
  19. 111
      pantheon/src/test/java/tech/pegasys/pantheon/util/BlockExporterTest.java

@ -14,12 +14,12 @@ package tech.pegasys.pantheon;
import static org.apache.logging.log4j.LogManager.getLogger;
import tech.pegasys.pantheon.chainimport.ChainImporter;
import tech.pegasys.pantheon.chainexport.RlpBlockExporter;
import tech.pegasys.pantheon.chainimport.JsonBlockImporter;
import tech.pegasys.pantheon.chainimport.RlpBlockImporter;
import tech.pegasys.pantheon.cli.PantheonCommand;
import tech.pegasys.pantheon.controller.PantheonController;
import tech.pegasys.pantheon.services.PantheonPluginContextImpl;
import tech.pegasys.pantheon.util.BlockExporter;
import tech.pegasys.pantheon.util.BlockImporter;
import org.apache.logging.log4j.Logger;
import picocli.CommandLine.RunLast;
@ -36,9 +36,9 @@ public final class Pantheon {
final PantheonCommand pantheonCommand =
new PantheonCommand(
logger,
new BlockImporter(),
new BlockExporter(),
ChainImporter::new,
new RlpBlockImporter(),
JsonBlockImporter::new,
RlpBlockExporter::new,
new RunnerBuilder(),
new PantheonController.Builder(),
new PantheonPluginContextImpl(),

@ -0,0 +1,94 @@
/*
* Copyright 2018 ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.chainexport;
import static com.google.common.base.Preconditions.checkArgument;
import tech.pegasys.pantheon.ethereum.chain.Blockchain;
import tech.pegasys.pantheon.ethereum.core.Block;
import tech.pegasys.pantheon.ethereum.core.BlockHeader;
import tech.pegasys.pantheon.ethereum.core.Hash;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Optional;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/** Pantheon Block Export Util. */
public abstract class BlockExporter {
private static final Logger LOG = LogManager.getLogger();
private final Blockchain blockchain;
protected BlockExporter(final Blockchain blockchain) {
this.blockchain = blockchain;
}
/**
* Export blocks that are stored in Pantheon's block storage.
*
* @param outputFile the path at which to save the exported block data
* @param maybeStartBlock the starting index of the block list to export (inclusive)
* @param maybeEndBlock the ending index of the block list to export (exclusive), if not specified
* a single block will be export
* @throws IOException if an I/O error occurs while writing data to disk
*/
public void exportBlocks(
final File outputFile,
final Optional<Long> maybeStartBlock,
final Optional<Long> maybeEndBlock)
throws IOException {
// Get range to export
final long startBlock = maybeStartBlock.orElse(BlockHeader.GENESIS_BLOCK_NUMBER);
final long endBlock = maybeEndBlock.orElse(blockchain.getChainHeadBlockNumber() + 1L);
checkArgument(startBlock >= 0 && endBlock >= 0, "Start and end blocks must be greater than 0.");
checkArgument(startBlock < endBlock, "Start block must be less than end block");
// Append to file if a range is specified
final boolean append = maybeStartBlock.isPresent();
FileOutputStream outputStream = new FileOutputStream(outputFile, append);
LOG.info(
"Exporting blocks [{},{}) to file {} (appending: {})",
startBlock,
endBlock,
outputFile.toString(),
Boolean.toString(append));
long blockNumber = 0L;
for (long i = startBlock; i < endBlock; i++) {
Optional<Hash> blockHash = blockchain.getBlockHashByNumber(i);
if (blockHash.isEmpty()) {
LOG.warn("Unable to export blocks [{} - {}). Blocks not found.", i, endBlock);
break;
}
final Block block = blockchain.getBlockByHash(blockHash.get());
blockNumber = block.getHeader().getNumber();
if (blockNumber % 100 == 0) {
LOG.info("Export at block {}", blockNumber);
}
exportBlock(outputStream, block);
}
outputStream.close();
LOG.info("Export complete at block {}", blockNumber);
}
protected abstract void exportBlock(final FileOutputStream outputStream, final Block block)
throws IOException;
}

@ -0,0 +1,35 @@
/*
* Copyright 2019 ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.chainexport;
import tech.pegasys.pantheon.ethereum.chain.Blockchain;
import tech.pegasys.pantheon.ethereum.core.Block;
import tech.pegasys.pantheon.ethereum.rlp.RLP;
import tech.pegasys.pantheon.util.bytes.BytesValue;
import java.io.FileOutputStream;
import java.io.IOException;
public class RlpBlockExporter extends BlockExporter {
public RlpBlockExporter(final Blockchain blockchain) {
super(blockchain);
}
@Override
protected void exportBlock(final FileOutputStream outputStream, final Block block)
throws IOException {
final BytesValue rlp = RLP.encode(block::writeTo);
outputStream.write(rlp.getArrayUnsafe());
}
}

@ -12,6 +12,8 @@
*/
package tech.pegasys.pantheon.chainimport;
import tech.pegasys.pantheon.chainimport.internal.BlockData;
import tech.pegasys.pantheon.chainimport.internal.ChainData;
import tech.pegasys.pantheon.config.GenesisConfigOptions;
import tech.pegasys.pantheon.controller.PantheonController;
import tech.pegasys.pantheon.ethereum.blockcreation.MiningCoordinator;
@ -39,13 +41,18 @@ import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class ChainImporter<C> {
/**
* Tool for importing blocks with transactions from human-readable json.
*
* @param <C> The consensus algorithm context
*/
public class JsonBlockImporter<C> {
private static final Logger LOG = LogManager.getLogger();
private final ObjectMapper mapper;
private final PantheonController<C> controller;
public ChainImporter(final PantheonController<C> controller) {
public JsonBlockImporter(final PantheonController<C> controller) {
this.controller = controller;
mapper = new ObjectMapper();
// Jdk8Module allows us to easily parse {@code Optional} values from json

@ -10,7 +10,7 @@
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.util;
package tech.pegasys.pantheon.chainimport;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.logging.log4j.LogManager.getLogger;
@ -41,8 +41,8 @@ import java.util.concurrent.Semaphore;
import com.google.common.base.MoreObjects;
import org.apache.logging.log4j.Logger;
/** Pantheon Block Import Util. */
public class BlockImporter {
/** Tool for importing rlp-encoded block data from files. */
public class RlpBlockImporter {
private static final Logger LOG = getLogger();
private final Semaphore blockBacklog = new Semaphore(2);
@ -60,7 +60,7 @@ public class BlockImporter {
* @return the import result
* @throws IOException On Failure
*/
public <C> BlockImporter.ImportResult importBlockchain(
public <C> RlpBlockImporter.ImportResult importBlockchain(
final Path blocks, final PantheonController<C> pantheonController) throws IOException {
final ProtocolSchedule<C> protocolSchedule = pantheonController.getProtocolSchedule();
final ProtocolContext<C> context = pantheonController.getProtocolContext();
@ -125,7 +125,8 @@ public class BlockImporter {
if (previousBlockFuture != null) {
previousBlockFuture.join();
}
return new BlockImporter.ImportResult(blockchain.getChainHead().getTotalDifficulty(), count);
return new RlpBlockImporter.ImportResult(
blockchain.getChainHead().getTotalDifficulty(), count);
} finally {
validationExecutor.shutdownNow();
try {

@ -10,9 +10,9 @@
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.chainimport;
package tech.pegasys.pantheon.chainimport.internal;
import tech.pegasys.pantheon.chainimport.TransactionData.NonceProvider;
import tech.pegasys.pantheon.chainimport.internal.TransactionData.NonceProvider;
import tech.pegasys.pantheon.ethereum.core.Account;
import tech.pegasys.pantheon.ethereum.core.Address;
import tech.pegasys.pantheon.ethereum.core.Hash;

@ -10,7 +10,7 @@
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.chainimport;
package tech.pegasys.pantheon.chainimport.internal;
import java.util.List;

@ -10,7 +10,7 @@
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.chainimport;
package tech.pegasys.pantheon.chainimport.internal;
import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair;
import tech.pegasys.pantheon.crypto.SECP256K1.PrivateKey;

@ -31,6 +31,7 @@ import static tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration.DEFA
import tech.pegasys.pantheon.PantheonInfo;
import tech.pegasys.pantheon.Runner;
import tech.pegasys.pantheon.RunnerBuilder;
import tech.pegasys.pantheon.chainimport.RlpBlockImporter;
import tech.pegasys.pantheon.cli.config.EthNetworkConfig;
import tech.pegasys.pantheon.cli.config.NetworkName;
import tech.pegasys.pantheon.cli.converter.MetricCategoryConverter;
@ -51,7 +52,8 @@ import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand;
import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand.KeyLoader;
import tech.pegasys.pantheon.cli.subcommands.RetestethSubCommand;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ChainImporterFactory;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.JsonBlockImporterFactory;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.RlpBlockExporterFactory;
import tech.pegasys.pantheon.cli.subcommands.operator.OperatorSubCommand;
import tech.pegasys.pantheon.cli.subcommands.rlp.RLPSubCommand;
import tech.pegasys.pantheon.cli.util.ConfigOptionSearchAndRunHandler;
@ -93,8 +95,6 @@ import tech.pegasys.pantheon.services.PantheonEventsImpl;
import tech.pegasys.pantheon.services.PantheonPluginContextImpl;
import tech.pegasys.pantheon.services.PicoCLIOptionsImpl;
import tech.pegasys.pantheon.services.kvstore.RocksDbConfiguration;
import tech.pegasys.pantheon.util.BlockExporter;
import tech.pegasys.pantheon.util.BlockImporter;
import tech.pegasys.pantheon.util.PermissioningConfigurationValidator;
import tech.pegasys.pantheon.util.bytes.BytesValue;
import tech.pegasys.pantheon.util.number.Fraction;
@ -153,12 +153,12 @@ import picocli.CommandLine.ParameterException;
public class PantheonCommand implements DefaultCommandValues, Runnable {
private final Logger logger;
private final ChainImporterFactory chainImporterFactory;
private CommandLine commandLine;
private final BlockImporter blockImporter;
private final BlockExporter blockExporter;
private final RlpBlockImporter rlpBlockImporter;
private final JsonBlockImporterFactory jsonBlockImporterFactory;
private final RlpBlockExporterFactory rlpBlockExporterFactory;
final NetworkingOptions networkingOptions = NetworkingOptions.create();
final SynchronizerOptions synchronizerOptions = SynchronizerOptions.create();
@ -633,17 +633,17 @@ public class PantheonCommand implements DefaultCommandValues, Runnable {
public PantheonCommand(
final Logger logger,
final BlockImporter blockImporter,
final BlockExporter blockExporter,
final ChainImporterFactory chainImporterFactory,
final RlpBlockImporter rlpBlockImporter,
final JsonBlockImporterFactory jsonBlockImporterFactory,
final RlpBlockExporterFactory rlpBlockExporterFactory,
final RunnerBuilder runnerBuilder,
final PantheonController.Builder controllerBuilderFactory,
final PantheonPluginContextImpl pantheonPluginContext,
final Map<String, String> environment) {
this.logger = logger;
this.blockImporter = blockImporter;
this.blockExporter = blockExporter;
this.chainImporterFactory = chainImporterFactory;
this.rlpBlockImporter = rlpBlockImporter;
this.rlpBlockExporterFactory = rlpBlockExporterFactory;
this.jsonBlockImporterFactory = jsonBlockImporterFactory;
this.runnerBuilder = runnerBuilder;
this.controllerBuilderFactory = controllerBuilderFactory;
this.pantheonPluginContext = pantheonPluginContext;
@ -690,7 +690,10 @@ public class PantheonCommand implements DefaultCommandValues, Runnable {
commandLine.addSubcommand(
BlocksSubCommand.COMMAND_NAME,
new BlocksSubCommand(
blockImporter, blockExporter, chainImporterFactory, resultHandler.out()));
rlpBlockImporter,
jsonBlockImporterFactory,
rlpBlockExporterFactory,
resultHandler.out()));
commandLine.addSubcommand(
PublicKeySubCommand.COMMAND_NAME,
new PublicKeySubCommand(resultHandler.out(), getKeyLoader()));
@ -1363,7 +1366,7 @@ public class PantheonCommand implements DefaultCommandValues, Runnable {
}
}
private Path dataDir() {
public Path dataDir() {
if (isFullInstantiation()) {
return standaloneCommands.dataPath.toAbsolutePath();
} else if (isDocker) {

@ -0,0 +1,17 @@
/*
* Copyright 2019 ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.cli.subcommands.blocks;
public enum BlockExportFormat {
RLP
}

@ -12,7 +12,7 @@
*/
package tech.pegasys.pantheon.cli.subcommands.blocks;
public enum BlockFormat {
public enum BlockImportFormat {
RLP,
JSON
}

@ -13,34 +13,33 @@
package tech.pegasys.pantheon.cli.subcommands.blocks;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.nio.charset.StandardCharsets.UTF_8;
import static tech.pegasys.pantheon.cli.DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP;
import static tech.pegasys.pantheon.cli.DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP;
import static tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.COMMAND_NAME;
import tech.pegasys.pantheon.chainimport.ChainImporter;
import tech.pegasys.pantheon.chainexport.RlpBlockExporter;
import tech.pegasys.pantheon.chainimport.JsonBlockImporter;
import tech.pegasys.pantheon.chainimport.RlpBlockImporter;
import tech.pegasys.pantheon.cli.PantheonCommand;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ExportSubCommand;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ImportSubCommand;
import tech.pegasys.pantheon.controller.PantheonController;
import tech.pegasys.pantheon.ethereum.ProtocolContext;
import tech.pegasys.pantheon.ethereum.chain.Blockchain;
import tech.pegasys.pantheon.ethereum.core.Address;
import tech.pegasys.pantheon.ethereum.core.Block;
import tech.pegasys.pantheon.ethereum.core.MiningParameters;
import tech.pegasys.pantheon.ethereum.core.Wei;
import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration;
import tech.pegasys.pantheon.metrics.prometheus.MetricsService;
import tech.pegasys.pantheon.util.BlockExporter;
import tech.pegasys.pantheon.util.BlockImporter;
import tech.pegasys.pantheon.util.bytes.BytesValue;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.nio.file.Paths;
import java.util.Optional;
import io.vertx.core.Vertx;
@ -75,20 +74,21 @@ public class BlocksSubCommand implements Runnable {
@Spec
private CommandSpec spec; // Picocli injects reference to command spec
private final BlockImporter blockImporter;
private final BlockExporter blockExporter;
private final ChainImporterFactory chainImporterFactory;
private final RlpBlockImporter rlpBlockImporter;
private final JsonBlockImporterFactory jsonBlockImporterFactory;
private final RlpBlockExporterFactory rlpBlockExporterFactory;
private final PrintStream out;
public BlocksSubCommand(
final BlockImporter blockImporter,
final BlockExporter blockExporter,
final ChainImporterFactory chainImporterFactory,
final RlpBlockImporter rlpBlockImporter,
final JsonBlockImporterFactory jsonBlockImporterFactory,
final RlpBlockExporterFactory rlpBlockExporterFactory,
final PrintStream out) {
this.blockImporter = blockImporter;
this.blockExporter = blockExporter;
this.chainImporterFactory = chainImporterFactory;
this.rlpBlockImporter = rlpBlockImporter;
this.rlpBlockExporterFactory = rlpBlockExporterFactory;
this.jsonBlockImporterFactory = jsonBlockImporterFactory;
this.out = out;
}
@ -125,7 +125,7 @@ public class BlocksSubCommand implements Runnable {
description =
"The type of data to be imported, possible values are: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).",
arity = "1..1")
private final BlockFormat format = BlockFormat.RLP;
private final BlockImportFormat format = BlockImportFormat.RLP;
@SuppressWarnings("unused")
@Spec
@ -136,7 +136,8 @@ public class BlocksSubCommand implements Runnable {
LOG.info("Import {} block data from {}", format, blocksImportFile);
checkCommand(parentCommand);
checkNotNull(parentCommand.blockImporter);
checkNotNull(parentCommand.rlpBlockImporter);
checkNotNull(parentCommand.jsonBlockImporterFactory);
Optional<MetricsService> metricsService = initMetrics(parentCommand);
@ -199,14 +200,14 @@ public class BlocksSubCommand implements Runnable {
private <T> void importJsonBlocks(final PantheonController<T> controller, final Path path)
throws IOException {
ChainImporter<T> importer = parentCommand.chainImporterFactory.get(controller);
JsonBlockImporter<T> importer = parentCommand.jsonBlockImporterFactory.get(controller);
final String jsonData = Files.readString(path);
importer.importChain(jsonData);
}
private <T> void importRlpBlocks(final PantheonController<T> controller, final Path path)
throws IOException {
parentCommand.blockImporter.importBlockchain(path, controller);
parentCommand.rlpBlockImporter.importBlockchain(path, controller);
}
}
@ -226,7 +227,6 @@ public class BlocksSubCommand implements Runnable {
@Option(
names = "--start-block",
required = true,
paramLabel = MANDATORY_LONG_FORMAT_HELP,
description = "the starting index of the block list to export (inclusive)",
arity = "1..1")
@ -241,71 +241,125 @@ public class BlocksSubCommand implements Runnable {
arity = "1..1")
private final Long endBlock = null;
@Option(
names = "--format",
hidden = true,
description =
"The format to export, possible values are: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).",
arity = "1..1")
private final BlockExportFormat format = BlockExportFormat.RLP;
@Option(
names = "--to",
required = true,
paramLabel = MANDATORY_FILE_FORMAT_HELP,
description = "File to write the block list instead of standard output",
arity = "1..1")
private File blocksExportFile = null;
@SuppressWarnings("unused")
@Spec
private CommandSpec spec;
@Override
public void run() {
LOG.info("Runs export sub command");
LOG.info("Export {} block data to file {}", format, blocksExportFile.toPath());
checkCommand(this, startBlock, endBlock);
final Optional<MetricsService> metricsService = initMetrics(parentCommand);
Optional<MetricsService> metricsService = initMetrics(parentCommand);
final PantheonController<?> controller = createPantheonController();
try {
final BlockExporter.ExportResult exportResult =
parentCommand.blockExporter.exportBlockchain(
parentCommand.parentCommand.buildController(), startBlock, endBlock);
outputBlock(exportResult.blocks);
if (exportResult.blocks.isEmpty()) {
throw new ExecutionException(new CommandLine(this), "No block found at the given index");
} else if (!exportResult.allBlocksAreFound) {
throw new ExecutionException(
new CommandLine(this),
"Partial export due to inability to recover all requested blocks");
switch (format) {
case RLP:
exportRlpFormat(controller);
break;
default:
throw new ParameterException(
spec.commandLine(), "Unsupported format: " + format.toString());
}
} catch (IOException e) {
throw new ExecutionException(
spec.commandLine(), "An error occurred while exporting blocks.", e);
} finally {
metricsService.ifPresent(MetricsService::stop);
}
}
private static void checkCommand(
private PantheonController<?> createPantheonController() {
return parentCommand.parentCommand.buildController();
}
private void exportRlpFormat(final PantheonController<?> controller) throws IOException {
final ProtocolContext<?> context = controller.getProtocolContext();
RlpBlockExporter exporter =
parentCommand.rlpBlockExporterFactory.get(context.getBlockchain());
exporter.exportBlocks(blocksExportFile, getStartBlock(), getEndBlock());
}
private void checkCommand(
final ExportSubCommand exportSubCommand, final Long startBlock, final Long endBlock) {
checkNotNull(exportSubCommand.parentCommand);
checkNotNull(exportSubCommand.parentCommand.blockExporter);
checkNotNull(startBlock);
if (startBlock < 0) {
throw new CommandLine.ParameterException(
new CommandLine(exportSubCommand),
"--start-block must be greater than or equal to zero");
final Optional<Long> maybeStartBlock = getStartBlock();
final Optional<Long> maybeEndBlock = getEndBlock();
maybeStartBlock
.filter(blockNum -> blockNum < 0)
.ifPresent(
(blockNum) -> {
throw new CommandLine.ParameterException(
spec.commandLine(),
"Parameter --start-block ("
+ blockNum
+ ") must be greater than or equal to zero.");
});
maybeEndBlock
.filter(blockNum -> blockNum < 0)
.ifPresent(
(blockNum) -> {
throw new CommandLine.ParameterException(
spec.commandLine(),
"Parameter --end-block ("
+ blockNum
+ ") must be greater than or equal to zero.");
});
if (maybeStartBlock.isPresent() && maybeEndBlock.isPresent()) {
if (endBlock <= startBlock) {
throw new CommandLine.ParameterException(
spec.commandLine(),
"Parameter --end-block ("
+ endBlock
+ ") must be greater start block ("
+ startBlock
+ ").");
}
}
if (endBlock != null && startBlock >= endBlock) {
// Error if data directory is empty
Path databasePath =
Paths.get(
parentCommand.parentCommand.dataDir().toAbsolutePath().toString(),
PantheonController.DATABASE_PATH);
File databaseDirectory = new File(databasePath.toString());
if (!databaseDirectory.isDirectory() || databaseDirectory.list().length == 0) {
// Empty data directory, nothing to export
throw new CommandLine.ParameterException(
new CommandLine(exportSubCommand), "--end-block must be greater than --start-block");
spec.commandLine(),
"Chain is empty. Unable to export blocks from specified data directory: "
+ databaseDirectory.toString());
}
}
private void outputBlock(final List<Block> blocks) {
if (blocksExportFile != null) {
final Path path = blocksExportFile.toPath();
try (final BufferedWriter fileWriter = Files.newBufferedWriter(path, UTF_8)) {
fileWriter.write(blocks.toString());
} catch (final IOException e) {
throw new ExecutionException(
new CommandLine(this), "An error occurred while trying to write the exported blocks");
}
} else {
parentCommand.out.println(blocks.toString());
}
private Optional<Long> getStartBlock() {
return Optional.ofNullable(startBlock);
}
private Optional<Long> getEndBlock() {
return Optional.ofNullable(endBlock);
}
}
@ -326,7 +380,12 @@ public class BlocksSubCommand implements Runnable {
}
@FunctionalInterface
public interface ChainImporterFactory {
<T> ChainImporter<T> get(PantheonController<T> controller);
public interface JsonBlockImporterFactory {
<T> JsonBlockImporter<T> get(PantheonController<T> controller);
}
@FunctionalInterface
public interface RlpBlockExporterFactory {
RlpBlockExporter get(Blockchain blockchain);
}
}

@ -1,88 +0,0 @@
/*
* Copyright 2018 ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.util;
import tech.pegasys.pantheon.controller.PantheonController;
import tech.pegasys.pantheon.ethereum.ProtocolContext;
import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain;
import tech.pegasys.pantheon.ethereum.core.Block;
import tech.pegasys.pantheon.ethereum.core.Hash;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import com.google.common.base.MoreObjects;
/** Pantheon Block Export Util. */
public class BlockExporter {
/**
* Export blocks that are stored in Pantheon's block storage.
*
* @param pantheonController the PantheonController that defines blockchain behavior
* @param <C> the consensus context type
* @param startBlock the starting index of the block list to export (inclusive)
* @param endBlock the ending index of the block list to export (exclusive), if not specified a
* single block will be export
* @return the export result
*/
public <C> ExportResult exportBlockchain(
final PantheonController<C> pantheonController, final Long startBlock, final Long endBlock) {
final ProtocolContext<C> context = pantheonController.getProtocolContext();
final MutableBlockchain blockchain = context.getBlockchain();
final Long sanitizedEndBlock = sanitizedEndBlockIndex(startBlock, endBlock);
final List<Block> blocks = new ArrayList<>();
for (long currentBlockIndex = startBlock;
currentBlockIndex < sanitizedEndBlock;
currentBlockIndex += 1) {
Optional<Hash> blockHashByNumber = blockchain.getBlockHashByNumber(currentBlockIndex);
blockHashByNumber.ifPresent(hash -> blocks.add(blockchain.getBlockByHash(hash)));
}
final boolean allBlocksAreFound = blocks.size() == (sanitizedEndBlock - startBlock);
return new ExportResult(blocks, allBlocksAreFound);
}
private Long sanitizedEndBlockIndex(final Long startBlock, final Long endBlock) {
if (endBlock == null) {
return startBlock + 1;
} else {
return endBlock;
}
}
public static final class ExportResult {
public final List<Block> blocks;
public final boolean allBlocksAreFound;
ExportResult(final List<Block> blocks, final boolean allBlocksAreFound) {
this.blocks = blocks;
this.allBlocksAreFound = allBlocksAreFound;
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("blocks", blocks)
.add("allBlocksAreFound", allBlocksAreFound)
.toString();
}
}
}

@ -0,0 +1,245 @@
/*
* Copyright 2018 ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.chainexport;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import tech.pegasys.pantheon.chainimport.RlpBlockImporter;
import tech.pegasys.pantheon.config.GenesisConfigFile;
import tech.pegasys.pantheon.controller.PantheonController;
import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair;
import tech.pegasys.pantheon.ethereum.chain.Blockchain;
import tech.pegasys.pantheon.ethereum.core.Block;
import tech.pegasys.pantheon.ethereum.core.BlockBody;
import tech.pegasys.pantheon.ethereum.core.BlockHeader;
import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider;
import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder;
import tech.pegasys.pantheon.ethereum.core.PrivacyParameters;
import tech.pegasys.pantheon.ethereum.eth.EthProtocolConfiguration;
import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration;
import tech.pegasys.pantheon.ethereum.eth.transactions.TransactionPoolConfiguration;
import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule;
import tech.pegasys.pantheon.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
import tech.pegasys.pantheon.ethereum.util.RawBlockIterator;
import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem;
import tech.pegasys.pantheon.testutil.BlockTestUtil;
import tech.pegasys.pantheon.testutil.TestClock;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Optional;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
/** Tests for {@link BlockExporter}. */
public final class RlpBlockExporterTest {
@ClassRule public static final TemporaryFolder folder = new TemporaryFolder();
private static Blockchain blockchain;
private static long chainHead;
private static ProtocolSchedule<?> protocolSchedule;
@BeforeClass
public static void setupBlockchain() throws IOException {
final PantheonController<?> controller = createController();
final Path blocks = folder.newFile("1000.blocks").toPath();
BlockTestUtil.write1000Blocks(blocks);
blockchain = importBlocks(controller, blocks);
chainHead = blockchain.getChainHeadBlockNumber();
protocolSchedule = controller.getProtocolSchedule();
}
private static Blockchain importBlocks(
final PantheonController<?> controller, final Path blocksFile) throws IOException {
final RlpBlockImporter blockImporter = new RlpBlockImporter();
blockImporter.importBlockchain(blocksFile, controller);
return controller.getProtocolContext().getBlockchain();
}
private static PantheonController<?> createController() throws IOException {
final Path dataDir = folder.newFolder().toPath();
return new PantheonController.Builder()
.fromGenesisConfig(GenesisConfigFile.mainnet())
.synchronizerConfiguration(SynchronizerConfiguration.builder().build())
.ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
.storageProvider(new InMemoryStorageProvider())
.networkId(1)
.miningParameters(new MiningParametersTestBuilder().enabled(false).build())
.nodeKeys(KeyPair.generate())
.metricsSystem(new NoOpMetricsSystem())
.privacyParameters(PrivacyParameters.DEFAULT)
.dataDirectory(dataDir)
.clock(TestClock.fixed())
.transactionPoolConfiguration(TransactionPoolConfiguration.builder().build())
.build();
}
@Test
public void exportBlocks_noBounds() throws IOException {
final File outputPath = folder.newFile();
final RlpBlockExporter exporter = new RlpBlockExporter(blockchain);
exporter.exportBlocks(outputPath, Optional.empty(), Optional.empty());
// Iterate over blocks and check that they match expectations
RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath());
long currentBlockNumber = 0;
while (blockIterator.hasNext()) {
final Block actual = blockIterator.next();
final Block expected = getBlock(blockchain, currentBlockNumber);
assertThat(actual).isEqualTo(expected);
currentBlockNumber++;
}
// Check that we iterated to the end of the chain
assertThat(currentBlockNumber).isEqualTo(chainHead + 1L);
}
@Test
public void exportBlocks_withLowerBound() throws IOException {
final File outputPath = folder.newFile();
final RlpBlockExporter exporter = new RlpBlockExporter(blockchain);
final long lowerBound = 990;
exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.empty());
// Iterate over blocks and check that they match expectations
RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath());
long currentBlockNumber = lowerBound;
while (blockIterator.hasNext()) {
final Block actual = blockIterator.next();
final Block expected = getBlock(blockchain, currentBlockNumber);
assertThat(actual).isEqualTo(expected);
currentBlockNumber++;
}
// Check that we iterated to the end of the chain
assertThat(currentBlockNumber).isEqualTo(chainHead + 1L);
}
@Test
public void exportBlocks_withUpperBound() throws IOException {
final File outputPath = folder.newFile();
final RlpBlockExporter exporter = new RlpBlockExporter(blockchain);
final long upperBound = 10;
exporter.exportBlocks(outputPath, Optional.empty(), Optional.of(upperBound));
// Iterate over blocks and check that they match expectations
RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath());
long currentBlockNumber = 0;
while (blockIterator.hasNext()) {
final Block actual = blockIterator.next();
final Block expected = getBlock(blockchain, currentBlockNumber);
assertThat(actual).isEqualTo(expected);
currentBlockNumber++;
}
// Check that we iterated to the end of the chain
assertThat(currentBlockNumber).isEqualTo(upperBound);
}
@Test
public void exportBlocks_withUpperAndLowerBounds() throws IOException {
final File outputPath = folder.newFile();
final RlpBlockExporter exporter = new RlpBlockExporter(blockchain);
final long lowerBound = 5;
final long upperBound = 10;
exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.of(upperBound));
// Iterate over blocks and check that they match expectations
RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath());
long currentBlockNumber = lowerBound;
while (blockIterator.hasNext()) {
final Block actual = blockIterator.next();
final Block expected = getBlock(blockchain, currentBlockNumber);
assertThat(actual).isEqualTo(expected);
currentBlockNumber++;
}
// Check that we iterated to the end of the chain
assertThat(currentBlockNumber).isEqualTo(upperBound);
}
@Test
public void exportBlocks_withRangeBeyondChainHead() throws IOException {
final File outputPath = folder.newFile();
final RlpBlockExporter exporter = new RlpBlockExporter(blockchain);
final long lowerBound = chainHead - 10;
final long upperBound = chainHead + 10;
exporter.exportBlocks(outputPath, Optional.of(lowerBound), Optional.of(upperBound));
// Iterate over blocks and check that they match expectations
RawBlockIterator blockIterator = getBlockIterator(outputPath.toPath());
long currentBlockNumber = lowerBound;
while (blockIterator.hasNext()) {
final Block actual = blockIterator.next();
final Block expected = getBlock(blockchain, currentBlockNumber);
assertThat(actual).isEqualTo(expected);
currentBlockNumber++;
}
// Check that we iterated to the end of the chain
assertThat(currentBlockNumber).isEqualTo(chainHead + 1L);
}
@Test
public void exportBlocks_negativeStartNumber() throws IOException {
final File outputPath = folder.newFile();
final RlpBlockExporter exporter = new RlpBlockExporter(blockchain);
assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.of(-1L), Optional.empty()))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("greater than 0");
}
@Test
public void exportBlocks_negativeEndNumber() throws IOException {
final File outputPath = folder.newFile();
final RlpBlockExporter exporter = new RlpBlockExporter(blockchain);
assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.empty(), Optional.of(-1L)))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("greater than 0");
}
@Test
public void exportBlocks_outOfOrderBounds() throws IOException {
final File outputPath = folder.newFile();
final RlpBlockExporter exporter = new RlpBlockExporter(blockchain);
assertThatThrownBy(() -> exporter.exportBlocks(outputPath, Optional.of(10L), Optional.of(2L)))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("Start block must be less than end block");
}
private RawBlockIterator getBlockIterator(final Path blocks) throws IOException {
return new RawBlockIterator(
blocks,
rlp ->
BlockHeader.readFrom(rlp, ScheduleBasedBlockHeaderFunctions.create(protocolSchedule)));
}
private Block getBlock(final Blockchain blockchain, final long blockNumber) {
final BlockHeader header = blockchain.getBlockHeader(blockNumber).get();
final BlockBody body = blockchain.getBlockBody(header.getHash()).get();
return new Block(header, body);
}
}

@ -54,7 +54,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
public abstract class ChainImporterTest {
public abstract class JsonBlockImporterTest {
@Rule public final TemporaryFolder folder = new TemporaryFolder();
@ -62,14 +62,14 @@ public abstract class ChainImporterTest {
protected final GenesisConfigFile genesisConfigFile;
protected final boolean isEthash;
public ChainImporterTest(final String consensusEngine) throws IOException {
public JsonBlockImporterTest(final String consensusEngine) throws IOException {
this.consensusEngine = consensusEngine;
final String genesisData = getFileContents("genesis.json");
this.genesisConfigFile = GenesisConfigFile.fromConfig(genesisData);
this.isEthash = genesisConfigFile.getConfigOptions().isEthHash();
}
public static class SingletonTests extends ChainImporterTest {
public static class SingletonTests extends JsonBlockImporterTest {
public SingletonTests() throws IOException {
super("unsupported");
}
@ -77,7 +77,7 @@ public abstract class ChainImporterTest {
@Test
public void importChain_unsupportedConsensusAlgorithm() throws IOException {
final PantheonController<?> controller = createController();
final ChainImporter<?> importer = new ChainImporter<>(controller);
final JsonBlockImporter<?> importer = new JsonBlockImporter<>(controller);
final String jsonData = getFileContents("clique", "blocks-import-valid.json");
@ -90,7 +90,7 @@ public abstract class ChainImporterTest {
}
@RunWith(Parameterized.class)
public static class ParameterizedTests extends ChainImporterTest {
public static class ParameterizedTests extends JsonBlockImporterTest {
public ParameterizedTests(final String consensusEngine) throws IOException {
super(consensusEngine);
@ -105,7 +105,7 @@ public abstract class ChainImporterTest {
@Test
public void importChain_validJson_withBlockNumbers() throws IOException {
final PantheonController<?> controller = createController();
final ChainImporter<?> importer = new ChainImporter<>(controller);
final JsonBlockImporter<?> importer = new JsonBlockImporter<>(controller);
final String jsonData = getFileContents("blocks-import-valid.json");
importer.importChain(jsonData);
@ -196,7 +196,7 @@ public abstract class ChainImporterTest {
@Test
public void importChain_validJson_noBlockIdentifiers() throws IOException {
final PantheonController<?> controller = createController();
final ChainImporter<?> importer = new ChainImporter<>(controller);
final JsonBlockImporter<?> importer = new JsonBlockImporter<>(controller);
final String jsonData = getFileContents("blocks-import-valid-no-block-identifiers.json");
importer.importChain(jsonData);
@ -287,7 +287,7 @@ public abstract class ChainImporterTest {
@Test
public void importChain_validJson_withParentHashes() throws IOException {
final PantheonController<?> controller = createController();
final ChainImporter<?> importer = new ChainImporter<>(controller);
final JsonBlockImporter<?> importer = new JsonBlockImporter<>(controller);
String jsonData = getFileContents("blocks-import-valid.json");
@ -338,7 +338,7 @@ public abstract class ChainImporterTest {
@Test
public void importChain_invalidParent() throws IOException {
final PantheonController<?> controller = createController();
final ChainImporter<?> importer = new ChainImporter<>(controller);
final JsonBlockImporter<?> importer = new JsonBlockImporter<>(controller);
final String jsonData = getFileContents("blocks-import-invalid-bad-parent.json");
@ -350,7 +350,7 @@ public abstract class ChainImporterTest {
@Test
public void importChain_invalidTransaction() throws IOException {
final PantheonController<?> controller = createController();
final ChainImporter<?> importer = new ChainImporter<>(controller);
final JsonBlockImporter<?> importer = new JsonBlockImporter<>(controller);
final String jsonData = getFileContents("blocks-import-invalid-bad-tx.json");
@ -363,7 +363,7 @@ public abstract class ChainImporterTest {
@Test
public void importChain_specialFields() throws IOException {
final PantheonController<?> controller = createController();
final ChainImporter<?> importer = new ChainImporter<>(controller);
final JsonBlockImporter<?> importer = new JsonBlockImporter<>(controller);
final String jsonData = getFileContents("blocks-import-special-fields.json");

@ -10,7 +10,7 @@
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.util;
package tech.pegasys.pantheon.chainimport;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.assertj.core.api.Assertions.assertThat;
@ -39,12 +39,12 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
/** Tests for {@link BlockImporter}. */
public final class BlockImporterTest {
/** Tests for {@link RlpBlockImporter}. */
public final class RlpBlockImporterTest {
@Rule public final TemporaryFolder folder = new TemporaryFolder();
BlockImporter blockImporter = new BlockImporter();
private RlpBlockImporter rlpBlockImporter = new RlpBlockImporter();
@Test
public void blockImport() throws IOException {
@ -66,8 +66,8 @@ public final class BlockImporterTest {
.clock(TestClock.fixed())
.transactionPoolConfiguration(TransactionPoolConfiguration.builder().build())
.build();
final BlockImporter.ImportResult result =
blockImporter.importBlockchain(source, targetController);
final RlpBlockImporter.ImportResult result =
rlpBlockImporter.importBlockchain(source, targetController);
// Don't count the Genesis block
assertThat(result.count).isEqualTo(999);
assertThat(result.td).isEqualTo(UInt256.of(21991996248790L));
@ -105,7 +105,8 @@ public final class BlockImporterTest {
.clock(TestClock.fixed())
.transactionPoolConfiguration(TransactionPoolConfiguration.builder().build())
.build();
final BlockImporter.ImportResult result = blockImporter.importBlockchain(source, controller);
final RlpBlockImporter.ImportResult result =
rlpBlockImporter.importBlockchain(source, controller);
// Don't count the Genesis block
assertThat(result.count).isEqualTo(958);

@ -23,7 +23,9 @@ import static org.mockito.Mockito.when;
import tech.pegasys.pantheon.Runner;
import tech.pegasys.pantheon.RunnerBuilder;
import tech.pegasys.pantheon.chainimport.ChainImporter;
import tech.pegasys.pantheon.chainexport.RlpBlockExporter;
import tech.pegasys.pantheon.chainimport.JsonBlockImporter;
import tech.pegasys.pantheon.chainimport.RlpBlockImporter;
import tech.pegasys.pantheon.cli.config.EthNetworkConfig;
import tech.pegasys.pantheon.cli.options.EthProtocolOptions;
import tech.pegasys.pantheon.cli.options.MetricsCLIOptions;
@ -32,7 +34,8 @@ import tech.pegasys.pantheon.cli.options.RocksDBOptions;
import tech.pegasys.pantheon.cli.options.SynchronizerOptions;
import tech.pegasys.pantheon.cli.options.TransactionPoolOptions;
import tech.pegasys.pantheon.cli.subcommands.PublicKeySubCommand.KeyLoader;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.ChainImporterFactory;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.JsonBlockImporterFactory;
import tech.pegasys.pantheon.cli.subcommands.blocks.BlocksSubCommand.RlpBlockExporterFactory;
import tech.pegasys.pantheon.controller.PantheonController;
import tech.pegasys.pantheon.controller.PantheonControllerBuilder;
import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair;
@ -48,8 +51,6 @@ import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule;
import tech.pegasys.pantheon.ethereum.permissioning.PermissioningConfiguration;
import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration;
import tech.pegasys.pantheon.services.PantheonPluginContextImpl;
import tech.pegasys.pantheon.util.BlockExporter;
import tech.pegasys.pantheon.util.BlockImporter;
import tech.pegasys.pantheon.util.bytes.BytesValue;
import java.io.ByteArrayOutputStream;
@ -101,9 +102,9 @@ public abstract class CommandTestAbstract {
@Mock protected ProtocolContext<Object> mockProtocolContext;
@Mock protected BlockBroadcaster mockBlockBroadcaster;
@Mock protected PantheonController<Object> mockController;
@Mock protected BlockImporter mockBlockImporter;
@Mock protected BlockExporter mockBlockExporter;
@Mock protected ChainImporter<?> chainImporter;
@Mock protected RlpBlockExporter rlpBlockExporter;
@Mock protected JsonBlockImporter<?> jsonBlockImporter;
@Mock protected RlpBlockImporter rlpBlockImporter;
@Mock protected Logger mockLogger;
@Mock protected PantheonPluginContextImpl mockPantheonPluginContext;
@ -215,8 +216,9 @@ public abstract class CommandTestAbstract {
}
@SuppressWarnings("unchecked")
private <T> ChainImporter<T> chainImporterFactory(final PantheonController<T> controller) {
return (ChainImporter<T>) chainImporter;
private <T> JsonBlockImporter<T> jsonBlockImporterFactory(
final PantheonController<T> controller) {
return (JsonBlockImporter<T>) jsonBlockImporter;
}
private TestPantheonCommand parseCommand(
@ -227,9 +229,9 @@ public abstract class CommandTestAbstract {
final TestPantheonCommand pantheonCommand =
new TestPantheonCommand(
mockLogger,
mockBlockImporter,
mockBlockExporter,
this::chainImporterFactory,
rlpBlockImporter,
this::jsonBlockImporterFactory,
(blockchain) -> rlpBlockExporter,
mockRunnerBuilder,
mockControllerBuilderFactory,
keyLoader,
@ -257,9 +259,9 @@ public abstract class CommandTestAbstract {
TestPantheonCommand(
final Logger mockLogger,
final BlockImporter mockBlockImporter,
final BlockExporter mockBlockExporter,
final ChainImporterFactory chainImporterFactory,
final RlpBlockImporter mockBlockImporter,
final JsonBlockImporterFactory jsonBlockImporterFactory,
final RlpBlockExporterFactory rlpBlockExporterFactory,
final RunnerBuilder mockRunnerBuilder,
final PantheonController.Builder controllerBuilderFactory,
final KeyLoader keyLoader,
@ -268,8 +270,8 @@ public abstract class CommandTestAbstract {
super(
mockLogger,
mockBlockImporter,
mockBlockExporter,
chainImporterFactory,
jsonBlockImporterFactory,
rlpBlockExporterFactory,
mockRunnerBuilder,
controllerBuilderFactory,
pantheonPluginContext,

@ -14,36 +14,21 @@ package tech.pegasys.pantheon.cli.subcommands.blocks;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.contentOf;
import static org.assertj.core.util.Arrays.asList;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import tech.pegasys.pantheon.cli.CommandTestAbstract;
import tech.pegasys.pantheon.config.GenesisConfigFile;
import tech.pegasys.pantheon.controller.PantheonController;
import tech.pegasys.pantheon.crypto.SECP256K1;
import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain;
import tech.pegasys.pantheon.ethereum.core.Block;
import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider;
import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder;
import tech.pegasys.pantheon.ethereum.core.PrivacyParameters;
import tech.pegasys.pantheon.ethereum.eth.EthProtocolConfiguration;
import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration;
import tech.pegasys.pantheon.ethereum.eth.transactions.TransactionPoolConfiguration;
import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem;
import tech.pegasys.pantheon.testutil.BlockTestUtil;
import tech.pegasys.pantheon.testutil.TestClock;
import tech.pegasys.pantheon.util.BlockExporter;
import tech.pegasys.pantheon.util.BlockImporter;
import java.io.File;
import java.io.IOException;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
import org.junit.Rule;
import org.junit.Test;
@ -87,9 +72,9 @@ public class BlocksSubCommandTest extends CommandTestAbstract {
+ System.lineSeparator();
private static final String EXPECTED_BLOCK_EXPORT_USAGE =
"Usage: pantheon blocks export [-hV] [--end-block=<LONG>] --start-block=<LONG>"
"Usage: pantheon blocks export [-hV] [--end-block=<LONG>] [--start-block=<LONG>]"
+ System.lineSeparator()
+ " [--to=<FILE>]"
+ " --to=<FILE>"
+ System.lineSeparator()
+ "This command export a specific block from storage"
+ System.lineSeparator()
@ -161,7 +146,7 @@ public class BlocksSubCommandTest extends CommandTestAbstract {
parseCommand(
BLOCK_SUBCOMMAND_NAME, BLOCK_IMPORT_SUBCOMMAND_NAME, "--from", fileToImport.getPath());
verify(mockBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any());
verify(rlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any());
assertThat(pathArgumentCaptor.getValue()).isEqualByComparingTo(fileToImport.toPath());
@ -180,7 +165,7 @@ public class BlocksSubCommandTest extends CommandTestAbstract {
"--from",
fileToImport.getPath());
verify(mockBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any());
verify(rlpBlockImporter).importBlockchain(pathArgumentCaptor.capture(), any());
assertThat(pathArgumentCaptor.getValue()).isEqualByComparingTo(fileToImport.toPath());
@ -207,107 +192,221 @@ public class BlocksSubCommandTest extends CommandTestAbstract {
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).isEmpty();
verify(chainImporter, times(1)).importChain(stringArgumentCaptor.capture());
verify(jsonBlockImporter, times(1)).importChain(stringArgumentCaptor.capture());
assertThat(stringArgumentCaptor.getValue()).isEqualTo(fileContent);
}
// Export sub-sub-command
@Test
public void callingBlockExportSubCommandWithoutStartBlockMustDisplayErrorAndUsage() {
parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME);
final String expectedErrorOutputStart = "Missing required option '--start-block=<LONG>'";
public void blocksExport_missingFileParam() throws IOException {
createDbDirectory(true);
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME);
final String expectedErrorOutputStart = "Missing required option '--to=<FILE>'";
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart);
verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any());
}
@Test
public void callingBlockExportSubCommandHelpMustDisplayUsage() {
parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--help");
assertThat(commandOutput.toString()).startsWith(EXPECTED_BLOCK_EXPORT_USAGE);
assertThat(commandErrorOutput.toString()).isEmpty();
public void blocksExport_noDbDirectory() throws IOException {
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath());
final String expectedErrorOutputStart =
"Chain is empty. Unable to export blocks from specified data directory: "
+ folder.getRoot().getAbsolutePath()
+ "/"
+ PantheonController.DATABASE_PATH;
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart);
verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any());
}
@Test
public void callingBlockExportSubCommandWithNegativeStartBlockMustDisplayErrorAndUsage() {
final String expectedErrorOutputStart = "--start-block must be greater than or equal to zero";
parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=-1");
public void blocksExport_emptyDbDirectory() throws IOException {
createDbDirectory(false);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath());
final String expectedErrorOutputStart =
"Chain is empty. Unable to export blocks from specified data directory: "
+ folder.getRoot().getAbsolutePath()
+ "/"
+ PantheonController.DATABASE_PATH;
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart);
verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any());
}
@Test
public void
callingBlockExportSubCommandWithEndBlockGreaterThanStartBlockMustDisplayErrorMessage() {
final String expectedErrorOutputStart = "--end-block must be greater than --start-block";
public void blocksExport_noStartOrEnd() throws IOException {
createDbDirectory(true);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=2", "--end-block=1");
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath());
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart);
assertThat(commandErrorOutput.toString()).isEmpty();
verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.empty(), Optional.empty());
}
@Test
public void callingBlockExportSubCommandWithEndBlockEqualToStartBlockMustDisplayErrorMessage() {
final String expectedErrorOutputStart = "--end-block must be greater than --start-block";
public void blocksExport_withStartAndNoEnd() throws IOException {
createDbDirectory(true);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--start-block=2", "--end-block=2");
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath(),
"--start-block=1");
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart);
assertThat(commandErrorOutput.toString()).isEmpty();
verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.of(1L), Optional.empty());
}
@Test
public void blocksExport_withEndAndNoStart() throws IOException {
createDbDirectory(true);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath(),
"--end-block=10");
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).isEmpty();
verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.empty(), Optional.of(10L));
}
@Test
public void callingExportSubCommandWithFilePathMustWriteBlockInThisFile() throws Exception {
public void blocksExport_withStartAndEnd() throws IOException {
createDbDirectory(true);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath(),
"--start-block=1",
"--end-block=10");
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).isEmpty();
final long startBlock = 0L;
final long endBlock = 2L;
final PantheonController<?> pantheonController = initPantheonController();
final MutableBlockchain blockchain = pantheonController.getProtocolContext().getBlockchain();
final File outputFile = File.createTempFile("export", "store");
verify(rlpBlockExporter, times(1)).exportBlocks(outputFile, Optional.of(1L), Optional.of(10L));
}
mockBlockExporter = new BlockExporter();
@Test
public void blocksExport_withOutOfOrderStartAndEnd() throws IOException {
createDbDirectory(true);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath(),
"--start-block=10",
"--end-block=1");
assertThat(commandErrorOutput.toString())
.contains("Parameter --end-block (1) must be greater start block (10)");
assertThat(commandOutput.toString()).isEmpty();
doReturn(pantheonController).when(mockControllerBuilder).build();
verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any());
}
@Test
public void blocksExport_withEmptyRange() throws IOException {
createDbDirectory(true);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--start-block=" + startBlock,
"--end-block=" + endBlock,
"--to=" + outputFile.getPath());
"--to",
outputFile.getPath(),
"--start-block=10",
"--end-block=10");
assertThat(commandErrorOutput.toString())
.contains("Parameter --end-block (10) must be greater start block (10)");
assertThat(commandOutput.toString()).isEmpty();
final Block blockFromBlockchain =
blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get());
verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any());
}
final Block secondBlockFromBlockchain =
blockchain.getBlockByHash(blockchain.getBlockHashByNumber(endBlock - 1).get());
@Test
public void blocksExport_withInvalidStart() throws IOException {
createDbDirectory(true);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath(),
"--start-block=-1");
assertThat(commandErrorOutput.toString())
.contains("Parameter --start-block (-1) must be greater than or equal to zero");
assertThat(commandOutput.toString()).isEmpty();
assertThat(contentOf(outputFile))
.isEqualTo(asList(new Block[] {blockFromBlockchain, secondBlockFromBlockchain}).toString());
verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any());
}
@Test
public void blocksExport_withInvalidEnd() throws IOException {
createDbDirectory(true);
final File outputFile = folder.newFile("blocks.bin");
parseCommand(
"--data-path=" + folder.getRoot().getAbsolutePath(),
BLOCK_SUBCOMMAND_NAME,
BLOCK_EXPORT_SUBCOMMAND_NAME,
"--to",
outputFile.getPath(),
"--end-block=-1");
assertThat(commandErrorOutput.toString())
.contains("Parameter --end-block (-1) must be greater than or equal to zero");
assertThat(commandOutput.toString()).isEmpty();
verify(rlpBlockExporter, never()).exportBlocks(any(), any(), any());
}
@Test
public void callingBlockExportSubCommandHelpMustDisplayUsage() {
parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_EXPORT_SUBCOMMAND_NAME, "--help");
assertThat(commandOutput.toString()).startsWith(EXPECTED_BLOCK_EXPORT_USAGE);
assertThat(commandErrorOutput.toString()).isEmpty();
}
private PantheonController<?> initPantheonController() throws IOException {
final BlockImporter blockImporter = new BlockImporter();
final Path dataDir = folder.newFolder().toPath();
final Path source = dataDir.resolve("1000.blocks");
BlockTestUtil.write1000Blocks(source);
final PantheonController<?> targetController =
new PantheonController.Builder()
.fromGenesisConfig(GenesisConfigFile.mainnet())
.synchronizerConfiguration(SynchronizerConfiguration.builder().build())
.ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
.storageProvider(new InMemoryStorageProvider())
.networkId(1)
.miningParameters(new MiningParametersTestBuilder().enabled(false).build())
.nodeKeys(SECP256K1.KeyPair.generate())
.metricsSystem(new NoOpMetricsSystem())
.privacyParameters(PrivacyParameters.DEFAULT)
.dataDirectory(dataDir)
.clock(TestClock.fixed())
.transactionPoolConfiguration(TransactionPoolConfiguration.builder().build())
.build();
blockImporter.importBlockchain(source, targetController);
return targetController;
private void createDbDirectory(final boolean createDataFiles) throws IOException {
File dbDir = folder.newFolder(PantheonController.DATABASE_PATH);
if (createDataFiles) {
Path dataFilePath = Paths.get(dbDir.getAbsolutePath(), "0000001.sst");
final boolean success = new File(dataFilePath.toString()).createNewFile();
assertThat(success).isTrue();
}
}
}

@ -1,111 +0,0 @@
/*
* Copyright 2018 ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package tech.pegasys.pantheon.util;
import static org.assertj.core.api.Assertions.assertThat;
import tech.pegasys.pantheon.config.GenesisConfigFile;
import tech.pegasys.pantheon.controller.PantheonController;
import tech.pegasys.pantheon.crypto.SECP256K1;
import tech.pegasys.pantheon.ethereum.chain.MutableBlockchain;
import tech.pegasys.pantheon.ethereum.core.Block;
import tech.pegasys.pantheon.ethereum.core.InMemoryStorageProvider;
import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder;
import tech.pegasys.pantheon.ethereum.core.PrivacyParameters;
import tech.pegasys.pantheon.ethereum.eth.EthProtocolConfiguration;
import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration;
import tech.pegasys.pantheon.ethereum.eth.transactions.TransactionPoolConfiguration;
import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem;
import tech.pegasys.pantheon.testutil.BlockTestUtil;
import tech.pegasys.pantheon.testutil.TestClock;
import java.io.IOException;
import java.nio.file.Path;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
/** Tests for {@link BlockExporter}. */
public final class BlockExporterTest {
@ClassRule public static final TemporaryFolder folder = new TemporaryFolder();
private static PantheonController<?> targetController;
private final BlockExporter blockExporter = new BlockExporter();
@BeforeClass
public static void initPantheonController() throws IOException {
final BlockImporter blockImporter = new BlockImporter();
final Path dataDir = folder.newFolder().toPath();
final Path source = dataDir.resolve("1000.blocks");
BlockTestUtil.write1000Blocks(source);
targetController =
new PantheonController.Builder()
.fromGenesisConfig(GenesisConfigFile.mainnet())
.synchronizerConfiguration(SynchronizerConfiguration.builder().build())
.ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig())
.storageProvider(new InMemoryStorageProvider())
.networkId(1)
.miningParameters(new MiningParametersTestBuilder().enabled(false).build())
.nodeKeys(SECP256K1.KeyPair.generate())
.metricsSystem(new NoOpMetricsSystem())
.privacyParameters(PrivacyParameters.DEFAULT)
.dataDirectory(dataDir)
.clock(TestClock.fixed())
.transactionPoolConfiguration(TransactionPoolConfiguration.builder().build())
.build();
blockImporter.importBlockchain(source, targetController);
}
@Test
public void callingBlockExporterWithOnlyStartBlockShouldReturnOneBlock() throws Exception {
final long startBlock = 0L;
final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain();
final Block blockFromBlockchain =
blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get());
BlockExporter.ExportResult exportResult =
blockExporter.exportBlockchain(targetController, startBlock, null);
assertThat(exportResult.blocks).contains(blockFromBlockchain);
}
@Test
public void callingBlockExporterWithStartBlockAndBlockShouldReturnSeveralBlocks()
throws Exception {
final long startBlock = 0L;
final long endBlock = 1L;
final MutableBlockchain blockchain = targetController.getProtocolContext().getBlockchain();
final Block blockFromBlockchain =
blockchain.getBlockByHash(blockchain.getBlockHashByNumber(startBlock).get());
final Block secondBlockFromBlockchain =
blockchain.getBlockByHash(blockchain.getBlockHashByNumber(endBlock - 1).get());
BlockExporter.ExportResult exportResult =
blockExporter.exportBlockchain(targetController, startBlock, endBlock);
assertThat(exportResult.blocks)
.contains(blockFromBlockchain)
.contains(secondBlockFromBlockchain);
}
}
Loading…
Cancel
Save