Bulk block file import (#1033)

Extend the `blocks import` subcommand to accept multiple files in the
command line and import them in the same batch.

Signed-off-by: Danno Ferrin <danno.ferrin@gmail.com>
pull/1043/head
Danno Ferrin 5 years ago committed by GitHub
parent d4208892c0
commit 2d79171746
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      besu/src/main/java/org/hyperledger/besu/Besu.java
  2. 34
      besu/src/main/java/org/hyperledger/besu/chainimport/RlpBlockImporter.java
  3. 24
      besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
  4. 107
      besu/src/main/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommand.java
  5. 12
      besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java
  6. 50
      besu/src/test/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommandTest.java

@ -38,7 +38,7 @@ public final class Besu {
final BesuCommand besuCommand =
new BesuCommand(
logger,
new RlpBlockImporter(),
RlpBlockImporter::new,
JsonBlockImporter::new,
RlpBlockExporter::new,
new RunnerBuilder(),

@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
import org.hyperledger.besu.ethereum.util.RawBlockIterator;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
@ -45,7 +46,7 @@ import com.google.common.base.MoreObjects;
import org.apache.logging.log4j.Logger;
/** Tool for importing rlp-encoded block data from files. */
public class RlpBlockImporter {
public class RlpBlockImporter implements Closeable {
private static final Logger LOG = getLogger();
private final Semaphore blockBacklog = new Semaphore(2);
@ -133,20 +134,6 @@ public class RlpBlockImporter {
}
return new RlpBlockImporter.ImportResult(
blockchain.getChainHead().getTotalDifficulty(), count);
} finally {
validationExecutor.shutdownNow();
try {
validationExecutor.awaitTermination(5, SECONDS);
} catch (final Exception e) {
LOG.error("Error shutting down validatorExecutor.", e);
}
importExecutor.shutdownNow();
try {
importExecutor.awaitTermination(5, SECONDS);
} catch (final Exception e) {
LOG.error("Error shutting down importExecutor", e);
}
besuController.close();
}
}
@ -218,6 +205,23 @@ public class RlpBlockImporter {
header.getNumber(), blockchain.getChainHeadBlockNumber())));
}
@Override
public void close() {
validationExecutor.shutdownNow();
try {
validationExecutor.awaitTermination(5, SECONDS);
} catch (final Exception e) {
LOG.error("Error shutting down validatorExecutor.", e);
}
importExecutor.shutdownNow();
try {
importExecutor.awaitTermination(5, SECONDS);
} catch (final Exception e) {
LOG.error("Error shutting down importExecutor", e);
}
}
public static final class ImportResult {
public final Difficulty td;

@ -33,6 +33,8 @@ import static org.hyperledger.besu.metrics.prometheus.MetricsConfiguration.DEFAU
import org.hyperledger.besu.BesuInfo;
import org.hyperledger.besu.Runner;
import org.hyperledger.besu.RunnerBuilder;
import org.hyperledger.besu.chainexport.RlpBlockExporter;
import org.hyperledger.besu.chainimport.JsonBlockImporter;
import org.hyperledger.besu.chainimport.RlpBlockImporter;
import org.hyperledger.besu.cli.config.EthNetworkConfig;
import org.hyperledger.besu.cli.config.NetworkName;
@ -54,8 +56,6 @@ import org.hyperledger.besu.cli.subcommands.PasswordSubCommand;
import org.hyperledger.besu.cli.subcommands.PublicKeySubCommand;
import org.hyperledger.besu.cli.subcommands.RetestethSubCommand;
import org.hyperledger.besu.cli.subcommands.blocks.BlocksSubCommand;
import org.hyperledger.besu.cli.subcommands.blocks.BlocksSubCommand.JsonBlockImporterFactory;
import org.hyperledger.besu.cli.subcommands.blocks.BlocksSubCommand.RlpBlockExporterFactory;
import org.hyperledger.besu.cli.subcommands.operator.OperatorSubCommand;
import org.hyperledger.besu.cli.subcommands.rlp.RLPSubCommand;
import org.hyperledger.besu.cli.util.BesuCommandCustomFactory;
@ -80,6 +80,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguratio
import org.hyperledger.besu.ethereum.api.tls.FileBasedPasswordProvider;
import org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration;
import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.core.Address;
import org.hyperledger.besu.ethereum.core.Hash;
import org.hyperledger.besu.ethereum.core.MiningParameters;
@ -150,6 +151,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -195,9 +197,9 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
private CommandLine commandLine;
private final RlpBlockImporter rlpBlockImporter;
private final JsonBlockImporterFactory jsonBlockImporterFactory;
private final RlpBlockExporterFactory rlpBlockExporterFactory;
private final Supplier<RlpBlockImporter> rlpBlockImporter;
private final Function<BesuController<?>, JsonBlockImporter<?>> jsonBlockImporterFactory;
private final Function<Blockchain, RlpBlockExporter> rlpBlockExporterFactory;
final NetworkingOptions networkingOptions = NetworkingOptions.create();
final SynchronizerOptions synchronizerOptions = SynchronizerOptions.create();
@ -1001,9 +1003,9 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
public BesuCommand(
final Logger logger,
final RlpBlockImporter rlpBlockImporter,
final JsonBlockImporterFactory jsonBlockImporterFactory,
final RlpBlockExporterFactory rlpBlockExporterFactory,
final Supplier<RlpBlockImporter> rlpBlockImporter,
final Function<BesuController<?>, JsonBlockImporter<?>> jsonBlockImporterFactory,
final Function<Blockchain, RlpBlockExporter> rlpBlockExporterFactory,
final RunnerBuilder runnerBuilder,
final BesuController.Builder controllerBuilderFactory,
final BesuPluginContextImpl besuPluginContext,
@ -1024,9 +1026,9 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
@VisibleForTesting
protected BesuCommand(
final Logger logger,
final RlpBlockImporter rlpBlockImporter,
final JsonBlockImporterFactory jsonBlockImporterFactory,
final RlpBlockExporterFactory rlpBlockExporterFactory,
final Supplier<RlpBlockImporter> rlpBlockImporter,
final Function<BesuController<?>, JsonBlockImporter<?>> jsonBlockImporterFactory,
final Function<Blockchain, RlpBlockExporter> rlpBlockExporterFactory,
final RunnerBuilder runnerBuilder,
final BesuController.Builder controllerBuilderFactory,
final BesuPluginContextImpl besuPluginContext,

@ -44,7 +44,11 @@ import java.nio.file.Paths;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
import io.vertx.core.Vertx;
import org.apache.logging.log4j.LogManager;
@ -56,6 +60,7 @@ import picocli.CommandLine.ExecutionException;
import picocli.CommandLine.Model.CommandSpec;
import picocli.CommandLine.Option;
import picocli.CommandLine.ParameterException;
import picocli.CommandLine.Parameters;
import picocli.CommandLine.ParentCommand;
import picocli.CommandLine.Spec;
@ -79,17 +84,16 @@ public class BlocksSubCommand implements Runnable {
@Spec
private CommandSpec spec; // Picocli injects reference to command spec
private final RlpBlockImporter rlpBlockImporter;
private final JsonBlockImporterFactory jsonBlockImporterFactory;
private final RlpBlockExporterFactory rlpBlockExporterFactory;
private final Supplier<RlpBlockImporter> rlpBlockImporter;
private final Function<BesuController<?>, JsonBlockImporter<?>> jsonBlockImporterFactory;
private final Function<Blockchain, RlpBlockExporter> rlpBlockExporterFactory;
private final PrintStream out;
public BlocksSubCommand(
final RlpBlockImporter rlpBlockImporter,
final JsonBlockImporterFactory jsonBlockImporterFactory,
final RlpBlockExporterFactory rlpBlockExporterFactory,
final Supplier<RlpBlockImporter> rlpBlockImporter,
final Function<BesuController<?>, JsonBlockImporter<?>> jsonBlockImporterFactory,
final Function<Blockchain, RlpBlockExporter> rlpBlockExporterFactory,
final PrintStream out) {
this.rlpBlockImporter = rlpBlockImporter;
this.rlpBlockExporterFactory = rlpBlockExporterFactory;
@ -116,13 +120,18 @@ public class BlocksSubCommand implements Runnable {
@ParentCommand
private BlocksSubCommand parentCommand; // Picocli injects reference to parent command
@Parameters(
paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP,
description = "Files containing blocks to import.",
arity = "0..*")
private final List<Path> blockImportFiles = new ArrayList<>();
@Option(
names = "--from",
required = true,
paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP,
description = "File containing blocks to import.",
arity = "1..1")
private final File blocksImportFile = null;
arity = "0..*")
private final List<Path> blockImportFileOption = blockImportFiles;
@Option(
names = "--format",
@ -150,37 +159,44 @@ public class BlocksSubCommand implements Runnable {
@Override
public void run() {
parentCommand.parentCommand.configureLogging(false);
LOG.info("Import {} block data from {}", format, blocksImportFile);
checkCommand(parentCommand);
checkNotNull(parentCommand.rlpBlockImporter);
checkNotNull(parentCommand.jsonBlockImporterFactory);
if (blockImportFileOption.isEmpty()) {
throw new ParameterException(spec.commandLine(), "No files specified to import.");
}
LOG.info("Import {} block data from {} files", format, blockImportFiles.size());
final Optional<MetricsService> metricsService = initMetrics(parentCommand);
try {
// As blocksImportFile even if initialized as null is injected by PicoCLI and param is
// mandatory. So we are sure it's always not null, we can remove the warning.
//noinspection ConstantConditions
final Path path = blocksImportFile.toPath();
final BesuController<?> controller = createController();
switch (format) {
case RLP:
importRlpBlocks(controller, path);
break;
case JSON:
importJsonBlocks(controller, path);
break;
default:
throw new ParameterException(
spec.commandLine(), "Unsupported format: " + format.toString());
try (final BesuController<?> controller = createController()) {
for (final Path path : blockImportFiles) {
try {
LOG.info("Importing from {}", path);
switch (format) {
case RLP:
importRlpBlocks(controller, path);
break;
case JSON:
importJsonBlocks(controller, path);
break;
}
} catch (final FileNotFoundException e) {
if (blockImportFiles.size() == 1) {
throw new ExecutionException(
spec.commandLine(), "Could not find file to import: " + path);
} else {
LOG.error("Could not find file to import: {}", path);
}
} catch (final Exception e) {
if (blockImportFiles.size() == 1) {
throw new ExecutionException(
spec.commandLine(), "Unable to import blocks from " + path, e);
} else {
LOG.error("Unable to import blocks from " + path, e);
}
}
}
} catch (final FileNotFoundException e) {
throw new ExecutionException(
spec.commandLine(), "Could not find file to import: " + blocksImportFile);
} catch (final IOException e) {
throw new ExecutionException(
spec.commandLine(), "Unable to import blocks from " + blocksImportFile, e);
} finally {
metricsService.ifPresent(MetricsService::stop);
}
@ -224,17 +240,20 @@ public class BlocksSubCommand implements Runnable {
0.0);
}
private <T> void importJsonBlocks(final BesuController<T> controller, final Path path)
private void importJsonBlocks(final BesuController<?> controller, final Path path)
throws IOException {
final JsonBlockImporter<T> importer = parentCommand.jsonBlockImporterFactory.get(controller);
final JsonBlockImporter<?> importer =
parentCommand.jsonBlockImporterFactory.apply(controller);
final String jsonData = Files.readString(path);
importer.importChain(jsonData);
}
private <T> void importRlpBlocks(final BesuController<T> controller, final Path path)
private void importRlpBlocks(final BesuController<?> controller, final Path path)
throws IOException {
parentCommand.rlpBlockImporter.importBlockchain(path, controller, skipPow);
try (final RlpBlockImporter rlpBlockImporter = parentCommand.rlpBlockImporter.get()) {
rlpBlockImporter.importBlockchain(path, controller, skipPow);
}
}
}
@ -318,7 +337,7 @@ public class BlocksSubCommand implements Runnable {
private void exportRlpFormat(final BesuController<?> controller) throws IOException {
final ProtocolContext<?> context = controller.getProtocolContext();
final RlpBlockExporter exporter =
parentCommand.rlpBlockExporterFactory.get(context.getBlockchain());
parentCommand.rlpBlockExporterFactory.apply(context.getBlockchain());
exporter.exportBlocks(blocksExportFile, getStartBlock(), getEndBlock());
}
@ -402,14 +421,4 @@ public class BlocksSubCommand implements Runnable {
}
return metricsService;
}
@FunctionalInterface
public interface JsonBlockImporterFactory {
<T> JsonBlockImporter<T> get(BesuController<T> controller);
}
@FunctionalInterface
public interface RlpBlockExporterFactory {
RlpBlockExporter get(Blockchain blockchain);
}
}

@ -35,7 +35,6 @@ import org.hyperledger.besu.cli.options.MetricsCLIOptions;
import org.hyperledger.besu.cli.options.NetworkingOptions;
import org.hyperledger.besu.cli.options.SynchronizerOptions;
import org.hyperledger.besu.cli.options.TransactionPoolOptions;
import org.hyperledger.besu.cli.subcommands.blocks.BlocksSubCommand;
import org.hyperledger.besu.controller.BesuController;
import org.hyperledger.besu.controller.BesuControllerBuilder;
import org.hyperledger.besu.controller.NoopPluginServiceFactory;
@ -45,6 +44,7 @@ import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration;
import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration;
import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration;
import org.hyperledger.besu.ethereum.chain.Blockchain;
import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager;
import org.hyperledger.besu.ethereum.eth.sync.BlockBroadcaster;
import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
@ -76,6 +76,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.function.Supplier;
import io.vertx.core.Vertx;
import io.vertx.core.VertxOptions;
@ -289,7 +291,7 @@ public abstract class CommandTestAbstract {
mockLogger,
nodeKey,
keyPair,
rlpBlockImporter,
() -> rlpBlockImporter,
this::jsonBlockImporterFactory,
(blockchain) -> rlpBlockExporter,
mockRunnerBuilder,
@ -323,9 +325,9 @@ public abstract class CommandTestAbstract {
final Logger mockLogger,
final NodeKey mockNodeKey,
final SECP256K1.KeyPair keyPair,
final RlpBlockImporter mockBlockImporter,
final BlocksSubCommand.JsonBlockImporterFactory jsonBlockImporterFactory,
final BlocksSubCommand.RlpBlockExporterFactory rlpBlockExporterFactory,
final Supplier<RlpBlockImporter> mockBlockImporter,
final Function<BesuController<?>, JsonBlockImporter<?>> jsonBlockImporterFactory,
final Function<Blockchain, RlpBlockExporter> rlpBlockExporterFactory,
final RunnerBuilder mockRunnerBuilder,
final BesuController.Builder controllerBuilderFactory,
final BesuPluginContextImpl besuPluginContext,

@ -58,20 +58,21 @@ public class BlocksSubCommandTest extends CommandTestAbstract {
private static final String EXPECTED_BLOCK_IMPORT_USAGE =
"Usage: besu blocks import [-hV] [--skip-pow-validation-enabled]\n"
+ " [--format=<format>] --from=<FILE>\n"
+ " [--start-time=<startTime>]\n"
+ " [--format=<format>] [--start-time=<startTime>] [--from\n"
+ " [=<FILE>...]]... [<FILE>...]\n"
+ "This command imports blocks from a file into the database.\n"
+ " --format=<format> The type of data to be imported, possible values are:\n"
+ " RLP, JSON (default: RLP).\n"
+ " --from=<FILE> File containing blocks to import.\n"
+ " -h, --help Show this help message and exit.\n"
+ " [<FILE>...] Files containing blocks to import.\n"
+ " --format=<format> The type of data to be imported, possible values\n"
+ " are: RLP, JSON (default: RLP).\n"
+ " --from[=<FILE>...] File containing blocks to import.\n"
+ " -h, --help Show this help message and exit.\n"
+ " --skip-pow-validation-enabled\n"
+ " Skip proof of work validation when importing.\n"
+ " Skip proof of work validation when importing.\n"
+ " --start-time=<startTime>\n"
+ " The timestamp in seconds of the first block for JSON\n"
+ " imports. Subsequent blocks will be 1 second later.\n"
+ " (default: current time)\n"
+ " -V, --version Print version information and exit.\n";
+ " The timestamp in seconds of the first block for JSON\n"
+ " imports. Subsequent blocks will be 1 second later.\n"
+ " (default: current time)\n"
+ " -V, --version Print version information and exit.\n";
private static final String EXPECTED_BLOCK_EXPORT_USAGE =
"Usage: besu blocks export [-hV] [--end-block=<LONG>] [--start-block=<LONG>]"
@ -130,7 +131,7 @@ public class BlocksSubCommandTest extends CommandTestAbstract {
@Test
public void callingBlockImportSubCommandWithoutPathMustDisplayErrorAndUsage() {
parseCommand(BLOCK_SUBCOMMAND_NAME, BLOCK_IMPORT_SUBCOMMAND_NAME);
final String expectedErrorOutputStart = "Missing required option '--from=<FILE>'";
final String expectedErrorOutputStart = "No files specified to import.";
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).startsWith(expectedErrorOutputStart);
}
@ -175,6 +176,31 @@ public class BlocksSubCommandTest extends CommandTestAbstract {
assertThat(commandErrorOutput.toString()).isEmpty();
}
@Test
public void blocksImport_rlpFormatMultiple() throws Exception {
final File fileToImport = temp.newFile("blocks.file");
final File file2ToImport = temp.newFile("blocks2.file");
final File file3ToImport = temp.newFile("blocks3.file");
parseCommand(
BLOCK_SUBCOMMAND_NAME,
BLOCK_IMPORT_SUBCOMMAND_NAME,
"--format",
"RLP",
fileToImport.getPath(),
file2ToImport.getPath(),
file3ToImport.getPath());
verify(rlpBlockImporter, times(3))
.importBlockchain(pathArgumentCaptor.capture(), any(), anyBoolean());
assertThat(pathArgumentCaptor.getAllValues())
.containsExactlyInAnyOrder(
fileToImport.toPath(), file2ToImport.toPath(), file3ToImport.toPath());
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).isEmpty();
}
@Test
public void blocksImport_jsonFormat() throws Exception {
final String fileContent = "test";

Loading…
Cancel
Save