Database Migration acceptance testing - using pre-generated database archives (#430)

Signed-off-by: Abdelhamid Bakhta <abdelhamid.bakhta@consensys.net>

.circleci/config.yml

@@ -244,4 +244,3 @@ workflows:
             - acceptanceTests
             - referenceTests
             - buildDocker
-

.gitattributes

@@ -2,6 +2,9 @@
 *.jar -text
 *.bat -text
 *.pcap binary
+*.zip binary
+*.tar binary
+*.tar.gz binary
 *.blocks binary
 *.eot binary
 *.svg binary

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/account/Account.java

@@ -16,16 +16,19 @@ package org.hyperledger.besu.tests.acceptance.dsl.account;
 
 import org.hyperledger.besu.crypto.SECP256K1.KeyPair;
 import org.hyperledger.besu.crypto.SECP256K1.PrivateKey;
+import org.hyperledger.besu.crypto.SECP256K1.PublicKey;
 import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.ethereum.core.Hash;
 import org.hyperledger.besu.tests.acceptance.dsl.blockchain.Amount;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.account.ExpectAccountBalance;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.account.ExpectAccountBalanceAtBlock;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.account.ExpectAccountBalanceNotChanging;
 import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthTransactions;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.util.Optional;
 
 import org.apache.tuweni.bytes.Bytes32;
 import org.web3j.crypto.Credentials;
@@ -35,15 +38,35 @@ public class Account {
 
   private final EthTransactions eth;
   private final String name;
-  private final KeyPair keyPair;
+  private final Optional<PrivateKey> privateKey;
+  private final Optional<PublicKey> publicKey;
+  private final Address address;
   private long nonce = 0;
 
-  private Account(final EthTransactions eth, final String name, final KeyPair keyPair) {
+  private Account(
+      final EthTransactions eth,
+      final String name,
+      final Address address,
+      final Optional<KeyPair> keyPair) {
     this.name = name;
-    this.keyPair = keyPair;
+    this.privateKey = keyPair.map(KeyPair::getPrivateKey);
+    this.publicKey = keyPair.map(KeyPair::getPublicKey);
+    this.address = address;
     this.eth = eth;
   }
 
+  private Account(final EthTransactions eth, final String name, final KeyPair keyPair) {
+    this(
+        eth,
+        name,
+        Address.extract(Hash.hash(keyPair.getPublicKey().getEncodedBytes())),
+        Optional.of(keyPair));
+  }
+
+  public static Account create(final EthTransactions eth, final Address address) {
+    return new Account(eth, address.toString(), address, Optional.empty());
+  }
+
   public static Account create(final EthTransactions eth, final String name) {
     return new Account(eth, name, KeyPair.generate());
   }
@@ -54,9 +77,16 @@ public class Account {
         eth, name, KeyPair.create(PrivateKey.create(Bytes32.fromHexString(privateKey))));
   }
 
-  public Credentials web3jCredentials() {
+  public Optional<Credentials> web3jCredentials() {
-    return Credentials.create(
+    if (!publicKey.isPresent() || !privateKey.isPresent()) {
-        keyPair.getPrivateKey().toString(), keyPair.getPublicKey().toString());
+      return Optional.empty();
+    }
+    return Optional.of(Credentials.create(privateKey.get().toString(), publicKey.get().toString()));
+  }
+
+  public Credentials web3jCredentialsOrThrow() {
+    return web3jCredentials()
+        .orElseThrow(() -> new IllegalStateException("Account is missing required signing key."));
   }
 
   public BigInteger getNextNonce() {
@@ -64,7 +94,7 @@ public class Account {
   }
 
   public String getAddress() {
-    return Address.extract(Hash.hash(keyPair.getPublicKey().getEncodedBytes())).toString();
+    return address.toString();
   }
 
   public Condition balanceEquals(final int expectedBalance) {
@@ -76,6 +106,10 @@ public class Account {
         eth, this, expectedBalance.getValue(), expectedBalance.getUnit());
   }
 
+  public Condition balanceAtBlockEquals(final Amount expectedBalance, final BigInteger block) {
+    return new ExpectAccountBalanceAtBlock(eth, this, block, expectedBalance.getValue(), Unit.WEI);
+  }
+
   public Condition balanceDoesNotChange(final int startingBalance) {
     return new ExpectAccountBalanceNotChanging(
         eth, this, BigDecimal.valueOf(startingBalance), Unit.ETHER);
@@ -89,8 +123,8 @@ public class Account {
         + ", name='"
         + name
         + '\''
-        + ", keyPair="
+        + ", address="
-        + keyPair
+        + address
         + ", nonce="
         + nonce
         + '}';

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/account/Accounts.java

@@ -14,6 +14,7 @@
  */
 package org.hyperledger.besu.tests.acceptance.dsl.account;
 
+import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthTransactions;
 
 public class Accounts {
@@ -47,4 +48,8 @@ public class Accounts {
   public Account createAccount(final String accountName) {
     return Account.create(eth, accountName);
   }
+
+  public Account createAccount(final Address address) {
+    return Account.create(eth, address);
+  }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/blockchain/Blockchain.java

@@ -16,6 +16,7 @@ package org.hyperledger.besu.tests.acceptance.dsl.blockchain;
 
 import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.blockchain.ExpectBeneficiary;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.blockchain.ExpectBlockNumber;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.blockchain.ExpectBlockNumberAbove;
 import org.hyperledger.besu.tests.acceptance.dsl.condition.blockchain.ExpectMinimumBlockNumber;
 import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode;
@@ -53,6 +54,10 @@ public class Blockchain {
     return new ExpectBlockNumberAbove(eth, futureHeight(node, blocksAheadOfLatest), timeout);
   }
 
+  public Condition currentHeight(final long blockNumber) {
+    return new ExpectBlockNumber(eth, BigInteger.valueOf(blockNumber));
+  }
+
   private BigInteger futureHeight(final Node node, final int blocksAheadOfLatest) {
     return currentHeight(node).add(BigInteger.valueOf(blocksAheadOfLatest));
   }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/account/ExpectAccountBalanceAtBlock.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance.dsl.condition.account;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.web3j.utils.Convert.toWei;
+
+import org.hyperledger.besu.tests.acceptance.dsl.WaitUtils;
+import org.hyperledger.besu.tests.acceptance.dsl.account.Account;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition;
+import org.hyperledger.besu.tests.acceptance.dsl.node.Node;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthTransactions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import org.web3j.utils.Convert.Unit;
+
+public class ExpectAccountBalanceAtBlock implements Condition {
+
+  private final EthTransactions eth;
+  private final Account account;
+  private final BigInteger block;
+  private final BigInteger expectedBalance;
+
+  public ExpectAccountBalanceAtBlock(
+      final EthTransactions eth,
+      final Account account,
+      final BigInteger block,
+      final BigDecimal expectedBalance,
+      final Unit balanceUnit) {
+    this.account = account;
+    this.eth = eth;
+    this.block = block;
+    this.expectedBalance = toWei(expectedBalance, balanceUnit).toBigIntegerExact();
+  }
+
+  @Override
+  public void verify(final Node node) {
+    WaitUtils.waitFor(
+        () ->
+            assertThat(node.execute(eth.getBalanceAtBlock(account, block)))
+                .isEqualTo(expectedBalance));
+  }
+}

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/blockchain/ExpectBlockNumber.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance.dsl.condition.blockchain;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition;
+import org.hyperledger.besu.tests.acceptance.dsl.node.Node;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthTransactions;
+
+import java.math.BigInteger;
+
+public class ExpectBlockNumber implements Condition {
+
+  private final EthTransactions eth;
+  private final BigInteger blockNumber;
+
+  public ExpectBlockNumber(final EthTransactions eth, final BigInteger blockNumber) {
+    this.blockNumber = blockNumber;
+    this.eth = eth;
+  }
+
+  @Override
+  public void verify(final Node node) {
+    assertThat(node.execute(eth.blockNumber())).isEqualTo(blockNumber);
+  }
+}

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java

@@ -108,6 +108,7 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable
 
   public BesuNode(
       final String name,
+      final Optional<Path> dataPath,
       final MiningParameters miningParameters,
       final JsonRpcConfiguration jsonRpcConfiguration,
       final WebSocketConfiguration webSocketConfiguration,
@@ -128,7 +129,7 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable
       throws IOException {
     this.bootnodeEligible = bootnodeEligible;
     this.revertReasonEnabled = revertReasonEnabled;
-    this.homeDirectory = Files.createTempDirectory("acctest");
+    this.homeDirectory = dataPath.orElseGet(BesuNode::createTmpDataDirectory);
     keyfilePath.ifPresent(
         path -> {
           try {
@@ -166,6 +167,14 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable
     LOG.info("Created BesuNode {}", this.toString());
   }
 
+  private static Path createTmpDataDirectory() {
+    try {
+      return Files.createTempDirectory("acctest");
+    } catch (final IOException e) {
+      throw new RuntimeException("Unable to create temporary data directory", e);
+    }
+  }
+
   @Override
   public boolean isJsonRpcEnabled() {
     return jsonRpcConfiguration().isEnabled();

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java

@@ -23,12 +23,14 @@ import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationProvider;
 
+import java.nio.file.Path;
 import java.util.List;
 import java.util.Optional;
 
 public class BesuNodeConfiguration {
 
   private final String name;
+  private final Optional<Path> dataPath;
   private final MiningParameters miningParameters;
   private final JsonRpcConfiguration jsonRpcConfiguration;
   private final WebSocketConfiguration webSocketConfiguration;
@@ -47,8 +49,9 @@ public class BesuNodeConfiguration {
   private final List<String> staticNodes;
   private final Optional<PrivacyParameters> privacyParameters;
 
-  public BesuNodeConfiguration(
+  BesuNodeConfiguration(
       final String name,
+      final Optional<Path> dataPath,
       final MiningParameters miningParameters,
       final JsonRpcConfiguration jsonRpcConfiguration,
       final WebSocketConfiguration webSocketConfiguration,
@@ -73,6 +76,7 @@ public class BesuNodeConfiguration {
     this.metricsConfiguration = metricsConfiguration;
     this.permissioningConfiguration = permissioningConfiguration;
     this.keyFilePath = keyFilePath;
+    this.dataPath = dataPath;
     this.devMode = devMode;
     this.genesisConfigProvider = genesisConfigProvider;
     this.p2pEnabled = p2pEnabled;
@@ -114,6 +118,10 @@ public class BesuNodeConfiguration {
     return keyFilePath;
   }
 
+  public Optional<Path> getDataPath() {
+    return dataPath;
+  }
+
   public boolean isDevMode() {
     return devMode;
   }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java

@@ -14,6 +14,7 @@
  */
 package org.hyperledger.besu.tests.acceptance.dsl.node.configuration;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Collections.singletonList;
 
 import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration;
@@ -29,6 +30,7 @@ import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.Gene
 
 import java.io.File;
 import java.net.URISyntaxException;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -38,6 +40,7 @@ import java.util.Optional;
 public class BesuNodeConfigurationBuilder {
 
   private String name;
+  private Optional<Path> dataPath = Optional.empty();
   private MiningParameters miningParameters =
       new MiningParametersTestBuilder().enabled(false).build();
   private JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault();
@@ -68,6 +71,12 @@ public class BesuNodeConfigurationBuilder {
     return this;
   }
 
+  public BesuNodeConfigurationBuilder dataPath(final Path dataPath) {
+    checkNotNull(dataPath);
+    this.dataPath = Optional.of(dataPath);
+    return this;
+  }
+
   public BesuNodeConfigurationBuilder miningEnabled() {
     this.miningParameters = new MiningParametersTestBuilder().enabled(true).build();
     this.jsonRpcConfiguration.addRpcApi(RpcApis.MINER);
@@ -249,6 +258,7 @@ public class BesuNodeConfigurationBuilder {
   public BesuNodeConfiguration build() {
     return new BesuNodeConfiguration(
         name,
+        dataPath,
         miningParameters,
         jsonRpcConfiguration,
         webSocketConfiguration,

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -37,6 +37,7 @@ import java.net.URISyntaxException;
 import java.nio.file.Paths;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Function;
 
 import io.vertx.core.Vertx;
 
@@ -48,6 +49,7 @@ public class BesuNodeFactory {
   public BesuNode create(final BesuNodeConfiguration config) throws IOException {
     return new BesuNode(
         config.getName(),
+        config.getDataPath(),
         config.getMiningParameters(),
         config.getJsonRpcConfiguration(),
         config.getWebSocketConfiguration(),
@@ -93,6 +95,15 @@ public class BesuNodeFactory {
         new BesuNodeConfigurationBuilder().name(name).jsonRpcEnabled().webSocketEnabled().build());
   }
 
+  public BesuNode createNode(
+      final String name,
+      final Function<BesuNodeConfigurationBuilder, BesuNodeConfigurationBuilder> configModifier)
+      throws IOException {
+    final BesuNodeConfigurationBuilder configBuilder =
+        configModifier.apply(new BesuNodeConfigurationBuilder().name(name));
+    return create(configBuilder.build());
+  }
+
   public Node createArchiveNodeThatMustNotBeTheBootnode(final String name) throws IOException {
     return create(
         new BesuNodeConfigurationBuilder()

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java

@@ -84,6 +84,7 @@ public class PrivacyNode implements AutoCloseable {
     this.besu =
         new BesuNode(
             besuConfig.getName(),
+            besuConfig.getDataPath(),
             besuConfig.getMiningParameters(),
             besuConfig.getJsonRpcConfiguration(),
             besuConfig.getWebSocketConfiguration(),

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/account/TransferTransaction.java

@@ -87,7 +87,8 @@ public class TransferTransaction implements Transaction<Hash> {
             recipient.getAddress(),
             Convert.toWei(transferAmount, transferUnit).toBigIntegerExact());
 
-    return toHexString(TransactionEncoder.signMessage(transaction, sender.web3jCredentials()));
+    return toHexString(
+        TransactionEncoder.signMessage(transaction, sender.web3jCredentialsOrThrow()));
   }
 
   private Optional<BigInteger> getNonce() {

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthGetBalanceAtBlockTransaction.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance.dsl.transaction.eth;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.tests.acceptance.dsl.account.Account;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.NodeRequests;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.Transaction;
+
+import java.io.IOException;
+import java.math.BigInteger;
+
+import org.web3j.protocol.core.DefaultBlockParameter;
+import org.web3j.protocol.core.methods.response.EthGetBalance;
+
+public class EthGetBalanceAtBlockTransaction implements Transaction<BigInteger> {
+
+  private final Account account;
+  private final BigInteger block;
+
+  EthGetBalanceAtBlockTransaction(final Account account, final BigInteger block) {
+    this.account = account;
+    this.block = block;
+  }
+
+  @Override
+  public BigInteger execute(final NodeRequests node) {
+    try {
+      final EthGetBalance result =
+          node.eth()
+              .ethGetBalance(account.getAddress(), DefaultBlockParameter.valueOf(block))
+              .send();
+      assertThat(result).isNotNull();
+      assertThat(result.hasError()).isFalse();
+
+      return result.getBalance();
+
+    } catch (final IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthTransactions.java

@@ -43,6 +43,11 @@ public class EthTransactions {
     return new EthGetBalanceTransaction(account);
   }
 
+  public EthGetBalanceAtBlockTransaction getBalanceAtBlock(
+      final Account account, final BigInteger block) {
+    return new EthGetBalanceAtBlockTransaction(account, block);
+  }
+
   public EthAccountsTransaction accounts() {
     return new EthAccountsTransaction();
   }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/perm/AccountSmartContractPermissioningAllowAccountTransaction.java

@@ -72,6 +72,7 @@ public class AccountSmartContractPermissioningAllowAccountTransaction implements
             contractAddress.toString(),
             payload.toString());
 
-    return toHexString(TransactionEncoder.signMessage(transaction, sender.web3jCredentials()));
+    return toHexString(
+        TransactionEncoder.signMessage(transaction, sender.web3jCredentialsOrThrow()));
   }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/perm/AccountSmartContractPermissioningForbidAccountTransaction.java

@@ -73,6 +73,7 @@ public class AccountSmartContractPermissioningForbidAccountTransaction
             contractAddress.toString(),
             payload.toString());
 
-    return toHexString(TransactionEncoder.signMessage(transaction, sender.web3jCredentials()));
+    return toHexString(
+        TransactionEncoder.signMessage(transaction, sender.web3jCredentialsOrThrow()));
   }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/perm/NodeSmartContractPermissioningAllowNodeTransaction.java

@@ -78,6 +78,7 @@ public class NodeSmartContractPermissioningAllowNodeTransaction implements Trans
             contractAddress.toString(),
             payload.toString());
 
-    return toHexString(TransactionEncoder.signMessage(transaction, sender.web3jCredentials()));
+    return toHexString(
+        TransactionEncoder.signMessage(transaction, sender.web3jCredentialsOrThrow()));
   }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/perm/NodeSmartContractPermissioningForbidNodeTransaction.java

@@ -78,6 +78,7 @@ public class NodeSmartContractPermissioningForbidNodeTransaction implements Tran
             contractAddress.toString(),
             payload.toString());
 
-    return toHexString(TransactionEncoder.signMessage(transaction, sender.web3jCredentials()));
+    return toHexString(
+        TransactionEncoder.signMessage(transaction, sender.web3jCredentialsOrThrow()));
   }
 }

acceptance-tests/tests/build.gradle

@@ -13,6 +13,12 @@
 
 dependencies {
   testImplementation project(':acceptance-tests:dsl')
+  testImplementation project(':enclave')
+  testImplementation 'org.awaitility:awaitility'
+  testImplementation project(':consensus:clique')
+  testImplementation project(':ethereum:permissioning')
+  testImplementation project(':util')
+  testImplementation project(':plugin-api')
   testImplementation project(':besu')
   testImplementation project(':consensus:clique')
   implementation project(':crypto')
@@ -36,6 +42,11 @@ dependencies {
   testImplementation 'org.web3j:besu'
   testImplementation 'tech.pegasys.ethsigner.internal:core'
   testImplementation 'tech.pegasys.ethsigner.internal:file-based'
+  testImplementation 'org.apache.commons:commons-compress'
+  testImplementation 'commons-io:commons-io'
+  compile('javax.activation:activation'){
+    force = true
+  }
 
   testCompile "com.github.tomakehurst:wiremock-jre8-standalone:2.25.1"
 }

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/database/DatabaseMigrationAcceptanceTest.java

@@ -0,0 +1,206 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.hyperledger.besu.tests.acceptance.database;
+
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
+import org.hyperledger.besu.ethereum.core.Address;
+import org.hyperledger.besu.ethereum.core.Wei;
+import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase;
+import org.hyperledger.besu.tests.acceptance.dsl.blockchain.Amount;
+import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode;
+import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.BesuNodeConfigurationBuilder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class DatabaseMigrationAcceptanceTest extends AcceptanceTestBase {
+  private final String testName;
+  private final String dataPath;
+  private final long expectedChainHeight;
+  private Path hostDataPath;
+  private BesuNode node;
+  private final List<AccountData> testAccounts;
+
+  public DatabaseMigrationAcceptanceTest(
+      final String testName,
+      final String dataPath,
+      final long expectedChainHeight,
+      final List<AccountData> testAccounts) {
+    this.testName = testName;
+    this.dataPath = dataPath;
+    this.expectedChainHeight = expectedChainHeight;
+    this.testAccounts = testAccounts;
+  }
+
+  @Parameters(name = "{0}")
+  public static Object[][] getParameters() {
+    return new Object[][] {
+      // First 10 blocks of ropsten
+      new Object[] {
+        "Before versioning was enabled",
+        "version0",
+        0xA,
+        Arrays.asList(
+            new AccountData(
+                "0xd1aeb42885a43b72b518182ef893125814811048",
+                BigInteger.valueOf(0xA),
+                Wei.fromHexString("0x2B5E3AF16B1880000"))),
+      },
+      new Object[] {
+        "After versioning was enabled and using multiple RocksDB columns",
+        "version1",
+        0xA,
+        Arrays.asList(
+            new AccountData(
+                "0xd1aeb42885a43b72b518182ef893125814811048",
+                BigInteger.valueOf(0xA),
+                Wei.fromHexString("0x2B5E3AF16B1880000")))
+      }
+    };
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    final URL rootURL = DatabaseMigrationAcceptanceTest.class.getResource(dataPath);
+    hostDataPath = copyDataDir(rootURL);
+    final Path databaseArchive =
+        Paths.get(
+            DatabaseMigrationAcceptanceTest.class
+                .getResource(String.format("%s/besu-db-archive.tar.gz", dataPath))
+                .toURI());
+    extract(databaseArchive, hostDataPath.toAbsolutePath().toString());
+    node = besu.createNode(testName, this::configureNode);
+    cluster.start(node);
+  }
+
+  private BesuNodeConfigurationBuilder configureNode(
+      final BesuNodeConfigurationBuilder nodeBuilder) {
+    final String genesisData = getGenesisConfiguration();
+    return nodeBuilder
+        .devMode(false)
+        .dataPath(hostDataPath)
+        .genesisConfigProvider((nodes) -> Optional.of(genesisData))
+        .jsonRpcEnabled();
+  }
+
+  private String getGenesisConfiguration() {
+    try {
+      return Resources.toString(
+          hostDataPath.resolve("genesis.json").toUri().toURL(), Charsets.UTF_8);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Test
+  public void shouldReturnCorrectBlockHeight() {
+    blockchain.currentHeight(expectedChainHeight).verify(node);
+  }
+
+  @Test
+  public void shouldReturnCorrectAccountBalance() {
+    testAccounts.forEach(
+        accountData ->
+            accounts
+                .createAccount(Address.fromHexString(accountData.accountAddress))
+                .balanceAtBlockEquals(
+                    Amount.wei(accountData.expectedBalance.toBigInteger()), accountData.block)
+                .verify(node));
+  }
+
+  private static void extract(final Path path, final String destDirectory) throws IOException {
+    try (TarArchiveInputStream fin =
+        new TarArchiveInputStream(
+            new GzipCompressorInputStream(new FileInputStream(path.toAbsolutePath().toString())))) {
+      TarArchiveEntry entry;
+      while ((entry = fin.getNextTarEntry()) != null) {
+        if (entry.isDirectory()) {
+          continue;
+        }
+        final File curfile = new File(destDirectory, entry.getName());
+        final File parent = curfile.getParentFile();
+        if (!parent.exists()) {
+          parent.mkdirs();
+        }
+        IOUtils.copy(fin, new FileOutputStream(curfile));
+      }
+    }
+  }
+
+  private Path copyDataDir(final URL url) {
+    if (url == null) {
+      throw new RuntimeException("Unable to locate resource.");
+    }
+
+    try {
+      final Path tmpDir = Files.createTempDirectory("data");
+      Files.delete(tmpDir);
+      final Path toCopy = Paths.get(url.toURI());
+      try (final Stream<Path> pathStream = Files.walk(toCopy)) {
+        pathStream.forEach(source -> copy(source, tmpDir.resolve(toCopy.relativize(source))));
+        return tmpDir.toAbsolutePath();
+      }
+    } catch (URISyntaxException | IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void copy(final Path source, final Path dest) {
+    try {
+      Files.copy(source, dest, REPLACE_EXISTING);
+    } catch (Exception e) {
+      throw new RuntimeException(e.getMessage(), e);
+    }
+  }
+
+  private static class AccountData {
+    private final String accountAddress;
+    private final BigInteger block;
+    private final Wei expectedBalance;
+
+    private AccountData(final String account, final BigInteger block, final Wei expectedBalance) {
+      this.accountAddress = account;
+      this.block = block;
+      this.expectedBalance = expectedBalance;
+    }
+  }
+}

gradle/check-licenses.gradle

@@ -147,6 +147,7 @@ downloadLicenses {
       (group('org.javassist')): apache,
       /// Explicilitly declare Apache 2.0 license for javassist
       (group('javax.ws.rs')): cddl1_1,
+      (group('javax.activation')): cddl1_1,
       (group('org.glassfish.jersey.core')): apache,
       (group('org.glassfish.jersey.bundles.repackaged')): apache,
       (group('org.glassfish.jersey.connectors')): apache

gradle/versions.gradle

@@ -17,9 +17,6 @@ dependencyManagement {
   dependencies {
     dependency 'com.fasterxml.jackson.core:jackson-databind:2.10.1'
     dependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.10.1'
-
-    dependency 'com.github.docker-java:docker-java:3.0.14'
-
     dependency 'com.github.tomakehurst:wiremock-jre8:2.25.1'
 
     dependency 'com.google.auto.service:auto-service:1.0-rc6'
@@ -107,5 +104,9 @@ dependencyManagement {
     dependency 'tech.pegasys.ethsigner.internal:core:0.4.0'
     dependency 'tech.pegasys.ethsigner.internal:file-based:0.4.0'
     dependency 'tech.pegasys.ethsigner.internal:signing-api:0.4.0'
+    dependency 'javax.activation:activation:1.1.1'
+    dependency 'org.apache.commons:commons-compress:1.20'
+    dependency 'commons-io:commons-io:2.6'
+
   }
 }