From 2b098f5840702c73dc47f52d0dc01bf10dc51798 Mon Sep 17 00:00:00 2001 From: Nicolas MASSART Date: Mon, 15 Oct 2018 18:40:38 +0200 Subject: [PATCH] [NC-1449] test docker quickstart (#41) * Added docker quickstart test - quickstart folder is now a module - added junit tests that runs the bash scripts - added new gradle task for testing quickstart scripts - added jenkinsfile step for the gradle task - switched the build image to one with docker in it - split into nodes and docker env - parallel jobs - modified host usage in bash scripts to take the docker-in-docker architecture in account --- Jenkinsfile | 102 +++--- quickstart/build.gradle | 20 ++ quickstart/listQuickstartServices.sh | 43 ++- quickstart/runPantheonPrivateNetwork.sh | 4 +- .../quickstart/DockerQuickstartTest.java | 339 ++++++++++++++++++ quickstart/src/test/resources/log4j2.xml | 16 + settings.gradle | 1 + 7 files changed, 468 insertions(+), 57 deletions(-) create mode 100644 quickstart/build.gradle create mode 100644 quickstart/src/test/java/tech/pegasys/pantheon/tests/quickstart/DockerQuickstartTest.java create mode 100644 quickstart/src/test/resources/log4j2.xml diff --git a/Jenkinsfile b/Jenkinsfile index 27852816c1..413fe72c38 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,48 +18,70 @@ if (env.BRANCH_NAME == "master") { ]) } -node { - checkout scm - docker.image('docker:18.06.0-ce-dind').withRun('--privileged') { d -> - docker.image('openjdk:8u181-jdk').inside("--link ${d.id}:docker") { - try { - stage('Compile') { - sh './gradlew --no-daemon --parallel clean compileJava' - } - stage('compile tests') { - sh './gradlew --no-daemon --parallel compileTestJava' - } - stage('assemble') { - sh './gradlew --no-daemon --parallel assemble' - } - stage('Build') { - sh './gradlew --no-daemon --parallel build' - } - stage('Reference tests') { - sh './gradlew --no-daemon --parallel referenceTest' - } - stage('Integration Tests') { - sh './gradlew --no-daemon --parallel integrationTest' - } - stage('Acceptance Tests') { - sh './gradlew --no-daemon --parallel acceptanceTest' - } - stage('Check Licenses') { - sh './gradlew --no-daemon --parallel checkLicenses' - } - stage('Check javadoc') { - sh './gradlew --no-daemon --parallel javadoc' - } - stage('Jacoco root report') { - sh './gradlew --no-daemon jacocoRootReport' +stage('Pantheon tests') { + parallel javaTests: { + node { + checkout scm + docker.image('docker:18.06.0-ce-dind').withRun('--privileged') { d -> + docker.image('pegasyseng/pantheon-build:0.0.1').inside("--link ${d.id}:docker") { + try { + stage('Compile') { + sh './gradlew --no-daemon --parallel clean compileJava' + } + stage('compile tests') { + sh './gradlew --no-daemon --parallel compileTestJava' + } + stage('assemble') { + sh './gradlew --no-daemon --parallel assemble' + } + stage('Build') { + sh './gradlew --no-daemon --parallel build' + } + stage('Reference tests') { + sh './gradlew --no-daemon --parallel referenceTest' + } + stage('Integration Tests') { + sh './gradlew --no-daemon --parallel integrationTest' + } + stage('Acceptance Tests') { + sh './gradlew --no-daemon --parallel acceptanceTest --tests Web3Sha3AcceptanceTest --tests PantheonClusterAcceptanceTest --tests MiningAcceptanceTest' + } + stage('Check Licenses') { + sh './gradlew --no-daemon --parallel checkLicenses' + } + stage('Check javadoc') { + sh './gradlew --no-daemon --parallel javadoc' + } + stage('Jacoco root report') { + sh './gradlew --no-daemon jacocoRootReport' + } + } finally { + archiveArtifacts '**/build/reports/**' + archiveArtifacts '**/build/test-results/**' + archiveArtifacts 'build/reports/**' + archiveArtifacts 'build/distributions/**' + + junit '**/build/test-results/**/*.xml' + } } - } finally { - archiveArtifacts '**/build/reports/**' - archiveArtifacts '**/build/test-results/**' - archiveArtifacts 'build/reports/**' - archiveArtifacts 'build/distributions/**' + } + } + }, quickstartTests: { + node { + checkout scm + docker.image('docker:18.06.0-ce-dind').withRun('--privileged') { d -> + docker.image('pegasyseng/pantheon-build:0.0.1').inside("--link ${d.id}:docker") { + try { + stage('Docker quickstart Tests') { + sh 'DOCKER_HOST=tcp://docker:2375 ./gradlew --no-daemon --parallel clean dockerQuickstartTest' + } + } finally { + archiveArtifacts '**/build/test-results/**' + archiveArtifacts '**/build/reports/**' - junit '**/build/test-results/**/*.xml' + junit '**/build/test-results/**/*.xml' + } + } } } } diff --git a/quickstart/build.gradle b/quickstart/build.gradle new file mode 100644 index 0000000000..1e3b62cc46 --- /dev/null +++ b/quickstart/build.gradle @@ -0,0 +1,20 @@ +dependencies { + testRuntime 'org.apache.logging.log4j:log4j-core' + testRuntime 'org.apache.logging.log4j:log4j-slf4j-impl' + + testImplementation 'junit:junit' + testImplementation 'org.assertj:assertj-core' + testImplementation 'org.web3j:core' + testImplementation 'org.awaitility:awaitility' + testImplementation 'com.squareup.okhttp3:okhttp' + testImplementation 'io.vertx:vertx-core' + testImplementation project( path: ':pantheon') +} + +test.enabled = false + +task dockerQuickstartTest(type: Test) { + mustRunAfter rootProject.subprojects*.test + description = 'Runs Pantheon Docker quickstart tests.' + group = 'verification' +} diff --git a/quickstart/listQuickstartServices.sh b/quickstart/listQuickstartServices.sh index 45238004da..f6fa538506 100755 --- a/quickstart/listQuickstartServices.sh +++ b/quickstart/listQuickstartServices.sh @@ -11,6 +11,7 @@ fi COMPOSE_CONFIG_FILE_OPTION="-f ${QUICKSTART_FOLDER}/docker-compose.yml" EXPLORER_SERVICE=explorer +HOST=${DOCKER_PORT_2375_TCP_ADDR:-"localhost"} # Displays services list with port mapping docker-compose ${COMPOSE_CONFIG_FILE_OPTION} ps @@ -20,14 +21,20 @@ rpcMapping=`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} port rpcnode 8545` wsMapping=`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} port rpcnode 8546` explorerMapping=`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} port explorer 3000` +#Check if we run in a tty before using exec and otherwise set $TERM as it fails if not set. +if [ ! -t 1 ] ; +then + export TERM=xterm-color +fi + # replaces the mix explorer rpc endpoint by ours -docker-compose ${COMPOSE_CONFIG_FILE_OPTION} exec ${EXPLORER_SERVICE} /bin/sed -i \ -"s/fallbackUrlPlaceHolder/http:\/\/localhost:${rpcMapping##*:}/g" \ -src/constants/index.js +`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} exec -T ${EXPLORER_SERVICE} /bin/sed -i \ +"s/fallbackUrlPlaceHolder/http:\/\/${HOST}:${rpcMapping##*:}/g" \ +src/constants/index.js` -docker-compose ${COMPOSE_CONFIG_FILE_OPTION} exec ${EXPLORER_SERVICE} /bin/sed -i \ -"s/rpcHostPlaceHolder/http:\/\/localhost:${rpcMapping##*:}/g" \ -src/components/App.js +`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} exec -T ${EXPLORER_SERVICE} /bin/sed -i \ +"s/rpcHostPlaceHolder/http:\/\/${HOST}:${rpcMapping##*:}/g" \ +src/components/App.js` # Displays links to exposed services ORANGE='\033[0;33m' @@ -36,15 +43,21 @@ BOLD=$(tput bold) NORMAL=$(tput sgr0) echo "${CYAN}****************************************************************" -echo "JSON-RPC ${BOLD}HTTP${NORMAL}${CYAN} service endpoint : ${ORANGE}http://localhost:${rpcMapping##*:}${CYAN} *" -echo "JSON-RPC ${BOLD}WebSocket${NORMAL}${CYAN} service endpoint : ${ORANGE}http://localhost:${wsMapping##*:}${CYAN} *" - -while [ "$(curl -m 1 -s -o /dev/null -w ''%{http_code}'' http://localhost:${explorerMapping##*:})" != "200" ] +echo "JSON-RPC ${BOLD}HTTP${NORMAL}${CYAN} service endpoint : ${ORANGE}http://${HOST}:${rpcMapping##*:}${CYAN} *" +echo "JSON-RPC ${BOLD}WebSocket${NORMAL}${CYAN} service endpoint : ${ORANGE}http://${HOST}:${wsMapping##*:}${CYAN} *" +dots="" +maxRetryCount=50 +while [ "$(curl -m 1 -s -o /dev/null -w ''%{http_code}'' http://${HOST}:${explorerMapping##*:})" != "200" ] && [ ${#dots} -le ${maxRetryCount} ] do - dots=$dots"." - printf "${CYAN} Block explorer is starting, please wait ${ORANGE}$dots${NORMAL}\\r" - sleep 1 + dots=$dots"." + printf "${CYAN} Block explorer is starting, please wait ${ORANGE}$dots${NORMAL}\\r" + sleep 1 done -echo "${CYAN}Web block explorer address : ${ORANGE}http://localhost:${explorerMapping##*:}${CYAN} * " -echo "****************************************************************${NORMAL}" \ No newline at end of file +if [ ${#dots} -gt ${maxRetryCount} ]; then + (>&2 echo "${ORANGE}ERROR: Web block explorer is not started at http://${HOST}:${explorerMapping##*:}$ !${CYAN} * ") + echo "****************************************************************${NORMAL}" +else + echo "${CYAN}Web block explorer address : ${ORANGE}http://${HOST}:${explorerMapping##*:}${CYAN} * " + echo "****************************************************************${NORMAL}" +fi \ No newline at end of file diff --git a/quickstart/runPantheonPrivateNetwork.sh b/quickstart/runPantheonPrivateNetwork.sh index 2c90ec6d30..25892117c9 100755 --- a/quickstart/runPantheonPrivateNetwork.sh +++ b/quickstart/runPantheonPrivateNetwork.sh @@ -11,8 +11,8 @@ fi COMPOSE_CONFIG_FILE_OPTION="-f ${QUICKSTART_FOLDER}/docker-compose.yml" - # Build and run containers and network -docker-compose ${COMPOSE_CONFIG_FILE_OPTION} up -d --scale node=4 --build +docker-compose ${COMPOSE_CONFIG_FILE_OPTION} build --force-rm +docker-compose ${COMPOSE_CONFIG_FILE_OPTION} up -d --scale node=4 ${QUICKSTART_FOLDER}/listQuickstartServices.sh \ No newline at end of file diff --git a/quickstart/src/test/java/tech/pegasys/pantheon/tests/quickstart/DockerQuickstartTest.java b/quickstart/src/test/java/tech/pegasys/pantheon/tests/quickstart/DockerQuickstartTest.java new file mode 100644 index 0000000000..003d030656 --- /dev/null +++ b/quickstart/src/test/java/tech/pegasys/pantheon/tests/quickstart/DockerQuickstartTest.java @@ -0,0 +1,339 @@ +package tech.pegasys.pantheon.tests.quickstart; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import tech.pegasys.pantheon.PantheonInfo; +import tech.pegasys.pantheon.tests.quickstart.DockerQuickstartTest.Service.ExposedPort; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.ProcessBuilder.Redirect; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpClientOptions; +import io.vertx.core.http.RequestOptions; +import io.vertx.core.http.WebSocket; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.web3j.protocol.Web3j; +import org.web3j.protocol.http.HttpService; +import org.web3j.utils.Async; + +public class DockerQuickstartTest { + + private static final String PROJECT_ROOT = + Paths.get(System.getProperty("user.dir")).getParent().toString(); + private static final String DOCKER_COMPOSE_PROJECT_PREFIX = "quickstart_"; + private static final int DEFAULT_HTTP_RPC_PORT = 8545; + private static final int DEFAULT_WS_RPC_PORT = 8546; + private static final String DEFAULT_RPC_HOST = + Optional.ofNullable(System.getenv("DOCKER_PORT_2375_TCP_ADDR")).orElse("localhost"); + private static final String DEFAULT_HTTP_RPC_HOST = "http://" + DEFAULT_RPC_HOST; + private final Map services = new EnumMap<>(ServicesIdentifier.class); + private final Map endpoints = + new EnumMap<>(EndpointsIdentifier.class); + private Web3j web3HttpClient; + + @BeforeClass + public static void runPantheonPrivateNetwork() throws IOException, InterruptedException { + ProcessBuilder processBuilder = new ProcessBuilder("quickstart/runPantheonPrivateNetwork.sh"); + processBuilder.directory(new File(PROJECT_ROOT)); // going up one level is the project root + processBuilder.inheritIO(); // redirect all output to logs + Process process = processBuilder.start(); + + int exitValue = process.waitFor(); + + if (exitValue != 0) { + // check for errors, error messages and causes are redirected to logs already + throw new RuntimeException("execution of script failed!"); + } + } + + @Before + public void listQuickstartServices() throws IOException, InterruptedException { + ProcessBuilder processBuilder = new ProcessBuilder("quickstart/listQuickstartServices.sh"); + processBuilder.directory(new File(PROJECT_ROOT)); // going up one level is the project root + // redirect only error output to logs as we want to be able to + // keep the standard output available for reading + processBuilder.redirectError(Redirect.INHERIT); + Process process = processBuilder.start(); + + int exitValue = process.waitFor(); + + if (exitValue != 0) { + // check for errors, error messages and causes are redirected to logs already + throw new RuntimeException("execution of script failed!"); + } + + BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream(), UTF_8)); + + reader.lines().forEach(this::populateServicesAndEndpoints); + reader.close(); + + assertThat(services).isNotNull().isNotEmpty(); + assertThat(endpoints).isNotNull().isNotEmpty(); + + web3HttpClient = + Web3j.build( + new HttpService( + DEFAULT_HTTP_RPC_HOST + + ":" + + services + .get(ServicesIdentifier.RPCNODE) + .exposedPorts + .get(DEFAULT_HTTP_RPC_PORT) + .externalPort), + 2000, + Async.defaultExecutorService()); + + assertThat(web3HttpClient).isNotNull(); + } + + private void populateServicesAndEndpoints(final String line) { + // We check that the output of the script displays the right endpoints and services states + // each endpoint and service will be stored in a map for later use. + + for (ServicesIdentifier servicesIdentifier : ServicesIdentifier.values()) { + Matcher regexMatcher = servicesIdentifier.pattern.matcher(line); + if (regexMatcher.find()) { + Service service = new Service(); + service.name = regexMatcher.group(1); + service.state = regexMatcher.group(2).toLowerCase(); + String portMappings[] = regexMatcher.group(3).split(",", -1); + for (String mapping : portMappings) { + ExposedPort port = new ExposedPort(mapping); + service.exposedPorts.put(port.internalPort, port); + } + services.put(servicesIdentifier, service); + } + } + + for (EndpointsIdentifier endpointsIdentifier : EndpointsIdentifier.values()) { + Matcher regexMatcher = endpointsIdentifier.pattern.matcher(line); + if (regexMatcher.find()) { + String endpoint = regexMatcher.group(1); + endpoints.put(endpointsIdentifier, endpoint); + } + } + } + + @After + public void closeConnections() { + assertThat(web3HttpClient).isNotNull(); + web3HttpClient.shutdown(); + } + + @AfterClass + public static void removePantheonPrivateNetwork() throws IOException, InterruptedException { + ProcessBuilder processBuilder = + new ProcessBuilder("quickstart/removePantheonPrivateNetwork.sh"); + processBuilder.inheritIO(); // redirect all output to logs + processBuilder.directory(new File(PROJECT_ROOT)); // going up one level is the project root + Process process = processBuilder.start(); + + int exitValue = process.waitFor(); + if (exitValue != 0) { + // check for errors, all output and then also error messages and causes are redirected to logs + throw new RuntimeException("execution of script failed!"); + } + } + + @Test + public void servicesShouldBeUp() { + Awaitility.await() + .ignoreExceptions() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> + Arrays.stream(ServicesIdentifier.values()) + .forEach( + servicesIdentifier -> + assertThat(services.get(servicesIdentifier).state).isEqualTo("up"))); + } + + @Test + public void servicesEndpointsShouldBeExposed() { + Awaitility.await() + .ignoreExceptions() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> + Arrays.stream(EndpointsIdentifier.values()) + .forEach( + endpointsIdentifier -> + assertThat(endpoints.get(endpointsIdentifier)) + .isNotNull() + .isNotEmpty())); + } + + @Test + public void rpcNodeShouldFindPeers() { + // Peers are those defined in docker-compose.yml and launched with scaling of 4 regular nodes + // which gives us 6 peers of the RPC node: bootnode, minernode and 4 regular nodes. + int expectecNumberOfPeers = 6; + + Awaitility.await() + .ignoreExceptions() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertThat(web3HttpClient.netPeerCount().send().getQuantity().intValueExact()) + .isEqualTo(expectecNumberOfPeers)); + } + + @Test + public void rpcNodeShouldReturnCorrectVersion() { + String expectedVersion = PantheonInfo.version(); + Awaitility.await() + .ignoreExceptions() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertThat(web3HttpClient.web3ClientVersion().send().getWeb3ClientVersion()) + .isEqualTo(expectedVersion)); + } + + @Test + public void mustMineSomeBlocks() { + // A bug occurred that failed mining after 2 blocks, so testing at least 10. + int expectedAtLeastBlockNumber = 10; + Awaitility.await() + .ignoreExceptions() + .atMost(5, TimeUnit.MINUTES) + .untilAsserted( + () -> + assertThat(web3HttpClient.ethBlockNumber().send().getBlockNumber().intValueExact()) + .isGreaterThan(expectedAtLeastBlockNumber)); + } + + @Test + public void webSocketRpcServiceMustConnect() { + RequestOptions options = new RequestOptions(); + options.setPort( + services + .get(ServicesIdentifier.RPCNODE) + .exposedPorts + .get(DEFAULT_WS_RPC_PORT) + .externalPort); + options.setHost(DEFAULT_RPC_HOST); + + final WebSocket[] wsConnection = new WebSocket[1]; + final Vertx vertx = Vertx.vertx(); + try { + vertx + .createHttpClient(new HttpClientOptions()) + .websocket(options, websocket -> wsConnection[0] = websocket); + + Awaitility.await() + .ignoreExceptions() + .atMost(30, TimeUnit.SECONDS) + .untilAsserted(() -> assertThat(wsConnection[0]).isNotNull()); + } finally { + assertThat(wsConnection[0]).isNotNull(); + wsConnection[0].close(); + } + } + + @Test + public void explorerShouldHaveModifiedHttpRpcEndpoint() throws IOException { + // we have to check that the sed command well replaced the endpoint placeholder with the + // real dynamic endpoint. But as this is a react app, we have to search for the value in the JS + // as we can't find it in the HTML page because source code is rendered dynamically. + Request request = + new Request.Builder() + .get() + .url(endpoints.get(EndpointsIdentifier.EXPLORER) + "/static/js/bundle.js") + .build(); + + OkHttpClient httpClient = new OkHttpClient(); + try (Response resp = httpClient.newCall(request).execute()) { + assertThat(resp.code()).isEqualTo(200); + assertThat(resp.body()).isNotNull(); + assertThat(resp.body().string()) + .containsOnlyOnce( + "var fallbackUrl = '" + endpoints.get(EndpointsIdentifier.HTTP_RPC) + "';"); + } + } + + private enum ServicesIdentifier { + BOOTNODE, + EXPLORER, + MINERNODE, + NODE, + RPCNODE; + + final Pattern pattern; + + ServicesIdentifier() { + pattern = + Pattern.compile( + "(^" + + DOCKER_COMPOSE_PROJECT_PREFIX + + this.name().toLowerCase() + + "_[0-9]+)\\s+.+\\s{3,}(\\w+)\\s+(.+)", + Pattern.DOTALL); + } + } + + private enum EndpointsIdentifier { + EXPLORER("Web block explorer address"), + HTTP_RPC("JSON-RPC.+HTTP.+service endpoint"), + WS_RPC("JSON-RPC.+WebSocket.+service endpoint"); + + final Pattern pattern; + + EndpointsIdentifier(final String lineLabel) { + pattern = Pattern.compile(lineLabel + ".+(http://.+:[0-9]+)", Pattern.DOTALL); + } + } + + static class Service { + + String name; + String state; + Map exposedPorts = new HashMap<>(); + + static class ExposedPort { + + final Integer internalPort; + final Integer externalPort; + private final Pattern pattern = Pattern.compile("[0-9]+", Pattern.DOTALL); + + ExposedPort(final String portDescription) { + if (portDescription.contains("->")) { + String[] ports = portDescription.split("->", 2); + + String[] internalPortInfos = ports[1].split("/", 2); + internalPort = Integer.valueOf(internalPortInfos[0]); + String[] externalPortInfos = ports[0].split(":", 2); + + externalPort = Integer.valueOf(externalPortInfos[1]); + } else { + Matcher regexMatcher = pattern.matcher(portDescription); + internalPort = regexMatcher.find() ? Integer.valueOf(regexMatcher.group(0)) : null; + externalPort = null; + } + } + } + } +} diff --git a/quickstart/src/test/resources/log4j2.xml b/quickstart/src/test/resources/log4j2.xml new file mode 100644 index 0000000000..81c6d799d2 --- /dev/null +++ b/quickstart/src/test/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + INFO + + + + + + + + + + + + diff --git a/settings.gradle b/settings.gradle index cbedd14481..67f3609484 100644 --- a/settings.gradle +++ b/settings.gradle @@ -19,3 +19,4 @@ include 'services:kvstore' include 'testutil' include 'util' include 'errorprone-checks' +include 'quickstart'