From 9cb6600456000c68aebec54670cb27ac017f6f61 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 31 Jan 2024 08:41:02 -0500 Subject: [PATCH] GitHub Actions CI/CD (#6427) Implements a CI/CD pipeline using Github Actions, to replace the current CircleCI implementation. https://wiki.hyperledger.org/pages/viewpage.action?pageId=80774216 --------- Signed-off-by: Justin Florentine Signed-off-by: Gabriel Fukushima Signed-off-by: Sally MacFarlane Signed-off-by: jflo Signed-off-by: RoboCopsGoneSock <158174948+RoboCopsGoneSock@users.noreply.github.com> Signed-off-by: Danno Ferrin (shemnon) Signed-off-by: Danno Ferrin Signed-off-by: Karim Taam Signed-off-by: Ameziane H Signed-off-by: ahamlat Signed-off-by: garyschulte Co-authored-by: Fabio Di Fabio Co-authored-by: Gabriel Fukushima Co-authored-by: Sally MacFarlane Co-authored-by: RoboCopsGoneSock <158174948+RoboCopsGoneSock@users.noreply.github.com> Co-authored-by: Danno Ferrin Co-authored-by: Karim TAAM Co-authored-by: garyschulte --- .github/workflows/acceptance-tests.yml | 114 ++++++++++++++ .github/workflows/artifacts.yml | 76 +++++++++ .github/workflows/checks.yml | 36 ----- .github/workflows/codeql.yml | 41 ++--- .github/workflows/dco-merge-group.yml | 10 -- .github/workflows/dco.yml | 20 --- .github/workflows/docker.yml | 113 ++++++++++++++ .../workflows/gradle-wrapper-validation.yml | 11 -- .github/workflows/integration-tests.yml | 73 +++++++++ .github/workflows/nightly.yml | 121 ++++++++++++++ .github/workflows/parallel-unit-tests.yml | 49 ++++++ .github/workflows/pr-checklist-on-open.yml | 8 +- .github/workflows/pre-review.yml | 103 ++++++++++++ .github/workflows/reference-tests.yml | 147 ++++++++++++++++++ .github/workflows/release.yml | 11 +- .github/workflows/repolinter.yml | 24 --- .github/workflows/sonarcloud.yml | 12 +- CHANGELOG.md | 1 + build.gradle | 15 +- ethereum/api/build.gradle | 26 ++++ 20 files changed, 869 insertions(+), 142 deletions(-) create mode 100644 .github/workflows/acceptance-tests.yml create mode 100644 .github/workflows/artifacts.yml delete mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/dco-merge-group.yml delete mode 100644 .github/workflows/dco.yml create mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/gradle-wrapper-validation.yml create mode 100644 .github/workflows/integration-tests.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/parallel-unit-tests.yml create mode 100644 .github/workflows/pre-review.yml create mode 100644 .github/workflows/reference-tests.yml delete mode 100644 .github/workflows/repolinter.yml diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml new file mode 100644 index 0000000000..b94218e608 --- /dev/null +++ b/.github/workflows/acceptance-tests.yml @@ -0,0 +1,114 @@ +name: acceptance-tests +on: + pull_request: + pull_request_review: + types: [submitted] + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + total-runners: 16 + +jobs: + shouldRun: + name: checks to ensure we should run + # necessary because there is no single PR approved event, need to check all comments/approvals/denials + runs-on: ubuntu-22.04 + outputs: + shouldRun: ${{steps.shouldRun.outputs.result}} + steps: + - name: required check + id: shouldRun + uses: actions/github-script@v7.0.1 + env: + # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main + RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + with: + script: | + const { RELEVANT_SHA } = process.env; + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: RELEVANT_SHA, + }); + const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); + const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success') > 0; + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); + + console.log("tests should be run = %j", shouldRun); + console.log("alreadyRun = %j", alreadyRun); + console.log("approvingReviews = %j", approvingReviews.length); + + return shouldRun; + acceptanceTestEthereum: + runs-on: ubuntu-22.04 + name: "Acceptance Runner" + needs: shouldRun + permissions: + statuses: write + checks: write + if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}} + strategy: + fail-fast: true + matrix: + runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: get acceptance test report + uses: dawidd6/action-download-artifact@v2 + with: + branch: main + name_is_regexp: true + name: 'acceptance-node-\d*\d-test-results' + path: tmp/junit-xml-reports-downloaded + if_no_artifact_found: true + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: Split tests + id: split-tests + uses: r7kamura/split-tests-by-timings@v0 + with: + reports: tmp/junit-xml-reports-downloaded + glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java' + total: ${{env.total-runners}} + index: ${{ matrix.runner_index }} + - name: write out test list + run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt + - name: format gradle args + #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter, + #then drop file extension, then insert --tests option between each. + run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g' > gradleArgs.txt + - name: run acceptance tests + run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: cleanup tempfiles + run: rm testList.txt gradleArgs.txt + - name: Upload Acceptance Test Results + uses: actions/upload-artifact@v3.1.0 + with: + name: acceptance-node-${{matrix.runner_index}}-test-results + path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: (success() || failure()) # always run even if the build step fails + with: + report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' + acceptance-tests: + runs-on: ubuntu-22.04 + needs: [ acceptanceTestEthereum ] + permissions: + checks: write + statuses: write + steps: + - name: consolidation + run: echo "consolidating statuses" diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml new file mode 100644 index 0000000000..3b33f43bce --- /dev/null +++ b/.github/workflows/artifacts.yml @@ -0,0 +1,76 @@ + +name: artifacts + +on: + release: + types: + - prereleased + +jobs: + artifacts: + runs-on: ubuntu-22.04 + permissions: + contents: write + steps: + - name: checkout + uses: actions/checkout@v4.1.1 + - name: Set up JDK 17 + uses: actions/setup-java@v4.0.0 + with: + distribution: 'temurin' + java-version: '17' + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: assemble distributions + run: + ./gradlew -Prelease.releaseVersion=${{github.ref_name}} assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: hashes + id: hashes + run: | + cd build/distributions + echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT + echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT + - name: upload tarball + uses: actions/upload-artifact@v3 + with: + path: 'build/distributions/besu*.tar.gz' + name: besu-${{ github.ref_name }}.tar.gz + - name: upload zipfile + uses: actions/upload-artifact@v3 + with: + path: 'build/distributions/besu*.zip' + name: besu-${{ github.ref_name }}.zip + - name: Upload Release assets + uses: softprops/action-gh-release@v1 + with: + append_body: true + files: | + build/distributions/besu*.tar.gz + build/distributions/besu*.zip + body: | + ${{steps.hashes.outputs.tarSha}} + ${{steps.hashes.outputs.zipSha}} + testWindows: + runs-on: windows-2022 + needs: artifacts + timeout-minutes: 10 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: adopt + java-version: 17 + - name: Download zip + uses: actions/download-artifact@v3 + with: + name: besu-${{ github.ref_name }}.zip + - name: test Besu + run: | + unzip besu-*.zip -d besu-tmp + cd besu-tmp + mv besu-* ../besu + cd .. + besu\bin\besu.bat --help + besu\bin\besu.bat --version + diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml deleted file mode 100644 index ee5aa159e9..0000000000 --- a/.github/workflows/checks.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: checks -on: - push: - branches: [ main ] - pull_request: - workflow_dispatch: - -jobs: - spotless: - runs-on: [besu-research-ubuntu-16] - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Set up Java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - cache: gradle - - name: spotless - run: ./gradlew --no-daemon --parallel clean spotlessCheck - javadoc_17: - runs-on: [besu-research-ubuntu-8] - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Set up Java 17 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - cache: gradle - - name: javadoc (JDK 17) - run: ./gradlew --no-daemon clean javadoc diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 590227e98d..f884212333 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -14,40 +14,29 @@ name: "CodeQL" on: push: branches: [ main ] - pull_request: - branches: [ main ] - paths-ignore: - - '**/*.json' - - '**/*.md' - - '**/*.properties' - - '**/*.txt' + pull_request: + branches: [ main ] + paths-ignore: + - '**/*.json' + - '**/*.md' + - '**/*.properties' + - '**/*.txt' jobs: analyze: name: Analyze - runs-on: [besu-research-ubuntu-16] + runs-on: ubuntu-22.04 permissions: actions: read contents: read security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'java' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - steps: - name: Checkout repository - uses: actions/checkout@v4 - + uses: actions/checkout@v4.1.1 - name: Set up Java - uses: actions/setup-java@v4 + uses: actions/setup-java@v4.0.0 with: distribution: 'temurin' java-version: 17 - cache: gradle - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 @@ -58,10 +47,10 @@ jobs: # Prefix the list here with "+" to use these queries and those in the config file. queries: security-and-quality,security-extended - # Autobuild failed (OOM) - # Hence, supply memory args for gradle build - - run: | - JAVA_OPTS="-Xmx1000M" ./gradlew --no-scan compileJava - + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: compileJava noscan + run: | + JAVA_OPTS="-Xmx2048M" ./gradlew --no-scan compileJava - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/dco-merge-group.yml b/.github/workflows/dco-merge-group.yml deleted file mode 100644 index fee29b6c5d..0000000000 --- a/.github/workflows/dco-merge-group.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: dco -on: - merge_group: - -jobs: - dco: - runs-on: [besu-research-ubuntu-8] - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index 5fa9931c77..0000000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: dco -on: - pull_request: - workflow_dispatch: - -jobs: - dco: - runs-on: [besu-research-ubuntu-8] - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - run: echo "This DCO job runs on pull_request event and workflow_dispatch" - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.2.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..c487ed84b2 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,113 @@ +name: docker +on: + release: + types: + - prereleased +env: + registry: ghcr.io + +jobs: + hadolint: + runs-on: ubuntu-22.04 + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: hadoLint_openj9-jdk_17 + run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile + - name: hadoLint_openjdk_17 + run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile + - name: hadoLint_openjdk_17_debug + run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile + - name: hadoLint_openjdk_latest + run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile + - name: hadoLint_graalvm + run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile + buildDocker: + needs: hadolint + permissions: + contents: read + packages: write + + strategy: + fail-fast: false + matrix: + platform: + - ubuntu-22.04 + - [self-hosted, ARM64] + runs-on: ${{ matrix.platform }} + steps: + - name: Prepare + id: prep + run: | + platform=${{ matrix.platform }} + if [ "$platform" = 'ubuntu-22.04' ]; then + echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT + echo "ARCH=amd64" >> $GITHUB_OUTPUT + else + echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT + echo "ARCH=arm64" >> $GITHUB_OUTPUT + fi + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: short sha + id: shortSha + run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: install goss + run: | + mkdir -p docker/reports + curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} + - name: build and test docker + uses: gradle/gradle-build-action@v2.12.0 + env: + architecture: ${{ steps.prep.outputs.ARCH }} + with: + arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }} + - name: login to ghcr + uses: docker/login-action@v3.0.0 + with: + registry: ${{ env.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: publish + env: + architecture: ${{ steps.prep.outputs.ARCH }} + run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }} + multiArch: + needs: buildDocker + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: login to ghcr + uses: docker/login-action@v3.0.0 + with: + registry: ${{ env.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: multi-arch docker + run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }} + diff --git a/.github/workflows/gradle-wrapper-validation.yml b/.github/workflows/gradle-wrapper-validation.yml deleted file mode 100644 index 6f44fe43e5..0000000000 --- a/.github/workflows/gradle-wrapper-validation.yml +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -name: "Validate Gradle Wrapper" -on: [push, pull_request] - -jobs: - validation: - name: "Gradle Wrapper Validation" - runs-on: [besu-research-ubuntu-8] - steps: - - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000000..68115ea40b --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,73 @@ +name: integration-tests +on: + pull_request: + pull_request_review: + types: + - submitted + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + +jobs: + shouldRun: + name: checks to ensure we should run + runs-on: ubuntu-22.04 + outputs: + shouldRun: ${{steps.shouldRun.outputs.result}} + steps: + - name: required check + id: shouldRun + uses: actions/github-script@v7.0.1 + env: + # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main + RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + with: + script: | + const { RELEVANT_SHA } = process.env; + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: RELEVANT_SHA, + }); + + const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests'); + const alreadyRun = intTested && intTested.find(({ state }) => state === 'success') > 0; + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); + + console.log("tests should be run = %j", shouldRun); + console.log("alreadyRun = %j", alreadyRun); + console.log("approvingReviews = %j", approvingReviews.length); + + return shouldRun; + integration-tests: + runs-on: ubuntu-22.04 + needs: shouldRun + if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} + permissions: + statuses: write + checks: write + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: run integration tests + run: ./gradlew integrationTest compileJmh -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: (success() || failure()) + with: + report_paths: '**/build/test-results/integrationTest/TEST-*.xml' + + diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..9d2778fba4 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,121 @@ +name: nightly + +on: + workflow_dispatch: + schedule: + # * is a special character in YAML so you have to quote this string + # expression evaluates to midnight every night + - cron: '0 0 * * *' + +env: + nightly-tag: develop + registry: ghcr.io + +jobs: + hadolint: + runs-on: ubuntu-22.04 + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: hadoLint_openj9-jdk_17 + run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile + - name: hadoLint_openjdk_17 + run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile + - name: hadoLint_openjdk_17_debug + run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile + - name: hadoLint_openjdk_latest + run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile + - name: hadoLint_graalvm + run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile + buildDocker: + needs: hadolint + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: + - ubuntu-22.04 + - [self-hosted, ARM64] + runs-on: ${{ matrix.platform }} + steps: + - name: Prepare + id: prep + run: | + platform=${{ matrix.platform }} + if [ "$platform" = 'ubuntu-22.04' ]; then + echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT + echo "ARCH=amd64" >> $GITHUB_OUTPUT + else + echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT + echo "ARCH=arm64" >> $GITHUB_OUTPUT + fi + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: short sha + id: shortSha + run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: build image + uses: gradle/gradle-build-action@v2.12.0 + with: + arguments: distDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main + - name: install goss + run: | + mkdir -p docker/reports + curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} + - name: test docker + uses: gradle/gradle-build-action@v2.12.0 + env: + architecture: ${{ steps.prep.outputs.ARCH }} + with: + arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main + - name: login to ghcr + uses: docker/login-action@v3.0.0 + with: + registry: ${{ env.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: publish + env: + architecture: ${{ steps.prep.outputs.ARCH }} + run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main + multiArch: + permissions: + contents: read + packages: write + needs: buildDocker + runs-on: ubuntu-22.04 + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: Login to DockerHub + uses: docker/login-action@v3.0.0 + with: + registry: ${{ env.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: multi-arch docker + run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main + diff --git a/.github/workflows/parallel-unit-tests.yml b/.github/workflows/parallel-unit-tests.yml new file mode 100644 index 0000000000..b12fa43655 --- /dev/null +++ b/.github/workflows/parallel-unit-tests.yml @@ -0,0 +1,49 @@ +name: parallel-unit-tests +#experimental work in progress - trying to figure out how to split tests across multi-modules by runtime +on: + workflow_dispatch: + +env: + GRADLE_OPTS: "-Dorg.gradle.daemon=false" + total-runners: 4 +jobs: + junit: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + runner_index: + - 0 + - 1 + - 2 + - 3 + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Split tests + id: split-tests + uses: chaosaffe/split-tests@v1-alpha.1 + with: + glob: '**/src/test/java/**/*.java' + split-total: ${{ env.total-runners }} + split-index: ${{ matrix.runner_index }} + line-count: true + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: write out test list + run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt + - name: debug testfile paths + run: cat testList.txt + - name: format gradle args + # regex means: truncate file paths to align with package name, replacing with tests switch, then drop file extension, + # then swap path delimiter with package delimiter + run: cat testList.txt | sed -e 's/[^ ]*src\/test\/java\//--tests\ /g' -e 's/\.java//g' -e 's/\//\./g' >> gradleArgs.txt + - name: debug test class list + run: cat gradleArgs.txt + - name: run unit tests + run: ./gradlew test `cat gradleArgs.txt` + diff --git a/.github/workflows/pr-checklist-on-open.yml b/.github/workflows/pr-checklist-on-open.yml index 3a4d4342c5..f849ba760b 100644 --- a/.github/workflows/pr-checklist-on-open.yml +++ b/.github/workflows/pr-checklist-on-open.yml @@ -6,9 +6,11 @@ on: jobs: checklist: name: "add checklist as a comment on newly opened PRs" - runs-on: [besu-research-ubuntu-8] + runs-on: ubuntu-22.04 + permissions: + pull-requests: write steps: - - uses: actions/github-script@v5 + - uses: actions/github-script@v7.0.1 with: github-token: ${{secrets.GITHUB_TOKEN}} script: | @@ -16,5 +18,5 @@ jobs: issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests' + body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests\n- [ ] I thought about running CI.\n- [ ] If I did not run CI, I ran as much locally as possible before pushing.\n-' }) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml new file mode 100644 index 0000000000..5abbcb6d0a --- /dev/null +++ b/.github/workflows/pre-review.yml @@ -0,0 +1,103 @@ +name: pre-review + +on: + pull_request: + workflow_dispatch: + +permissions: + statuses: write + checks: write + +jobs: + repolint: + name: "Repository Linting" + runs-on: ubuntu-22.04 + container: ghcr.io/todogroup/repolinter:v0.11.2 + steps: + - name: Checkout Code + uses: actions/checkout@v4.1.1 + - name: Lint Repo + run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown + gradle-wrapper: + name: "Gradle Wrapper Validation" + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4.1.1 + - uses: gradle/wrapper-validation-action@v1.1.0 + spotless: + runs-on: ubuntu-22.04 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: Setup Gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: run spotless + run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + compile: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + needs: [spotless, gradle-wrapper, repolint] + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: Setup Gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: Gradle Compile + run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + unitTests: + env: + GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}} + runs-on: ubuntu-22.04 + needs: [ compile ] + permissions: + checks: write + statuses: write + strategy: + fail-fast: true + matrix: + gradle_args: + - "test -x besu:test -x consensus:test -x crypto:test -x ethereum:eth:test -x ethereum:api:test -x ethereum:core:test" + - "besu:test consensus:test crypto:test" + - "ethereum:api:testBonsai" + - "ethereum:api:testForest" + - "ethereum:api:testRemainder" + - "ethereum:core:test" + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: Setup Gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: run unit tests + id: unitTest + run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() # always run even if the build step fails + with: + report_paths: '**/test-results/**/TEST-*.xml' + annotate_only: true + pre-review: + runs-on: ubuntu-22.04 + needs: [unitTests] + permissions: + checks: write + statuses: write + steps: + - name: consolidation + run: echo "consolidating statuses" diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml new file mode 100644 index 0000000000..1f2cac7b15 --- /dev/null +++ b/.github/workflows/reference-tests.yml @@ -0,0 +1,147 @@ +name: reference-tests +on: + pull_request: + pull_request_review: + types: + - submitted + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + total-runners: 6 + +jobs: + shouldRun: + name: checks to ensure we should run + # necessary because there is no single PR approved event, need to check all comments/approvals/denials + # might also be a job running, and additional approvals + runs-on: ubuntu-22.04 + outputs: + shouldRun: ${{steps.shouldRun.outputs.result}} + steps: + - name: required check + id: shouldRun + uses: actions/github-script@v7.0.1 + env: + # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main + RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + with: + script: | + const { RELEVANT_SHA } = process.env; + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: RELEVANT_SHA, + }); + + + const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests'); + const alreadyRun = refTested && refTested.find(({ state }) => state === 'success') > 0; + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); + + console.log("tests should be run = %j", shouldRun); + console.log("alreadyRun = %j", alreadyRun); + console.log("approvingReviews = %j", approvingReviews.length); + + return shouldRun; + + prepareReferenceTestEthereum: + runs-on: ubuntu-22.04 + needs: shouldRun + if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + with: + submodules: recursive + set-safe-directory: true + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: temurin + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: execute generate reference tests + run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: store generated tests + uses: actions/upload-artifact@v3 + with: + name: 'reference-tests' + path: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' + + referenceTestEthereum: + runs-on: ubuntu-22.04 + permissions: + statuses: write + checks: write + needs: + - prepareReferenceTestEthereum + if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} + strategy: + fail-fast: true + matrix: + runner_index: [0,1,2,3,4,5] + steps: + - name: Checkout Repo + uses: actions/checkout@v4.1.1 + with: + submodules: recursive + - name: Set up Java + uses: actions/setup-java@v4.0.0 + with: + distribution: adopt-openj9 + java-version: 17 + - name: retrieve generated tests + uses: actions/download-artifact@v3.0.2 + with: + name: 'reference-tests' + path: 'ethereum/referencetests/build/generated/sources/reference-test/' + - name: get reference test report + uses: dawidd6/action-download-artifact@v2 + with: + branch: main + name_is_regexp: true + name: 'reference-test-node-\d*\d-results' + path: tmp/ref-xml-reports-downloaded + if_no_artifact_found: true + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 + - name: Split tests + id: split-tests + uses: r7kamura/split-tests-by-timings@v0 + with: + reports: tmp/ref-xml-reports-downloaded + glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' + total: ${{env.total-runners}} + index: ${{ matrix.runner_index }} + - name: compose gradle args + run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt + - name: run reference tests + run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: reference-test-node-${{matrix.runner_index}}-results + path: '**/build/test-results/referenceTests/TEST-*.xml' + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() # always run even if the build step fails + with: + report_paths: '**/build/test-results/referenceTest/TEST-*.xml' + reference-tests: + runs-on: ubuntu-22.04 + needs: [ referenceTestEthereum ] + permissions: + checks: write + statuses: write + steps: + - name: consolidation + run: echo "consolidating statuses" + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 16fd67f94d..e0c4f0bb29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,13 +1,14 @@ name: release besu on: + workflow_dispatch: release: - types: released + types: [released] jobs: dockerPromoteX64: - runs-on: [besu-research-ubuntu-16] + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v4.1.1 + - uses: actions/setup-java@v4.0.0 with: distribution: 'temurin' # See 'Supported distributions' for available options java-version: '17' @@ -15,7 +16,7 @@ jobs: - name: Login to DockerHub run: echo '${{ secrets.DOCKER_PASSWORD_RW }}' | docker login -u '${{ secrets.DOCKER_USER_RW }}' --password-stdin - name: Setup Gradle - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v2.12.0 - name: Docker upload run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ secrets.DOCKER_ORG }}" dockerUploadRelease - name: Docker manifest diff --git a/.github/workflows/repolinter.yml b/.github/workflows/repolinter.yml deleted file mode 100644 index 983164e106..0000000000 --- a/.github/workflows/repolinter.yml +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Hyperledger Repolinter Action -name: Repolinter - -on: - workflow_dispatch: - push: - branches: - - master - - main - pull_request: - branches: - - master - - main - -jobs: - build: - runs-on: [besu-research-ubuntu-16] - container: ghcr.io/todogroup/repolinter:v0.10.1 - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Lint Repo - run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 92d9a7713e..7c4acee7e4 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -14,24 +14,26 @@ permissions: jobs: Analysis: runs-on: ubuntu-latest - if: github.repository == 'hyperledger/besu' steps: - name: checkout - uses: actions/checkout@v4 + uses: actions/checkout@v4.1.1 - name: Set up JDK 17 - uses: actions/setup-java@v4 + uses: actions/setup-java@v4.0.0 with: distribution: 'temurin' java-version: '17' - cache: gradle - name: Cache SonarCloud packages uses: actions/cache@v3 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar + - name: setup gradle + uses: gradle/gradle-build-action@v2.12.0 - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: ./gradlew build sonarqube --continue --info + SONAR_ORGANIZATION: ${{ env.SONAR_ORGANIZATION }} + SONAR_PROJECT_KEY: $ {{ env.SONAR_PROJECT_KEY }} + run: ./gradlew build sonarqube --continue --info -Dorg.gradle.parallel=true -Dorg.gradle.caching=true diff --git a/CHANGELOG.md b/CHANGELOG.md index b1681d5c1a..861e197772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - Log blob count when importing a block via Engine API [#6466](https://github.com/hyperledger/besu/pull/6466) - Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390) - Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303) +- Github Actions based build. - Introduce caching mechanism to optimize Keccak hash calculations for account storage slots during block processing [#6452](https://github.com/hyperledger/besu/pull/6452) - Added configuration options for `pragueTime` to genesis file for Prague fork development [#6473](https://github.com/hyperledger/besu/pull/6473) - Moving trielog storage to RocksDB's blobdb to improve write amplications [#6289](https://github.com/hyperledger/besu/pull/6289) diff --git a/build.gradle b/build.gradle index 255a878d97..c8cf32a759 100644 --- a/build.gradle +++ b/build.gradle @@ -36,8 +36,9 @@ plugins { sonarqube { properties { - property "sonar.projectKey", "hyperledger_besu" - property "sonar.organization", "hyperledger" + property "sonar.projectKey", "$System.env.SONAR_PROJECT_KEY" + property "sonar.organization", "$System.env.SONAR_ORGANIZATION" + property "sonar.gradle.skipCompile", "true" property "sonar.host.url", "https://sonarcloud.io" property "sonar.coverage.jacoco.xmlReportPaths", "${buildDir}/reports/jacoco/jacocoRootReport/jacocoRootReport.xml" property "sonar.coverage.exclusions", "acceptance-tests/**/*" @@ -646,6 +647,8 @@ task autocomplete(type: JavaExec) { } } +def archiveBuildVersion = project.hasProperty('release.releaseVersion') ? project.property('release.releaseVersion') : "${rootProject.version}" + installDist { dependsOn checkLicense, untunedStartScripts, evmToolStartScripts } distTar { @@ -654,6 +657,7 @@ distTar { delete fileTree(dir: 'build/distributions', include: '*.tar.gz') } compression = Compression.GZIP + setVersion(archiveBuildVersion) archiveExtension = 'tar.gz' } @@ -662,6 +666,7 @@ distZip { doFirst { delete fileTree(dir: 'build/distributions', include: '*.zip') } + setVersion(archiveBuildVersion) } publishing { @@ -983,6 +988,12 @@ task checkSpdxHeader(type: CheckSpdxHeader) { ].join("|") } +jacocoTestReport { + reports { + xml.enabled true + } +} + task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) { additionalSourceDirs.from files(subprojects.sourceSets.main.allSource.srcDirs) sourceDirectories.from files(subprojects.sourceSets.main.allSource.srcDirs) diff --git a/ethereum/api/build.gradle b/ethereum/api/build.gradle index ed395df8c5..e5836c46eb 100644 --- a/ethereum/api/build.gradle +++ b/ethereum/api/build.gradle @@ -168,3 +168,29 @@ tasks.register('generateTestBlockchain') { } } test.dependsOn(generateTestBlockchain) +/* + Utility tasks used to separate out long running suites of tests so they can be parallelized in CI + */ +tasks.register("testBonsai", Test) { + useJUnitPlatform() + filter { + includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*") + } + dependsOn(generateTestBlockchain) +} + +tasks.register("testForest", Test) { + useJUnitPlatform() + filter { + includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*") + } + dependsOn(generateTestBlockchain) +} + +tasks.register("testRemainder", Test) { + useJUnitPlatform() + filter { + excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*") + excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*") + } +}