besu/.github/workflows/reference-tests.yml

name: reference-tests
on:
  pull_request:
  pull_request_review:
    types:
      - submitted

env:
  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
  total-runners: 6

jobs:
  shouldRun:
    name: checks to ensure we should run
    # necessary because there is no single PR approved event, need to check all comments/approvals/denials
    # might also be a job running, and additional approvals
    runs-on: ubuntu-22.04
    outputs:
      shouldRun: ${{steps.shouldRun.outputs.result}}
    steps:
      - name: required check
        id: shouldRun
        uses: actions/github-script@v7.0.1
        env:
          # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main
          RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
        with:
          script: |
            const { RELEVANT_SHA } = process.env;
            const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({
              owner: context.repo.owner,
              repo: context.repo.repo,
              ref: RELEVANT_SHA,
            });
            
            
            const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests');
            const alreadyRun = refTested && refTested.find(({ state }) => state === 'success') > 0;
            const { data: reviews } = await github.rest.pulls.listReviews({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: context.issue.number,
            });
            const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED');
            const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' &&  (approvingReviews.length > 0);

              console.log("tests should be run = %j", shouldRun);
              console.log("alreadyRun = %j", alreadyRun);
              console.log("approvingReviews = %j", approvingReviews.length);

            return shouldRun;

  prepareReferenceTestEthereum:
    runs-on: ubuntu-22.04
    needs: shouldRun
    if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }}
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4.1.1
        with:
          submodules: recursive
          set-safe-directory: true
      - name: Set up Java
        uses: actions/setup-java@v4.0.0
        with:
          distribution: temurin
          java-version: 17
      - name: setup gradle
        uses: gradle/gradle-build-action@v2.12.0
      - name: execute generate reference tests
        run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
      - name: store generated tests
        uses: actions/upload-artifact@v3
        with:
          name: 'reference-tests'
          path: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java'

  referenceTestEthereum:
    runs-on: ubuntu-22.04
    permissions:
      statuses: write
      checks: write
    needs:
      - prepareReferenceTestEthereum
    if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }}
    strategy:
      fail-fast: true
      matrix:
        runner_index: [0,1,2,3,4,5]
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4.1.1
        with:
          submodules: recursive
      - name: Set up Java
        uses: actions/setup-java@v4.0.0
        with:
          distribution: adopt-openj9
          java-version: 17
      - name: retrieve generated tests
        uses: actions/download-artifact@v3.0.2
        with:
          name: 'reference-tests'
          path: 'ethereum/referencetests/build/generated/sources/reference-test/'
      - name: get reference test report
        uses: dawidd6/action-download-artifact@v2
        with:
          branch: main
          name_is_regexp: true
          name: 'reference-test-node-\d*\d-results'
          path: tmp/ref-xml-reports-downloaded
          if_no_artifact_found: true
      - name: setup gradle
        uses: gradle/gradle-build-action@v2.12.0
      - name: Split tests
        id: split-tests
        uses: r7kamura/split-tests-by-timings@v0
        with:
          reports: tmp/ref-xml-reports-downloaded
          glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java'
          total: ${{env.total-runners}}
          index: ${{ matrix.runner_index }}
      - name: compose gradle args
        run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt
      - name: run reference tests
        run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
      - name: Upload Test Report
        uses: actions/upload-artifact@v3
        if: always() # always run even if the previous step fails
        with:
          name: reference-test-node-${{matrix.runner_index}}-results
          path: '**/build/test-results/referenceTests/TEST-*.xml'
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v4
        if: success() || failure() # always run even if the build step fails
        with:
          report_paths: '**/build/test-results/referenceTest/TEST-*.xml'
  reference-tests:
    runs-on: ubuntu-22.04
    needs: [ referenceTestEthereum ]
    permissions:
      checks: write
      statuses: write
    steps:
      - name: consolidation
        run: echo "consolidating statuses"