Support removed and added test when splitting by time (#6990)

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

.github/workflows/acceptance-tests.yml

@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+  GRADLE_OPTS: "-Xmx6g"
   total-runners: 16
 
 jobs:
@@ -49,6 +49,10 @@ jobs:
         uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
         with:
           cache-disabled: true
+      - name: Compile acceptance tests
+        run: ./gradlew :acceptance-tests:tests:testClasses
+      - name: List acceptance tests
+        run: ./gradlew :acceptance-tests:tests:listAcceptanceTestNotPrivacy -Dorg.gradle.caching=true | fgrep org.hyperledger.besu.tests.acceptance > tmp/currentTests.list
       - name: Split tests
         run: .github/workflows/splitTestsByTime.sh tmp/junit-xml-reports-downloaded ${{env.total-runners}} ${{ matrix.runner_index }} > testList.txt
       - name: Upload Timing
@@ -57,6 +61,12 @@ jobs:
         with:
           name: acceptance-tests-timing
           path: 'tmp/timing.tsv'
+      - name: Upload Lists
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        if: matrix.runner_index == 0
+        with:
+          name: acceptance-tests-lists
+          path: 'tmp/*.list'
       - name: format gradle args
         # insert --tests option between each.
         run:  cat testList.txt | sed -e 's/^\| / --tests /g' | tee gradleArgs.txt

.github/workflows/splitTestsByTime.sh

@@ -26,27 +26,44 @@ for line in "${sorted[@]}"; do
 	test_time=${line_parts[0]//./} # convert to millis
 	test_time=${test_time##0} # remove leading zeros
 	test_name=${line_parts[1]}
-	
-	# Find index of min sum
-	idx_min_sum=0
-	min_sum=${sums[0]}
-	for ((i=0; i<SPLIT_COUNT; i++))
-	do		
-		if [[ ${sums[$i]} -lt $min_sum ]]
-		then
-			idx_min_sum=$i		
-			min_sum=${sums[$i]}
-		fi
-	done
-
-	# Add the test to the min sum list
-	min_sum_tests=${tests[$idx_min_sum]}
-	tests[$idx_min_sum]="${min_sum_tests}${test_name},"
-	
-	# Update the sums
-	((sums[idx_min_sum]+=test_time))
 
+  # Does the test still exists?
+  if grep -F -q --line-regexp "$test_name" tmp/currentTests.list
+  then
+    # Find index of min sum
+    idx_min_sum=0
+    min_sum=${sums[0]}
+    for ((i=0; i<SPLIT_COUNT; i++))
+    do
+      if [[ ${sums[$i]} -lt $min_sum ]]
+      then
+        idx_min_sum=$i
+        min_sum=${sums[$i]}
+      fi
+    done
+
+    # Add the test to the min sum list
+    min_sum_tests=${tests[$idx_min_sum]}
+    tests[$idx_min_sum]="${min_sum_tests}${test_name},"
+
+    # Update the sums
+    ((sums[idx_min_sum]+=test_time))
+
+    echo "$test_name" >> tmp/processedTests.list
+  fi
 done
 
+# Any new test?
+grep -F --line-regexp -v -f tmp/processedTests.list tmp/currentTests.list > tmp/newTests.list
+idx_new_test=0
+while read -r new_test_name
+do
+  idx_group=$(( idx_new_test % SPLIT_COUNT ))
+  group=${tests[$idx_group]}
+  tests[$idx_group]="${group}${new_test_name},"
+  idx_new_test=$(( idx_new_test + 1 ))
+done < tmp/newTests.list
+
+
 # return the requests index, without quotes to drop the last trailing space
 echo ${tests[$SPLIT_INDEX]//,/ }

acceptance-tests/tests/build.gradle

@@ -258,3 +258,32 @@ task acceptanceTestPermissioning(type: Test) {
 
   doFirst { mkdir "${buildDir}/jvmErrorLogs" }
 }
+
+// temporary solution to get a list of tests
+// Gradle >8.3 has a supported test dry-run option to achieve the same result
+task listAcceptanceTestNotPrivacy {
+  doLast {
+    def testExecutionSpec = tasks.getByName("acceptanceTestNotPrivacy") as Test
+
+    def processor = new org.gradle.api.internal.tasks.testing.TestClassProcessor() {
+        void startProcessing(org.gradle.api.internal.tasks.testing.TestResultProcessor processor) {}
+        void stop() {}
+        void stopNow() {}
+
+        void processTestClass(org.gradle.api.internal.tasks.testing.TestClassRunInfo info) {
+          def splitName = info.getTestClassName().split("\\.");
+          def testClassName = splitName[splitName.length-1];
+          if(testClassName.endsWith("Test") && !testClassName.startsWith("Abstract")) {
+            println(info.getTestClassName())
+          }
+        }
+      }
+
+    def detector = new org.gradle.api.internal.tasks.testing.detection.DefaultTestClassScanner(testExecutionSpec.getCandidateClassFiles(), testExecutionSpec.getTestFramework().getDetector()?.tap {
+      setTestClasses(testExecutionSpec.getTestClassesDirs().getFiles())
+      setTestClasspath(Collections.unmodifiableSet(testExecutionSpec.getClasspath().getFiles()))
+    }, processor)
+
+    detector.run()
+  }
+}