[NC-1615] Report reference tests that have been blacklisted rather than skipping them silently (#63)

ethereum/core/src/test/java/tech/pegasys/pantheon/ethereum/core/TransactionTest.java

@@ -1,5 +1,7 @@
 package tech.pegasys.pantheon.ethereum.core;
 
+import static org.junit.Assume.assumeTrue;
+
 import tech.pegasys.pantheon.ethereum.mainnet.TransactionValidator;
 import tech.pegasys.pantheon.ethereum.rlp.RLP;
 import tech.pegasys.pantheon.ethereum.vm.ReferenceTestProtocolSchedules;
@@ -39,12 +41,14 @@ public class TransactionTest {
             "TransactionWithGasLimitOverflow(2|63)", "TransactionWithGasLimitxPriceOverflow$")
         // Nonce is tracked with type long, large valued nonces can't currently be decoded
         .blacklist("TransactionWithHighNonce256")
-        .generator((name, spec, collector) -> collector.add(name, spec))
+        .generator((name, spec, collector) -> collector.add(name, spec, true))
         .generate(TEST_CONFIG_FILE_DIR_PATH);
   }
 
-  public TransactionTest(final String name, final TransactionTestCaseSpec spec) {
+  public TransactionTest(
+      final String name, final TransactionTestCaseSpec spec, final boolean runTest) {
     this.spec = spec;
+    assumeTrue("Test was blacklisted", runTest);
   }
 
   @Test

ethereum/core/src/test/java/tech/pegasys/pantheon/ethereum/vm/BlockchainReferenceTestTools.java

@@ -40,9 +40,7 @@ public class BlockchainReferenceTestTools {
           .generator(
               (testName, spec, collector) -> {
                 final String eip = spec.getNetwork();
-                if (NETWORKS_TO_RUN.contains(eip)) {
+                collector.add(testName + "[" + eip + "]", spec, NETWORKS_TO_RUN.contains(eip));
-                  collector.add(testName + "[" + eip + "]", spec);
-                }
               });
 
   static {

ethereum/core/src/test/java/tech/pegasys/pantheon/ethereum/vm/GeneralStateReferenceTestTools.java

@@ -47,15 +47,13 @@ public class GeneralStateReferenceTestTools {
                 for (final Map.Entry<String, List<GeneralStateTestCaseEipSpec>> entry :
                     stateSpec.finalStateSpecs().entrySet()) {
                   final String eip = entry.getKey();
-                  if (!EIPS_TO_RUN.contains(eip)) {
+                  final boolean runTest = EIPS_TO_RUN.contains(eip);
-                    continue;
-                  }
                   final List<GeneralStateTestCaseEipSpec> eipSpecs = entry.getValue();
                   if (eipSpecs.size() == 1) {
-                    collector.add(prefix + eip, eipSpecs.get(0));
+                    collector.add(prefix + eip, eipSpecs.get(0), runTest);
                   } else {
                     for (int i = 0; i < eipSpecs.size(); i++) {
-                      collector.add(prefix + eip + '[' + i + ']', eipSpecs.get(i));
+                      collector.add(prefix + eip + '[' + i + ']', eipSpecs.get(i), runTest);
                     }
                   }
                 }

ethereum/core/src/test/java/tech/pegasys/pantheon/ethereum/vm/VMReferenceTest.java

@@ -3,6 +3,7 @@ package tech.pegasys.pantheon.ethereum.vm;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
 import static tech.pegasys.pantheon.ethereum.vm.OperationTracer.NO_TRACING;
 
 import tech.pegasys.pantheon.ethereum.core.Gas;
@@ -88,9 +89,11 @@ public class VMReferenceTest extends AbstractRetryingTest {
         .generate(TEST_CONFIG_FILE_DIR_PATHS);
   }
 
-  public VMReferenceTest(final String name, final VMReferenceTestCaseSpec spec) {
+  public VMReferenceTest(
+      final String name, final VMReferenceTestCaseSpec spec, final boolean runTest) {
     this.name = name;
     this.spec = spec;
+    assumeTrue("Test was blacklisted", runTest);
   }
 
   @Override

ethereum/core/src/test/resources/tech/pegasys/pantheon/ethereum/vm/BlockchainReferenceTest.java.template

@@ -12,6 +12,8 @@ import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
+import static org.junit.Assume.assumeTrue;
+
 /** The blockchain test operation testing framework entry point. */
 @RunWith(Parameterized.class)
 public class %%TESTS_NAME%% {
@@ -28,9 +30,11 @@ public class %%TESTS_NAME%% {
 
   public %%TESTS_NAME%%(
       final String name,
-      final BlockchainReferenceTestCaseSpec spec) {
+      final BlockchainReferenceTestCaseSpec spec,
+      final boolean runTest) {
     this.name = name;
     this.spec = spec;
+    assumeTrue("Test was blacklisted", runTest);
   }
 
   @Test

ethereum/core/src/test/resources/tech/pegasys/pantheon/ethereum/vm/GeneralStateReferenceTest.java.template

@@ -12,6 +12,8 @@ import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
+import static org.junit.Assume.assumeTrue;
+
 /** The general state test operation testing framework entry point. */
 @RunWith(Parameterized.class)
 public class %%TESTS_NAME%% {
@@ -28,9 +30,11 @@ public class %%TESTS_NAME%% {
 
   public %%TESTS_NAME%%(
       final String name,
-      final GeneralStateTestCaseEipSpec spec) {
+      final GeneralStateTestCaseEipSpec spec,
+      final boolean runTest) {
     this.name = name;
     this.spec = spec;
+    assumeTrue("Test was blacklisted", runTest);
   }
 
   @Test

ethereum/rlp/src/test/java/tech/pegasys/pantheon/ethereum/rlp/InvalidRLPRefTest.java

@@ -1,5 +1,7 @@
 package tech.pegasys.pantheon.ethereum.rlp;
 
+import static org.junit.Assume.assumeTrue;
+
 import tech.pegasys.pantheon.testutil.JsonTestParameters;
 
 import java.util.Collection;
@@ -20,8 +22,10 @@ public class InvalidRLPRefTest {
 
   private final InvalidRLPRefTestCaseSpec spec;
 
-  public InvalidRLPRefTest(final String name, final InvalidRLPRefTestCaseSpec spec) {
+  public InvalidRLPRefTest(
+      final String name, final InvalidRLPRefTestCaseSpec spec, final boolean runTest) {
     this.spec = spec;
+    assumeTrue("Test was blacklisted", runTest);
   }
 
   @Parameters(name = "Name: {0}")

ethereum/rlp/src/test/java/tech/pegasys/pantheon/ethereum/rlp/RLPRefTest.java

@@ -1,6 +1,7 @@
 package tech.pegasys.pantheon.ethereum.rlp;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeTrue;
 
 import tech.pegasys.pantheon.testutil.JsonTestParameters;
 
@@ -19,8 +20,9 @@ public class RLPRefTest {
 
   private final RLPRefTestCaseSpec spec;
 
-  public RLPRefTest(final String name, final RLPRefTestCaseSpec spec) {
+  public RLPRefTest(final String name, final RLPRefTestCaseSpec spec, final boolean runTest) {
     this.spec = spec;
+    assumeTrue("Test was blacklisted", runTest);
   }
 
   @Parameters(name = "Name: {0}")

ethereum/trie/src/test/java/tech/pegasys/pantheon/ethereum/trie/TrieRefTest.java

@@ -1,6 +1,7 @@
 package tech.pegasys.pantheon.ethereum.trie;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assume.assumeTrue;
 
 import tech.pegasys.pantheon.testutil.JsonTestParameters;
 import tech.pegasys.pantheon.util.bytes.BytesValue;
@@ -20,8 +21,9 @@ public class TrieRefTest {
 
   private final TrieRefTestCaseSpec spec;
 
-  public TrieRefTest(final String name, final TrieRefTestCaseSpec spec) {
+  public TrieRefTest(final String name, final TrieRefTestCaseSpec spec, final boolean runTest) {
     this.spec = spec;
+    assumeTrue("Test was blacklisted", runTest);
   }
 
   @Parameters(name = "Name: {0}")

testutil/src/main/java/tech/pegasys/pantheon/testutil/JsonTestParameters.java

@@ -50,10 +50,8 @@ public class JsonTestParameters<S, T> {
     // memory when we run a single test, but it's not the case we're trying to optimize.
     private final List<Object[]> testParameters = new ArrayList<>(256);
 
-    public void add(final String name, final S value) {
+    public void add(final String name, final S value, final boolean runTest) {
-      if (includes(name)) {
+      testParameters.add(new Object[] {name, value, runTest && includes(name)});
-        testParameters.add(new Object[] {name, value});
-      }
     }
 
     private boolean includes(final String name) {
@@ -104,7 +102,7 @@ public class JsonTestParameters<S, T> {
 
   public static <T> JsonTestParameters<T, T> create(final Class<T> testCaseSpec) {
     return new JsonTestParameters<>(testCaseSpec, testCaseSpec)
-        .generator((name, testCase, collector) -> collector.add(name, testCase));
+        .generator((name, testCase, collector) -> collector.add(name, testCase, true));
   }
 
   public static <S, T> JsonTestParameters<S, T> create(
@@ -187,13 +185,13 @@ public class JsonTestParameters<S, T> {
     for (final String path : paths) {
       final URL url = classLoader.getResource(path);
       checkState(url != null, "Cannot find test directory " + path);
-      Path dir;
+      final Path dir;
       try {
         dir = Paths.get(url.toURI());
       } catch (final URISyntaxException e) {
         throw new RuntimeException("Problem converting URL to URI " + url, e);
       }
-      try (Stream<Path> s = Files.walk(dir)) {
+      try (final Stream<Path> s = Files.walk(dir)) {
         s.map(Path::toFile)
             .filter(f -> f.getPath().endsWith(".json"))
             .filter(f -> !fileExcludes.contains(f.getName()))