Generate mocha JSON output with --matrix (#601)

README.md

@@ -85,7 +85,8 @@ module.exports = {
 | measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] |
 | measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] |
 | modifierWhitelist | *String[]* | `[]` | List of modifier names (ex: "onlyOwner") to exclude from branch measurement. (Useful for modifiers which prepare something instead of acting as a gate.)) |
-| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39] |
+| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39]|
+| mochaJsonOutputPath | *String* | `./mochaOutput.json` | Relative path to write mocha JSON reporter object to. [More...][39]|
 | abiOutputPath | *String* | `./humanReadableAbis.json` | Relative path to write diff-able ABI data to |
 | istanbulFolder | *String* | `./coverage` |  Folder location for Istanbul coverage reports. |
 | istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2]  |

docs/advanced.md

@@ -118,6 +118,15 @@ to guess where bugs might exist in a given codebase.
 Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable
 test names to each line of code to a file named `testMatrix.json` in your project's root.
 
+It also generates a `mochaOutput.json` file which contains test run data similar to that
+generated by mocha's built-in [JSON reporter][27].
+
+In combination these data sets can be passed to Joram's Honig's [tarantula][29] tool which uses
+a fault localization algorithm to generate 'suspiciousness' ratings for each line of
+Solidity code in your project.
+
 [22]: https://github.com/JoranHonig/vertigo#vertigo
 [23]: http://spideruci.org/papers/jones05.pdf
 [25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md
+[27]: https://mochajs.org/api/reporters_json.js.html
+[29]: https://github.com/JoranHonig/tarantula

lib/api.js

@@ -35,6 +35,7 @@ class API {
     this.cwd = config.cwd || process.cwd();
     this.abiOutputPath = config.abiOutputPath || "humanReadableAbis.json";
     this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json";
+    this.mochaJsonOutputPath = config.mochaJsonOutputPath || "mochaOutput.json";
     this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js"
 
     this.defaultHook = () => {};
@@ -321,7 +322,7 @@ class API {
         id
       } = this.instrumenter.instrumentationData[hash];
 
-      if (type === 'line' && hits > 0){
+      if (type === 'line'){
         if (!this.testMatrix[contractPath]){
           this.testMatrix[contractPath] = {};
         }
@@ -329,21 +330,23 @@ class API {
           this.testMatrix[contractPath][id] = [];
         }
 
-        // Search for and exclude duplicate entries
-        let duplicate = false;
-        for (const item of this.testMatrix[contractPath][id]){
-          if (item.title === title && item.file === file){
-            duplicate = true;
-            break;
+        if (hits > 0){
+          // Search for and exclude duplicate entries
+          let duplicate = false;
+          for (const item of this.testMatrix[contractPath][id]){
+            if (item.title === title && item.file === file){
+              duplicate = true;
+              break;
+            }
           }
-        }
 
-        if (!duplicate) {
-          this.testMatrix[contractPath][id].push({title, file});
-        }
+          if (!duplicate) {
+            this.testMatrix[contractPath][id].push({title, file});
+          }
 
-        // Reset line data
-        this.instrumenter.instrumentationData[hash].hits = 0;
+          // Reset line data
+          this.instrumenter.instrumentationData[hash].hits = 0;
+        }
       }
     }
   }
@@ -363,6 +366,11 @@ class API {
     fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' '));
   }
 
+  saveMochaJsonOutput(data){
+    const outputPath = path.join(this.cwd, this.mochaJsonOutputPath);
+    fs.writeFileSync(outputPath, JSON.stringify(data, null, ' '));
+  }
+
   saveHumanReadableAbis(data){
     const abiPath = path.join(this.cwd, this.abiOutputPath);
     fs.writeFileSync(abiPath, JSON.stringify(data, null, ' '));

plugins/resources/matrix.js

@@ -1,7 +1,7 @@
 const mocha = require("mocha");
 const inherits = require("util").inherits;
 const Spec = mocha.reporters.Spec;
-
+const path = require('path');
 
 /**
  * This file adapted from mocha's stats-collector
@@ -40,10 +40,13 @@ function mochaStats(runner) {
 }
 
 /**
- * Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run
- * and collects data about which tests hit which lines of code.
- * This "test matrix" can be used as an input to
+ * Based on the Mocha 'Spec' reporter.
+ *
+ * Watches an Ethereum test suite run and collects data about which tests hit
+ * which lines of code. This "test matrix" can be used as an input to fault localization tools
+ * like: https://github.com/JoranHonig/tarantula
  *
+ * Mocha's JSON reporter output is also generated and saved to a separate file
  *
  * @param {Object} runner  mocha's runner
  * @param {Object} options reporter.options (see README example usage)
@@ -52,6 +55,11 @@ function Matrix(runner, options) {
   // Spec reporter
   Spec.call(this, runner, options);
 
+  const self = this;
+  const tests = [];
+  const failures = [];
+  const passes = [];
+
   // Initialize stats for Mocha 6+ epilogue
   if (!runner.stats) {
     mochaStats(runner);
@@ -60,7 +68,73 @@ function Matrix(runner, options) {
 
   runner.on("test end", (info) => {
     options.reporterOptions.collectTestMatrixData(info);
+    tests.push(info);
+  });
+
+  runner.on('pass', function(info) {
+    passes.push(info)
+  })
+  runner.on('fail', function(info) {
+    failures.push(info)
+  });
+
+  runner.once('end', function() {
+    delete self.stats.start;
+    delete self.stats.end;
+    delete self.stats.duration;
+
+    var obj = {
+      stats: self.stats,
+      tests: tests.map(clean),
+      failures: failures.map(clean),
+      passes: passes.map(clean)
+    };
+    runner.testResults = obj;
+    options.reporterOptions.saveMochaJsonOutput(obj)
   });
+
+  // >>>>>>>>>>>>>>>>>>>>>>>>>
+  // Mocha JSON Reporter Utils
+  // Code taken from:
+  // https://mochajs.org/api/reporters_json.js.html
+  // >>>>>>>>>>>>>>>>>>>>>>>>>
+  function clean(info) {
+    var err = info.err || {};
+    if (err instanceof Error) {
+      err = errorJSON(err);
+    }
+    return {
+      title: info.title,
+      fullTitle: info.fullTitle(),
+      file: path.relative(options.reporterOptions.cwd, info.file),
+      currentRetry: info.currentRetry(),
+      err: cleanCycles(err)
+    };
+  }
+
+  function cleanCycles(obj) {
+    var cache = [];
+    return JSON.parse(
+      JSON.stringify(obj, function(key, value) {
+        if (typeof value === 'object' && value !== null) {
+          if (cache.indexOf(value) !== -1) {
+            // Instead of going in a circle, we'll print [object Object]
+            return '' + value;
+          }
+          cache.push(value);
+        }
+        return value;
+      })
+    );
+  }
+
+  function errorJSON(err) {
+    var res = {};
+    Object.getOwnPropertyNames(err).forEach(function(key) {
+      res[key] = err[key];
+    }, err);
+    return res;
+  }
 }
 
 /**

plugins/resources/nomiclabs.utils.js

@@ -175,7 +175,9 @@ function collectTestMatrixData(args, env, api){
     mochaConfig = env.config.mocha || {};
     mochaConfig.reporter = api.matrixReporterPath;
     mochaConfig.reporterOptions = {
-      collectTestMatrixData: api.collectTestMatrixData.bind(api)
+      collectTestMatrixData: api.collectTestMatrixData.bind(api),
+      saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api),
+      cwd: api.cwd
     }
     env.config.mocha = mochaConfig;
   }

plugins/resources/truffle.utils.js

@@ -238,7 +238,9 @@ function collectTestMatrixData(config, api){
     config.mocha = config.mocha || {};
     config.mocha.reporter = api.matrixReporterPath;
     config.mocha.reporterOptions = {
-      collectTestMatrixData: api.collectTestMatrixData.bind(api)
+      collectTestMatrixData: api.collectTestMatrixData.bind(api),
+      saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api),
+      cwd: api.cwd
     }
   }
 }

test/integration/projects/matrix/.solcover.js

@@ -9,6 +9,7 @@ module.exports = {
   // "solidity-coverage/plugins/resources/matrix.js"
   matrixReporterPath: reporterPath,
   matrixOutputPath: "alternateTestMatrix.json",
+  mochaJsonOutputPath: "alternateMochaOutput.json",
 
   skipFiles: ['Migrations.sol'],
   silent: process.env.SILENT ? true : false,

test/integration/projects/matrix/contracts/MatrixA.sol

@@ -14,4 +14,8 @@ contract MatrixA {
     uint y = 5;
     return y;
   }
+
+  function unhit() public {
+    uint z = 7;
+  }
 }

test/integration/projects/matrix/expectedMochaOutput.json

@@ -0,0 +1,99 @@
+{
+ "stats": {
+  "suites": 2,
+  "tests": 6,
+  "passes": 6,
+  "pending": 0,
+  "failures": 0
+ },
+ "tests": [
+  {
+   "title": "sends to A",
+   "fullTitle": "Contract: Matrix A and B sends to A",
+   "file": "test/matrix_a_b.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "sends to A",
+   "fullTitle": "Contract: Matrix A and B sends to A",
+   "file": "test/matrix_a_b.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "calls B",
+   "fullTitle": "Contract: Matrix A and B calls B",
+   "file": "test/matrix_a_b.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "sends to B",
+   "fullTitle": "Contract: Matrix A and B sends to B",
+   "file": "test/matrix_a_b.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "sends",
+   "fullTitle": "Contract: MatrixA sends",
+   "file": "test/matrix_a.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "calls",
+   "fullTitle": "Contract: MatrixA calls",
+   "file": "test/matrix_a.js",
+   "currentRetry": 0,
+   "err": {}
+  }
+ ],
+ "failures": [],
+ "passes": [
+  {
+   "title": "sends to A",
+   "fullTitle": "Contract: Matrix A and B sends to A",
+   "file": "test/matrix_a_b.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "sends to A",
+   "fullTitle": "Contract: Matrix A and B sends to A",
+   "file": "test/matrix_a_b.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "calls B",
+   "fullTitle": "Contract: Matrix A and B calls B",
+   "file": "test/matrix_a_b.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "sends to B",
+   "fullTitle": "Contract: Matrix A and B sends to B",
+   "file": "test/matrix_a_b.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "sends",
+   "fullTitle": "Contract: MatrixA sends",
+   "file": "test/matrix_a.js",
+   "currentRetry": 0,
+   "err": {}
+  },
+  {
+   "title": "calls",
+   "fullTitle": "Contract: MatrixA calls",
+   "file": "test/matrix_a.js",
+   "currentRetry": 0,
+   "err": {}
+  }
+ ]
+}
+

test/integration/projects/matrix/expectedTestMatrixHardhat.json

@@ -21,7 +21,8 @@
     "title": "calls",
     "file": "test/matrix_a.js"
    }
-  ]
+  ],
+  "19": []
  },
  "contracts/MatrixB.sol": {
   "10": [
@@ -43,4 +44,5 @@
    }
   ]
  }
-}
+}
+

test/units/hardhat/flags.js

@@ -184,12 +184,18 @@ describe('Hardhat Plugin: command line options', function() {
     await this.env.run("coverage", taskArgs);
 
     // Integration test checks output path configurabililty
-    const altPath = path.join(process.cwd(), './alternateTestMatrix.json');
-    const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
-    const producedMatrix = require(altPath)
-    const expectedMatrix = require(expPath);
+    const altMatrixPath = path.join(process.cwd(), './alternateTestMatrix.json');
+    const expMatrixPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
+    const altMochaPath = path.join(process.cwd(), './alternateMochaOutput.json');
+    const expMochaPath = path.join(process.cwd(), './expectedMochaOutput.json');
+
+    const producedMatrix = require(altMatrixPath)
+    const expectedMatrix = require(expMatrixPath);
+    const producedMochaOutput = require(altMochaPath);
+    const expectedMochaOutput = require(expMochaPath);
 
     assert.deepEqual(producedMatrix, expectedMatrix);
+    assert.deepEqual(producedMochaOutput, expectedMochaOutput);
   });
 
   it('--abi', async function(){

test/units/truffle/flags.js

@@ -266,13 +266,18 @@ describe('Truffle Plugin: command line options', function() {
     await plugin(truffleConfig);
 
     // Integration test checks output path configurabililty
-    const altPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json'));
-    const expPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json'));
+    const altMatrixPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json'));
+    const expMatrixPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json'));
+    const altMochaPath = path.join(process.cwd(), mock.pathToTemp('./alternateMochaOutput.json'));
+    const expMochaPath = path.join(process.cwd(), mock.pathToTemp('./expectedMochaOutput.json'));
 
-    const producedMatrix = require(altPath)
-    const expectedMatrix = require(expPath);
+    const producedMatrix = require(altMatrixPath)
+    const expectedMatrix = require(expMatrixPath);
+    const producedMochaOutput = require(altMochaPath);
+    const expectedMochaOutput = require(expMochaPath);
 
     assert.deepEqual(producedMatrix, expectedMatrix);
+    assert.deepEqual(producedMochaOutput, expectedMochaOutput);
     process.env.TRUFFLE_TEST = false;
   });