diff --git a/README.md b/README.md index f42ac84..4dd4a45 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,8 @@ module.exports = { | measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] | | measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] | | modifierWhitelist | *String[]* | `[]` | List of modifier names (ex: "onlyOwner") to exclude from branch measurement. (Useful for modifiers which prepare something instead of acting as a gate.)) | -| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39] | +| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39]| +| mochaJsonOutputPath | *String* | `./mochaOutput.json` | Relative path to write mocha JSON reporter object to. [More...][39]| | abiOutputPath | *String* | `./humanReadableAbis.json` | Relative path to write diff-able ABI data to | | istanbulFolder | *String* | `./coverage` | Folder location for Istanbul coverage reports. | | istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2] | diff --git a/docs/advanced.md b/docs/advanced.md index 22508e1..724c7a0 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -118,6 +118,15 @@ to guess where bugs might exist in a given codebase. Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable test names to each line of code to a file named `testMatrix.json` in your project's root. +It also generates a `mochaOutput.json` file which contains test run data similar to that +generated by mocha's built-in [JSON reporter][27]. + +In combination these data sets can be passed to Joram's Honig's [tarantula][29] tool which uses +a fault localization algorithm to generate 'suspiciousness' ratings for each line of +Solidity code in your project. + [22]: https://github.com/JoranHonig/vertigo#vertigo [23]: http://spideruci.org/papers/jones05.pdf [25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md +[27]: https://mochajs.org/api/reporters_json.js.html +[29]: https://github.com/JoranHonig/tarantula diff --git a/lib/api.js b/lib/api.js index ef31254..94bb740 100644 --- a/lib/api.js +++ b/lib/api.js @@ -35,6 +35,7 @@ class API { this.cwd = config.cwd || process.cwd(); this.abiOutputPath = config.abiOutputPath || "humanReadableAbis.json"; this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json"; + this.mochaJsonOutputPath = config.mochaJsonOutputPath || "mochaOutput.json"; this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js" this.defaultHook = () => {}; @@ -321,7 +322,7 @@ class API { id } = this.instrumenter.instrumentationData[hash]; - if (type === 'line' && hits > 0){ + if (type === 'line'){ if (!this.testMatrix[contractPath]){ this.testMatrix[contractPath] = {}; } @@ -329,21 +330,23 @@ class API { this.testMatrix[contractPath][id] = []; } - // Search for and exclude duplicate entries - let duplicate = false; - for (const item of this.testMatrix[contractPath][id]){ - if (item.title === title && item.file === file){ - duplicate = true; - break; + if (hits > 0){ + // Search for and exclude duplicate entries + let duplicate = false; + for (const item of this.testMatrix[contractPath][id]){ + if (item.title === title && item.file === file){ + duplicate = true; + break; + } } - } - if (!duplicate) { - this.testMatrix[contractPath][id].push({title, file}); - } + if (!duplicate) { + this.testMatrix[contractPath][id].push({title, file}); + } - // Reset line data - this.instrumenter.instrumentationData[hash].hits = 0; + // Reset line data + this.instrumenter.instrumentationData[hash].hits = 0; + } } } } @@ -363,6 +366,11 @@ class API { fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' ')); } + saveMochaJsonOutput(data){ + const outputPath = path.join(this.cwd, this.mochaJsonOutputPath); + fs.writeFileSync(outputPath, JSON.stringify(data, null, ' ')); + } + saveHumanReadableAbis(data){ const abiPath = path.join(this.cwd, this.abiOutputPath); fs.writeFileSync(abiPath, JSON.stringify(data, null, ' ')); diff --git a/plugins/resources/matrix.js b/plugins/resources/matrix.js index 8701e53..7c02740 100644 --- a/plugins/resources/matrix.js +++ b/plugins/resources/matrix.js @@ -1,7 +1,7 @@ const mocha = require("mocha"); const inherits = require("util").inherits; const Spec = mocha.reporters.Spec; - +const path = require('path'); /** * This file adapted from mocha's stats-collector @@ -40,10 +40,13 @@ function mochaStats(runner) { } /** - * Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run - * and collects data about which tests hit which lines of code. - * This "test matrix" can be used as an input to + * Based on the Mocha 'Spec' reporter. + * + * Watches an Ethereum test suite run and collects data about which tests hit + * which lines of code. This "test matrix" can be used as an input to fault localization tools + * like: https://github.com/JoranHonig/tarantula * + * Mocha's JSON reporter output is also generated and saved to a separate file * * @param {Object} runner mocha's runner * @param {Object} options reporter.options (see README example usage) @@ -52,6 +55,11 @@ function Matrix(runner, options) { // Spec reporter Spec.call(this, runner, options); + const self = this; + const tests = []; + const failures = []; + const passes = []; + // Initialize stats for Mocha 6+ epilogue if (!runner.stats) { mochaStats(runner); @@ -60,7 +68,73 @@ function Matrix(runner, options) { runner.on("test end", (info) => { options.reporterOptions.collectTestMatrixData(info); + tests.push(info); + }); + + runner.on('pass', function(info) { + passes.push(info) + }) + runner.on('fail', function(info) { + failures.push(info) + }); + + runner.once('end', function() { + delete self.stats.start; + delete self.stats.end; + delete self.stats.duration; + + var obj = { + stats: self.stats, + tests: tests.map(clean), + failures: failures.map(clean), + passes: passes.map(clean) + }; + runner.testResults = obj; + options.reporterOptions.saveMochaJsonOutput(obj) }); + + // >>>>>>>>>>>>>>>>>>>>>>>>> + // Mocha JSON Reporter Utils + // Code taken from: + // https://mochajs.org/api/reporters_json.js.html + // >>>>>>>>>>>>>>>>>>>>>>>>> + function clean(info) { + var err = info.err || {}; + if (err instanceof Error) { + err = errorJSON(err); + } + return { + title: info.title, + fullTitle: info.fullTitle(), + file: path.relative(options.reporterOptions.cwd, info.file), + currentRetry: info.currentRetry(), + err: cleanCycles(err) + }; + } + + function cleanCycles(obj) { + var cache = []; + return JSON.parse( + JSON.stringify(obj, function(key, value) { + if (typeof value === 'object' && value !== null) { + if (cache.indexOf(value) !== -1) { + // Instead of going in a circle, we'll print [object Object] + return '' + value; + } + cache.push(value); + } + return value; + }) + ); + } + + function errorJSON(err) { + var res = {}; + Object.getOwnPropertyNames(err).forEach(function(key) { + res[key] = err[key]; + }, err); + return res; + } } /** diff --git a/plugins/resources/nomiclabs.utils.js b/plugins/resources/nomiclabs.utils.js index 21df995..1629b73 100644 --- a/plugins/resources/nomiclabs.utils.js +++ b/plugins/resources/nomiclabs.utils.js @@ -175,7 +175,9 @@ function collectTestMatrixData(args, env, api){ mochaConfig = env.config.mocha || {}; mochaConfig.reporter = api.matrixReporterPath; mochaConfig.reporterOptions = { - collectTestMatrixData: api.collectTestMatrixData.bind(api) + collectTestMatrixData: api.collectTestMatrixData.bind(api), + saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api), + cwd: api.cwd } env.config.mocha = mochaConfig; } diff --git a/plugins/resources/truffle.utils.js b/plugins/resources/truffle.utils.js index f1c9bd3..7d64059 100644 --- a/plugins/resources/truffle.utils.js +++ b/plugins/resources/truffle.utils.js @@ -238,7 +238,9 @@ function collectTestMatrixData(config, api){ config.mocha = config.mocha || {}; config.mocha.reporter = api.matrixReporterPath; config.mocha.reporterOptions = { - collectTestMatrixData: api.collectTestMatrixData.bind(api) + collectTestMatrixData: api.collectTestMatrixData.bind(api), + saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api), + cwd: api.cwd } } } diff --git a/test/integration/projects/matrix/.solcover.js b/test/integration/projects/matrix/.solcover.js index 992f82b..7c3db8c 100644 --- a/test/integration/projects/matrix/.solcover.js +++ b/test/integration/projects/matrix/.solcover.js @@ -9,6 +9,7 @@ module.exports = { // "solidity-coverage/plugins/resources/matrix.js" matrixReporterPath: reporterPath, matrixOutputPath: "alternateTestMatrix.json", + mochaJsonOutputPath: "alternateMochaOutput.json", skipFiles: ['Migrations.sol'], silent: process.env.SILENT ? true : false, diff --git a/test/integration/projects/matrix/contracts/MatrixA.sol b/test/integration/projects/matrix/contracts/MatrixA.sol index aeac9bc..4b76c36 100644 --- a/test/integration/projects/matrix/contracts/MatrixA.sol +++ b/test/integration/projects/matrix/contracts/MatrixA.sol @@ -14,4 +14,8 @@ contract MatrixA { uint y = 5; return y; } + + function unhit() public { + uint z = 7; + } } diff --git a/test/integration/projects/matrix/expectedMochaOutput.json b/test/integration/projects/matrix/expectedMochaOutput.json new file mode 100644 index 0000000..f2aa172 --- /dev/null +++ b/test/integration/projects/matrix/expectedMochaOutput.json @@ -0,0 +1,99 @@ +{ + "stats": { + "suites": 2, + "tests": 6, + "passes": 6, + "pending": 0, + "failures": 0 + }, + "tests": [ + { + "title": "sends to A", + "fullTitle": "Contract: Matrix A and B sends to A", + "file": "test/matrix_a_b.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "sends to A", + "fullTitle": "Contract: Matrix A and B sends to A", + "file": "test/matrix_a_b.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "calls B", + "fullTitle": "Contract: Matrix A and B calls B", + "file": "test/matrix_a_b.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "sends to B", + "fullTitle": "Contract: Matrix A and B sends to B", + "file": "test/matrix_a_b.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "sends", + "fullTitle": "Contract: MatrixA sends", + "file": "test/matrix_a.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "calls", + "fullTitle": "Contract: MatrixA calls", + "file": "test/matrix_a.js", + "currentRetry": 0, + "err": {} + } + ], + "failures": [], + "passes": [ + { + "title": "sends to A", + "fullTitle": "Contract: Matrix A and B sends to A", + "file": "test/matrix_a_b.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "sends to A", + "fullTitle": "Contract: Matrix A and B sends to A", + "file": "test/matrix_a_b.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "calls B", + "fullTitle": "Contract: Matrix A and B calls B", + "file": "test/matrix_a_b.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "sends to B", + "fullTitle": "Contract: Matrix A and B sends to B", + "file": "test/matrix_a_b.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "sends", + "fullTitle": "Contract: MatrixA sends", + "file": "test/matrix_a.js", + "currentRetry": 0, + "err": {} + }, + { + "title": "calls", + "fullTitle": "Contract: MatrixA calls", + "file": "test/matrix_a.js", + "currentRetry": 0, + "err": {} + } + ] +} + diff --git a/test/integration/projects/matrix/expectedTestMatrixHardhat.json b/test/integration/projects/matrix/expectedTestMatrixHardhat.json index 5afc260..6d41c88 100644 --- a/test/integration/projects/matrix/expectedTestMatrixHardhat.json +++ b/test/integration/projects/matrix/expectedTestMatrixHardhat.json @@ -21,7 +21,8 @@ "title": "calls", "file": "test/matrix_a.js" } - ] + ], + "19": [] }, "contracts/MatrixB.sol": { "10": [ @@ -43,4 +44,5 @@ } ] } -} \ No newline at end of file +} + diff --git a/test/units/hardhat/flags.js b/test/units/hardhat/flags.js index c3c99d5..c044f5a 100644 --- a/test/units/hardhat/flags.js +++ b/test/units/hardhat/flags.js @@ -184,12 +184,18 @@ describe('Hardhat Plugin: command line options', function() { await this.env.run("coverage", taskArgs); // Integration test checks output path configurabililty - const altPath = path.join(process.cwd(), './alternateTestMatrix.json'); - const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json'); - const producedMatrix = require(altPath) - const expectedMatrix = require(expPath); + const altMatrixPath = path.join(process.cwd(), './alternateTestMatrix.json'); + const expMatrixPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json'); + const altMochaPath = path.join(process.cwd(), './alternateMochaOutput.json'); + const expMochaPath = path.join(process.cwd(), './expectedMochaOutput.json'); + + const producedMatrix = require(altMatrixPath) + const expectedMatrix = require(expMatrixPath); + const producedMochaOutput = require(altMochaPath); + const expectedMochaOutput = require(expMochaPath); assert.deepEqual(producedMatrix, expectedMatrix); + assert.deepEqual(producedMochaOutput, expectedMochaOutput); }); it('--abi', async function(){ diff --git a/test/units/truffle/flags.js b/test/units/truffle/flags.js index 234f186..508c7f3 100644 --- a/test/units/truffle/flags.js +++ b/test/units/truffle/flags.js @@ -266,13 +266,18 @@ describe('Truffle Plugin: command line options', function() { await plugin(truffleConfig); // Integration test checks output path configurabililty - const altPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json')); - const expPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json')); + const altMatrixPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json')); + const expMatrixPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json')); + const altMochaPath = path.join(process.cwd(), mock.pathToTemp('./alternateMochaOutput.json')); + const expMochaPath = path.join(process.cwd(), mock.pathToTemp('./expectedMochaOutput.json')); - const producedMatrix = require(altPath) - const expectedMatrix = require(expPath); + const producedMatrix = require(altMatrixPath) + const expectedMatrix = require(expMatrixPath); + const producedMochaOutput = require(altMochaPath); + const expectedMochaOutput = require(expMochaPath); assert.deepEqual(producedMatrix, expectedMatrix); + assert.deepEqual(producedMochaOutput, expectedMochaOutput); process.env.TRUFFLE_TEST = false; });