diff --git a/README.md b/README.md index 0ffe838..ed90615 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ A working example can be found at [openzeppelin-contracts, here.][35] | solcoverjs | `--solcoverjs ./../.solcover.js` | Relative path from working directory to config. Useful for monorepo packages that share settings. (Path must be "./" prefixed) | | network | `--network development` | Use network settings defined in the Truffle or Buidler config | | temp[*][14] | `--temp build` | :warning: **Caution** :warning: Path to a *disposable* folder to store compilation artifacts in. Useful when your test setup scripts include hard-coded paths to a build directory. [More...][14] | +| matrix | `--matrix` | Generate a JSON object that maps which mocha tests hit which lines of code. (Useful +as an input for some fuzzing, mutation testing and fault-localization algorithms.) [More...][39]| [* Advanced use][14] @@ -99,6 +101,7 @@ module.exports = { | measureStatementCoverage | *boolean* | `true` | Computes statement (in addition to line) coverage. [More...][34] | | measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] | | measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] | +| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39] | | istanbulFolder | *String* | `./coverage` | Folder location for Istanbul coverage reports. | | istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2] | | mocha | *Object* | `{ }` | [Mocha options][3] to merge into existing mocha config. `grep` and `invert` are useful for skipping certain tests under coverage using tags in the test descriptions.| @@ -229,6 +232,7 @@ $ yarn [36]: https://hardhat.org/ [37]: https://github.com/sc-forks/solidity-coverage/blob/master/HARDHAT_README.md [38]: https://github.com/sindresorhus/globby#globbing-patterns +[39]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#generating-a-test-matrix [1001]: https://docs.soliditylang.org/en/v0.8.0/using-the-compiler.html#input-description [1002]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/faq.md#running-out-of-stack diff --git a/docs/advanced.md b/docs/advanced.md index f182002..22508e1 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -106,3 +106,18 @@ Setting the `measureStatementCoverage` and/or `measureFunctionCoverage` options improve performance, lower the cost of execution and minimize complications that arise from `solc`'s limits on how large the compilation payload can be. +## Generating a test matrix + +Some advanced testing strategies benefit from knowing which tests in a suite hit a +specific line of code. Examples include: ++ [mutation testing][22], where this data lets you select the correct subset of tests to check +a mutation with. ++ [fault localization techniques][23], where the complete data set is a key input to algorithms that try +to guess where bugs might exist in a given codebase. + +Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable +test names to each line of code to a file named `testMatrix.json` in your project's root. + +[22]: https://github.com/JoranHonig/vertigo#vertigo +[23]: http://spideruci.org/papers/jones05.pdf +[25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md diff --git a/docs/matrix.md b/docs/matrix.md new file mode 100644 index 0000000..585186c --- /dev/null +++ b/docs/matrix.md @@ -0,0 +1,421 @@ +### Test Matrix Example + +An example of output written to the file `./testMatrix.json` when coverage +is run with the `--matrix` cli flag. (Source project: [sc-forks/hardhat-e2e][1]) + +[1]: https://github.com/sc-forks/hardhat-e2e + + +```js +// Paths are relative to the project root directory +{ + // Solidity file name + "contracts/EtherRouter/EtherRouter.sol": { + + // Line number + "23": [ + { + // Grep-able mocha test title + "title": "Resolves methods routed through an EtherRouter proxy", + + // Selectable mocha test file + "file": "test/etherrouter.js" + } + ], + "42": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ], + "45": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ], + "61": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ] + }, + "contracts/EtherRouter/Factory.sol": { + "19": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ] + }, + "contracts/EtherRouter/Resolver.sol": { + "22": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ], + "26": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ], + "30": [ + { + "title": "Resolves methods routed through an EtherRouter proxy", + "file": "test/etherrouter.js" + } + ] + }, + "contracts/MetaCoin.sol": { + "16": [ + { + "title": "should put 10000 MetaCoin in the first account", + "file": "test/metacoin.js" + }, + { + "title": "should call a function that depends on a linked library", + "file": "test/metacoin.js" + }, + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + }, + { + "title": "a and b", + "file": "test/multicontract.js" + } + ], + "20": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "21": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "22": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "23": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "24": [ + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ], + "28": [ + { + "title": "should call a function that depends on a linked library", + "file": "test/metacoin.js" + } + ], + "32": [ + { + "title": "should put 10000 MetaCoin in the first account", + "file": "test/metacoin.js" + }, + { + "title": "should call a function that depends on a linked library", + "file": "test/metacoin.js" + }, + { + "title": "should send coin correctly", + "file": "test/metacoin.js" + } + ] + }, + "contracts/ConvertLib.sol": { + "6": [ + { + "title": "should call a function that depends on a linked library", + "file": "test/metacoin.js" + } + ] + }, + "contracts/MultiContractFile.sol": { + "7": [ + { + "title": "a and b", + "file": "test/multicontract.js" + }, + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + } + ], + "15": [ + { + "title": "a and b", + "file": "test/multicontract.js" + } + ] + }, + "contracts/VariableConstructor.sol": { + "8": [ + { + "title": "should should initialize with a short string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a medium length string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a long string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a random length string", + "file": "test/variableconstructor.js" + } + ] + }, + "contracts/VariableCosts.sol": { + "13": [ + { + "title": "should should initialize with a short string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a medium length string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a long string", + "file": "test/variableconstructor.js" + }, + { + "title": "should should initialize with a random length string", + "file": "test/variableconstructor.js" + }, + { + "title": "should add one", + "file": "test/variablecosts.js" + }, + { + "title": "should add three", + "file": "test/variablecosts.js" + }, + { + "title": "should add even 5!", + "file": "test/variablecosts.js" + }, + { + "title": "should delete one", + "file": "test/variablecosts.js" + }, + { + "title": "should delete three", + "file": "test/variablecosts.js" + }, + { + "title": "should delete five", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + }, + { + "title": "should set a random length string", + "file": "test/variablecosts.js" + }, + { + "title": "methods that do not throw", + "file": "test/variablecosts.js" + }, + { + "title": "methods that throw", + "file": "test/variablecosts.js" + }, + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + }, + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + } + ], + "29": [ + { + "title": "should add one", + "file": "test/variablecosts.js" + }, + { + "title": "should add three", + "file": "test/variablecosts.js" + }, + { + "title": "should add even 5!", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + } + ], + "30": [ + { + "title": "should add one", + "file": "test/variablecosts.js" + }, + { + "title": "should add three", + "file": "test/variablecosts.js" + }, + { + "title": "should add even 5!", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + } + ], + "34": [ + { + "title": "should delete one", + "file": "test/variablecosts.js" + }, + { + "title": "should delete three", + "file": "test/variablecosts.js" + }, + { + "title": "should delete five", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + } + ], + "35": [ + { + "title": "should delete one", + "file": "test/variablecosts.js" + }, + { + "title": "should delete three", + "file": "test/variablecosts.js" + }, + { + "title": "should delete five", + "file": "test/variablecosts.js" + }, + { + "title": "should add five and delete one", + "file": "test/variablecosts.js" + } + ], + "43": [ + { + "title": "should set a random length string", + "file": "test/variablecosts.js" + } + ], + "47": [ + { + "title": "methods that do not throw", + "file": "test/variablecosts.js" + }, + { + "title": "methods that throw", + "file": "test/variablecosts.js" + } + ], + "48": [ + { + "title": "methods that do not throw", + "file": "test/variablecosts.js" + } + ], + "52": [ + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + } + ], + "53": [ + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + } + ], + "54": [ + { + "title": "methods that call methods in other contracts", + "file": "test/variablecosts.js" + } + ] + }, + "contracts/Wallets/Wallet.sol": { + "8": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ], + "12": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ], + "17": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ], + "22": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ], + "23": [ + { + "title": "should allow contracts to have identically named methods", + "file": "test/variablecosts.js" + }, + { + "title": "should should allow transfers and sends", + "file": "test/wallet.js" + } + ] + } +} +``` \ No newline at end of file diff --git a/lib/api.js b/lib/api.js index 33ce2b5..d0f89c8 100644 --- a/lib/api.js +++ b/lib/api.js @@ -20,6 +20,7 @@ class API { constructor(config={}) { this.validator = new ConfigValidator() this.config = config || {}; + this.testMatrix = {}; // Validate this.validator.validate(this.config); @@ -30,6 +31,8 @@ class API { this.testsErrored = false; this.cwd = config.cwd || process.cwd(); + this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json"; + this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js" this.defaultHook = () => {}; this.onServerReady = config.onServerReady || this.defaultHook; @@ -296,6 +299,52 @@ class API { } }*/ + // ========================== + // Test Matrix Data Collector + // ========================== + /** + * @param {Object} testInfo Mocha object passed to reporter 'test end' event + */ + collectTestMatrixData(testInfo){ + const hashes = Object.keys(this.instrumenter.instrumentationData); + const title = testInfo.title; + const file = path.relative(this.cwd, testInfo.file); + + for (const hash of hashes){ + const { + contractPath, + hits, + type, + id + } = this.instrumenter.instrumentationData[hash]; + + if (type === 'line' && hits > 0){ + if (!this.testMatrix[contractPath]){ + this.testMatrix[contractPath] = {}; + } + if (!this.testMatrix[contractPath][id]){ + this.testMatrix[contractPath][id] = []; + } + + // Search for and exclude duplicate entries + let duplicate = false; + for (const item of this.testMatrix[contractPath][id]){ + if (item.title === title && item.file === file){ + duplicate = true; + break; + } + } + + if (!duplicate) { + this.testMatrix[contractPath][id].push({title, file}); + } + + // Reset line data + this.instrumenter.instrumentationData[hash].hits = 0; + } + } + } + // ======== // File I/O // ======== @@ -305,6 +354,12 @@ class API { fs.writeFileSync(covPath, JSON.stringify(data)); } + saveTestMatrix(){ + const matrixPath = path.join(this.cwd, this.matrixOutputPath); + const mapping = this.makeKeysRelative(this.testMatrix, this.cwd); + fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' ')); + } + // ===== // Paths // ===== diff --git a/package.json b/package.json index a8555d1..df695f7 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "scripts": { "nyc": "SILENT=true nyc --exclude '**/sc_temp/**' --exclude '**/test/**'", "test": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit", - "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit", + "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' --exclude 'plugins/resources/matrix.js' -- mocha test/units/* --timeout 100000 --no-warnings --exit", "test:debug": "node --max-old-space-size=4096 ./node_modules/.bin/mocha test/units/* --timeout 100000 --no-warnings --exit", "netlify": "./scripts/run-netlify.sh" }, @@ -35,6 +35,7 @@ "globby": "^10.0.1", "jsonschema": "^1.2.4", "lodash": "^4.17.15", + "mocha": "7.1.2", "node-emoji": "^1.10.0", "pify": "^4.0.1", "recursive-readdir": "^2.2.2", @@ -60,7 +61,6 @@ "ganache-cli": "6.12.2", "hardhat": "^2.9.3", "hardhat-gas-reporter": "^1.0.1", - "mocha": "5.2.0", "nyc": "^14.1.1", "solc": "^0.7.5", "truffle": "5.1.43", diff --git a/plugins/hardhat.plugin.js b/plugins/hardhat.plugin.js index d695be3..6342570 100644 --- a/plugins/hardhat.plugin.js +++ b/plugins/hardhat.plugin.js @@ -97,6 +97,7 @@ task("coverage", "Generates a code coverage report for tests") .addOptionalParam("testfiles", ui.flags.file, "", types.string) .addOptionalParam("solcoverjs", ui.flags.solcoverjs, "", types.string) .addOptionalParam('temp', ui.flags.temp, "", types.string) + .addFlag('matrix', ui.flags.testMatrix) .setAction(async function(args, env){ const API = require('./../lib/api'); @@ -232,6 +233,9 @@ task("coverage", "Generates a code coverage report for tests") ? nomiclabsUtils.getTestFilePaths(args.testfiles) : []; + // Optionally collect tests-per-line-of-code data + nomiclabsUtils.collectTestMatrixData(args, env, api); + try { failedTests = await env.run(TASK_TEST, {testFiles: testfiles}) } catch (e) { @@ -239,10 +243,13 @@ task("coverage", "Generates a code coverage report for tests") } await api.onTestsComplete(config); - // ======== - // Istanbul - // ======== - await api.report(); + // ================================= + // Output (Istanbul or Test Matrix) + // ================================= + (args.matrix) + ? await api.saveTestMatrix() + : await api.report(); + await api.onIstanbulComplete(config); } catch(e) { diff --git a/plugins/resources/matrix.js b/plugins/resources/matrix.js new file mode 100644 index 0000000..8701e53 --- /dev/null +++ b/plugins/resources/matrix.js @@ -0,0 +1,71 @@ +const mocha = require("mocha"); +const inherits = require("util").inherits; +const Spec = mocha.reporters.Spec; + + +/** + * This file adapted from mocha's stats-collector + * https://github.com/mochajs/mocha/blob/54475eb4ca35a2c9044a1b8c59a60f09c73e6c01/lib/stats-collector.js#L1-L83 + */ +const Date = global.Date; + +/** + * Provides stats such as test duration, number of tests passed / failed etc., by + * listening for events emitted by `runner`. + */ +function mochaStats(runner) { + const stats = { + suites: 0, + tests: 0, + passes: 0, + pending: 0, + failures: 0 + }; + + if (!runner) throw new Error("Missing runner argument"); + + runner.stats = stats; + + runner.on("pass", () => stats.passes++); + runner.on("fail", () => stats.failures++); + runner.on("pending", () => stats.pending++); + runner.on("test end", () => stats.tests++); + + runner.once("start", () => (stats.start = new Date())); + + runner.once("end", function() { + stats.end = new Date(); + stats.duration = stats.end - stats.start; + }); +} + +/** + * Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run + * and collects data about which tests hit which lines of code. + * This "test matrix" can be used as an input to + * + * + * @param {Object} runner mocha's runner + * @param {Object} options reporter.options (see README example usage) + */ +function Matrix(runner, options) { + // Spec reporter + Spec.call(this, runner, options); + + // Initialize stats for Mocha 6+ epilogue + if (!runner.stats) { + mochaStats(runner); + this.stats = runner.stats; + } + + runner.on("test end", (info) => { + options.reporterOptions.collectTestMatrixData(info); + }); +} + +/** + * Inherit from `Base.prototype`. + */ +inherits(Matrix, Spec); + +module.exports = Matrix; \ No newline at end of file diff --git a/plugins/resources/nomiclabs.ui.js b/plugins/resources/nomiclabs.ui.js index 770fcba..7027095 100644 --- a/plugins/resources/nomiclabs.ui.js +++ b/plugins/resources/nomiclabs.ui.js @@ -8,7 +8,9 @@ class PluginUI extends UI { super(log); this.flags = { - file: `Path (or glob) defining a subset of tests to run`, + testfiles: `Path (or glob) defining a subset of tests to run`, + + testMatrix: `Generate a json object which maps which unit tests hit which lines of code.`, solcoverjs: `Relative path from working directory to config. ` + `Useful for monorepo packages that share settings.`, @@ -16,7 +18,6 @@ class PluginUI extends UI { temp: `Path to a disposable folder to store compilation artifacts in. ` + `Useful when your test setup scripts include hard-coded paths to ` + `a build directory.`, - } } diff --git a/plugins/resources/nomiclabs.utils.js b/plugins/resources/nomiclabs.utils.js index fde8c30..4f2ed1c 100644 --- a/plugins/resources/nomiclabs.utils.js +++ b/plugins/resources/nomiclabs.utils.js @@ -37,6 +37,7 @@ function normalizeConfig(config, args={}){ config.logger = config.logger ? config.logger : {log: null}; config.solcoverjs = args.solcoverjs config.gasReporter = { enabled: false } + config.matrix = args.matrix; try { const hardhatPackage = require('hardhat/package.json'); @@ -165,6 +166,21 @@ function configureHttpProvider(networkConfig, api, ui){ networkConfig.url = `http://${api.host}:${api.port}`; } +/** + * Configures mocha to generate a json object which maps which tests + * hit which lines of code. + */ +function collectTestMatrixData(args, env, api){ + if (args.matrix){ + mochaConfig = env.config.mocha || {}; + mochaConfig.reporter = api.matrixReporterPath; + mochaConfig.reporterOptions = { + collectTestMatrixData: api.collectTestMatrixData.bind(api) + } + env.config.mocha = mochaConfig; + } +} + /** * Sets the default `from` account field in the network that will be used. * This needs to be done after accounts are fetched from the launched client. @@ -216,6 +232,7 @@ module.exports = { setupBuidlerNetwork, setupHardhatNetwork, getTestFilePaths, - setNetworkFrom + setNetworkFrom, + collectTestMatrixData } diff --git a/plugins/resources/plugin.utils.js b/plugins/resources/plugin.utils.js index 8ef7310..7bdeaa9 100644 --- a/plugins/resources/plugin.utils.js +++ b/plugins/resources/plugin.utils.js @@ -222,6 +222,13 @@ function loadSolcoverJS(config={}){ coverageConfig.cwd = config.workingDir; coverageConfig.originalContractsDir = config.contractsDir; + if (config.matrix){ + coverageConfig.measureBranchCoverage = false; + coverageConfig.measureFunctionCoverage = false; + coverageConfig.measureModifierCoverage = false; + coverageConfig.measureStatementCoverage = false; + } + // Solidity-Coverage writes to Truffle config config.mocha = config.mocha || {}; diff --git a/plugins/resources/truffle.utils.js b/plugins/resources/truffle.utils.js index 8e2cc70..5aaa21c 100644 --- a/plugins/resources/truffle.utils.js +++ b/plugins/resources/truffle.utils.js @@ -196,6 +196,20 @@ function normalizeConfig(config){ return config; } +/** + * Configures mocha to generate a json object which maps which tests + * hit which lines of code. + */ +function collectTestMatrixData(config, api){ + if (config.matrix){ + config.mocha = config.mocha || {}; + config.mocha.reporter = api.matrixReporterPath; + config.mocha.reporterOptions = { + collectTestMatrixData: api.collectTestMatrixData.bind(api) + } + } +} + /** * Replacement logger which filters out compilation warnings triggered by injected trace * function definitions. @@ -220,5 +234,6 @@ module.exports = { setNetworkFrom, loadLibrary, normalizeConfig, - filteredLogger + filteredLogger, + collectTestMatrixData } diff --git a/plugins/truffle.plugin.js b/plugins/truffle.plugin.js index c1850df..949077d 100644 --- a/plugins/truffle.plugin.js +++ b/plugins/truffle.plugin.js @@ -110,6 +110,7 @@ async function plugin(config){ await api.onCompileComplete(config); config.test_files = await truffleUtils.getTestFilePaths(config); + truffleUtils.collectTestMatrixData(config, api); // Run tests try { failures = await truffle.test.run(config) @@ -118,8 +119,13 @@ async function plugin(config){ } await api.onTestsComplete(config); - // Run Istanbul - await api.report(); + // ================================= + // Output (Istanbul or Test Matrix) + // ================================= + (config.matrix) + ? await api.saveTestMatrix() + : await api.report(); + await api.onIstanbulComplete(config); } catch(e){ diff --git a/scripts/run-metacoin.sh b/scripts/run-metacoin.sh index 0ed037c..a489a62 100755 --- a/scripts/run-metacoin.sh +++ b/scripts/run-metacoin.sh @@ -53,3 +53,13 @@ if [ ! -d "coverage" ]; then exit 1 fi +npx truffle run coverage --matrix + +# Test that coverage/ was generated +if [ ! -f "testMatrix.json" ]; then + echo "ERROR: no matrix file was created." + exit 1 +fi + +cat testMatrix.json + diff --git a/scripts/run-nomiclabs.sh b/scripts/run-nomiclabs.sh index aed4c3e..3d62b77 100755 --- a/scripts/run-nomiclabs.sh +++ b/scripts/run-nomiclabs.sh @@ -13,6 +13,13 @@ function verifyCoverageExists { fi } +function verifyMatrixExists { + if [ ! -f "testMatrix.json" ]; then + echo "ERROR: no matrix file was created." + exit 1 + fi +} + # Get rid of any caches sudo rm -rf node_modules echo "NVM CURRENT >>>>>" && nvm current @@ -47,6 +54,12 @@ npx hardhat coverage verifyCoverageExists +npx hardhat coverage --matrix + +verifyMatrixExists + +cat testMatrix.json + echo "" echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" echo "wighawag/hardhat-deploy " diff --git a/test/integration/projects/matrix/.solcover.js b/test/integration/projects/matrix/.solcover.js new file mode 100644 index 0000000..992f82b --- /dev/null +++ b/test/integration/projects/matrix/.solcover.js @@ -0,0 +1,16 @@ +// Testing hooks +const fn = (msg, config) => config.logger.log(msg); +const reporterPath = (process.env.TRUFFLE_TEST) + ? "./plugins/resources/matrix.js" + : "../plugins/resources/matrix.js"; + +module.exports = { + // This is loaded directly from `./plugins` during unit tests. The default val is + // "solidity-coverage/plugins/resources/matrix.js" + matrixReporterPath: reporterPath, + matrixOutputPath: "alternateTestMatrix.json", + + skipFiles: ['Migrations.sol'], + silent: process.env.SILENT ? true : false, + istanbulReporter: ['json-summary', 'text'], +} diff --git a/test/integration/projects/matrix/contracts/MatrixA.sol b/test/integration/projects/matrix/contracts/MatrixA.sol new file mode 100644 index 0000000..aeac9bc --- /dev/null +++ b/test/integration/projects/matrix/contracts/MatrixA.sol @@ -0,0 +1,17 @@ +pragma solidity ^0.7.0; + + +contract MatrixA { + uint x; + constructor() public { + } + + function sendFn() public { + x = 5; + } + + function callFn() public pure returns (uint){ + uint y = 5; + return y; + } +} diff --git a/test/integration/projects/matrix/contracts/MatrixB.sol b/test/integration/projects/matrix/contracts/MatrixB.sol new file mode 100644 index 0000000..b1981c2 --- /dev/null +++ b/test/integration/projects/matrix/contracts/MatrixB.sol @@ -0,0 +1,17 @@ +pragma solidity ^0.7.0; + + +contract MatrixB { + uint x; + constructor() public { + } + + function sendFn() public { + x = 5; + } + + function callFn() public pure returns (uint){ + uint y = 5; + return y; + } +} diff --git a/test/integration/projects/matrix/expectedTestMatrixHardhat.json b/test/integration/projects/matrix/expectedTestMatrixHardhat.json new file mode 100644 index 0000000..5afc260 --- /dev/null +++ b/test/integration/projects/matrix/expectedTestMatrixHardhat.json @@ -0,0 +1,46 @@ +{ + "contracts/MatrixA.sol": { + "10": [ + { + "title": "sends to A", + "file": "test/matrix_a_b.js" + }, + { + "title": "sends", + "file": "test/matrix_a.js" + } + ], + "14": [ + { + "title": "calls", + "file": "test/matrix_a.js" + } + ], + "15": [ + { + "title": "calls", + "file": "test/matrix_a.js" + } + ] + }, + "contracts/MatrixB.sol": { + "10": [ + { + "title": "sends to B", + "file": "test/matrix_a_b.js" + } + ], + "14": [ + { + "title": "calls B", + "file": "test/matrix_a_b.js" + } + ], + "15": [ + { + "title": "calls B", + "file": "test/matrix_a_b.js" + } + ] + } +} \ No newline at end of file diff --git a/test/integration/projects/matrix/expectedTestMatrixTruffle.json b/test/integration/projects/matrix/expectedTestMatrixTruffle.json new file mode 100644 index 0000000..f57487a --- /dev/null +++ b/test/integration/projects/matrix/expectedTestMatrixTruffle.json @@ -0,0 +1,46 @@ +{ + "contracts/MatrixA.sol": { + "10": [ + { + "title": "sends", + "file": "test/matrix_a.js" + }, + { + "title": "sends to A", + "file": "test/matrix_a_b.js" + } + ], + "14": [ + { + "title": "calls", + "file": "test/matrix_a.js" + } + ], + "15": [ + { + "title": "calls", + "file": "test/matrix_a.js" + } + ] + }, + "contracts/MatrixB.sol": { + "10": [ + { + "title": "sends to B", + "file": "test/matrix_a_b.js" + } + ], + "14": [ + { + "title": "calls B", + "file": "test/matrix_a_b.js" + } + ], + "15": [ + { + "title": "calls B", + "file": "test/matrix_a_b.js" + } + ] + } +} \ No newline at end of file diff --git a/test/integration/projects/matrix/hardhat.config.js b/test/integration/projects/matrix/hardhat.config.js new file mode 100644 index 0000000..b402a97 --- /dev/null +++ b/test/integration/projects/matrix/hardhat.config.js @@ -0,0 +1,9 @@ +require("@nomiclabs/hardhat-truffle5"); +require(__dirname + "/../plugins/nomiclabs.plugin"); + +module.exports={ + solidity: { + version: "0.7.3" + }, + logger: process.env.SILENT ? { log: () => {} } : console, +}; diff --git a/test/integration/projects/matrix/test/matrix_a.js b/test/integration/projects/matrix/test/matrix_a.js new file mode 100644 index 0000000..d1cde11 --- /dev/null +++ b/test/integration/projects/matrix/test/matrix_a.js @@ -0,0 +1,15 @@ +const MatrixA = artifacts.require("MatrixA"); + +contract("MatrixA", function(accounts) { + let instance; + + before(async () => instance = await MatrixA.new()) + + it('sends', async function(){ + await instance.sendFn(); + }); + + it('calls', async function(){ + await instance.callFn(); + }) +}); diff --git a/test/integration/projects/matrix/test/matrix_a_b.js b/test/integration/projects/matrix/test/matrix_a_b.js new file mode 100644 index 0000000..6e37de9 --- /dev/null +++ b/test/integration/projects/matrix/test/matrix_a_b.js @@ -0,0 +1,30 @@ +const MatrixA = artifacts.require("MatrixA"); +const MatrixB = artifacts.require("MatrixB"); + +contract("Matrix A and B", function(accounts) { + let instanceA; + let instanceB; + + before(async () => { + instanceA = await MatrixA.new(); + instanceB = await MatrixB.new(); + }) + + it('sends to A', async function(){ + await instanceA.sendFn(); + }); + + // Duplicate test title and file should *not* be duplicated in the output + it('sends to A', async function(){ + await instanceA.sendFn(); + }) + + it('calls B', async function(){ + await instanceB.callFn(); + }) + + it('sends to B', async function(){ + await instanceB.sendFn(); + }); + +}); diff --git a/test/integration/projects/matrix/truffle-config.js b/test/integration/projects/matrix/truffle-config.js new file mode 100644 index 0000000..bac9791 --- /dev/null +++ b/test/integration/projects/matrix/truffle-config.js @@ -0,0 +1,10 @@ +module.exports = { + networks: {}, + mocha: {}, + compilers: { + solc: { + version: "0.7.3" + } + }, + logger: process.env.SILENT ? { log: () => {} } : console, +} diff --git a/test/units/hardhat/flags.js b/test/units/hardhat/flags.js index 5af5949..6127e1a 100644 --- a/test/units/hardhat/flags.js +++ b/test/units/hardhat/flags.js @@ -172,5 +172,24 @@ describe('Hardhat Plugin: command line options', function() { verify.lineCoverage(expected); shell.rm('.solcover.js'); }); + + it('--matrix', async function(){ + const taskArgs = { + matrix: true + } + + mock.installFullProject('matrix'); + mock.hardhatSetupEnv(this); + + await this.env.run("coverage", taskArgs); + + // Integration test checks output path configurabililty + const altPath = path.join(process.cwd(), './alternateTestMatrix.json'); + const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json'); + const producedMatrix = require(altPath) + const expectedMatrix = require(expPath); + + assert.deepEqual(producedMatrix, expectedMatrix); + }); }); diff --git a/test/units/truffle/flags.js b/test/units/truffle/flags.js index 2365041..e073fc3 100644 --- a/test/units/truffle/flags.js +++ b/test/units/truffle/flags.js @@ -257,5 +257,23 @@ describe('Truffle Plugin: command line options', function() { `Should have used default coverage port 8545: ${mock.loggerOutput.val}` ); }); + + it('--matrix', async function(){ + process.env.TRUFFLE_TEST = true; // Path to reporter differs btw HH and Truffle + truffleConfig.matrix = true; + + mock.installFullProject('matrix'); + await plugin(truffleConfig); + + // Integration test checks output path configurabililty + const altPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json')); + const expPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json')); + + const producedMatrix = require(altPath) + const expectedMatrix = require(expPath); + + assert.deepEqual(producedMatrix, expectedMatrix); + process.env.TRUFFLE_TEST = false; + }); });