From abea00b21ca17ef5d37e17cfecbaa2f7c6b0fa9e Mon Sep 17 00:00:00 2001 From: cgewecke Date: Fri, 26 Jan 2024 09:25:57 -0800 Subject: [PATCH] Throw error when mocha parallel is set to true (#833) --- README.md | 2 ++ docs/advanced.md | 16 ++++++++++++++++ plugins/resources/plugin.ui.js | 5 +++++ plugins/resources/plugin.utils.js | 6 ++++++ test/integration/projects/parallel/.solcover.js | 7 +++++++ .../projects/parallel/contracts/ContractA.sol | 10 ++++++++++ .../projects/parallel/hardhat.config.js | 8 ++++++++ test/units/hardhat/errors.js | 17 +++++++++++++++++ 8 files changed, 71 insertions(+) create mode 100644 test/integration/projects/parallel/.solcover.js create mode 100644 test/integration/projects/parallel/contracts/ContractA.sol create mode 100644 test/integration/projects/parallel/hardhat.config.js diff --git a/README.md b/README.md index 5edd227..8e562b6 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ Common problems & questions: + [Running out of time][6] + [Running out of stack][1002] (Stack too deep) + [Running out of memory][5] ++ [Running in parallel (in CI)][1003] ## Example reports + [metacoin][9] (Istanbul HTML) @@ -216,4 +217,5 @@ $ yarn [39]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#generating-a-test-matrix [1001]: https://docs.soliditylang.org/en/v0.8.0/using-the-compiler.html#input-description [1002]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/faq.md#running-out-of-stack +[1003]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#parallelization-in-ci diff --git a/docs/advanced.md b/docs/advanced.md index ba8176e..e06af40 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -125,8 +125,24 @@ In combination these data sets can be passed to Joran Honig's [tarantula][29] to a fault localization algorithm to generate 'suspiciousness' ratings for each line of Solidity code in your project. +## Parallelization in CI + +Coverage does not work with the Hardhat's mocha parallel mode. However, it *is* possible to parallelize coverage in CI environments that support complex workflows. The core idea is to + ++ partition the set of test files passed to the coverage task ++ split coverage into several concurrent jobs, passing test file targets as arguments using the `--testfiles` command line flag ++ cache the coverage results in shared storage as each job completes ++ combine results in a final step (using the [instanbul-combine-updated][30] tool) + +There's a nice example of this being done in CircleCI [at Synthetix, here][31]. + +:bulb: **Pro Tip**: Codecov CI will automatically combine coverage reports sent to them as a batch - if you're using that service you don't need to do this yourself. + + [22]: https://github.com/JoranHonig/vertigo#vertigo [23]: http://spideruci.org/papers/jones05.pdf [25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md [27]: https://mochajs.org/api/reporters_json.js.html [29]: https://github.com/JoranHonig/tarantula +[30]: https://www.npmjs.com/package/istanbul-combine-updated +[31]: https://github.com/Synthetixio/synthetix/blob/bd54f4e9cfd1529d8ea2e5a7b4d0be4c988e1a03/.circleci/src/jobs/job-unit-tests-coverage.yml diff --git a/plugins/resources/plugin.ui.js b/plugins/resources/plugin.ui.js index e139182..08d201f 100644 --- a/plugins/resources/plugin.ui.js +++ b/plugins/resources/plugin.ui.js @@ -56,6 +56,11 @@ class PluginUI extends UI { `${c.red('the path you specified for it is wrong.')}`, 'tests-fail': `${x} ${c.bold(args[0])} ${c.red('test(s) failed under coverage.')}`, + + 'mocha-parallel-fail': `${c.red('Coverage cannot be run in mocha parallel mode. ')}` + + `${c.red('Set \`mocha: { parallel: false }\` in .solcover.js ')}` + + `${c.red('to disable the option for the coverage task. ')}` + + `${c.red('See the solidity-coverage README FAQ for info on parallelizing coverage in CI.')}`, } diff --git a/plugins/resources/plugin.utils.js b/plugins/resources/plugin.utils.js index 72a17c8..6de507f 100644 --- a/plugins/resources/plugin.utils.js +++ b/plugins/resources/plugin.utils.js @@ -232,6 +232,12 @@ function loadSolcoverJS(config={}){ ); } + // Per fvictorio recommendation in #691 + if (config.mocha.parallel) { + const message = ui.generate('mocha-parallel-fail'); + throw new Error(message); + } + return coverageConfig; } diff --git a/test/integration/projects/parallel/.solcover.js b/test/integration/projects/parallel/.solcover.js new file mode 100644 index 0000000..95d2b55 --- /dev/null +++ b/test/integration/projects/parallel/.solcover.js @@ -0,0 +1,7 @@ +module.exports = { + "silent": false, + "istanbulReporter": [ "json-summary", "text"], + "mocha": { + parallel: true // manually tested that setting to false overrides HH config + }, +} diff --git a/test/integration/projects/parallel/contracts/ContractA.sol b/test/integration/projects/parallel/contracts/ContractA.sol new file mode 100644 index 0000000..61c9454 --- /dev/null +++ b/test/integration/projects/parallel/contracts/ContractA.sol @@ -0,0 +1,10 @@ +pragma solidity ^0.7.0; + + +contract ContractA { + uint x; + + function set() public { + x = 5; + } +} diff --git a/test/integration/projects/parallel/hardhat.config.js b/test/integration/projects/parallel/hardhat.config.js new file mode 100644 index 0000000..13a4be5 --- /dev/null +++ b/test/integration/projects/parallel/hardhat.config.js @@ -0,0 +1,8 @@ +require(__dirname + "/../plugins/nomiclabs.plugin"); + +module.exports = { + mocha: { + parallel: true + }, + logger: process.env.SILENT ? { log: () => {} } : console, +}; diff --git a/test/units/hardhat/errors.js b/test/units/hardhat/errors.js index 845dbdb..18907a1 100644 --- a/test/units/hardhat/errors.js +++ b/test/units/hardhat/errors.js @@ -202,4 +202,21 @@ describe('Hardhat Plugin: error cases', function() { verify.coverageNotGenerated(hardhatConfig); }); + + it('mocha parallel option is true', async function(){ + mock.installFullProject('parallel'); + mock.hardhatSetupEnv(this); + + try { + await this.env.run("coverage"); + assert.fail() + } catch(err){ + assert( + err.message.includes('Coverage cannot be run in mocha parallel mode'), + `Should notify when mocha parallel flag is set:: ${err.message}` + ); + } + + verify.coverageNotGenerated(hardhatConfig); + }) })