Add test matrix generator (#593)

4 years ago · afd7ef0376
parent e76a165b63
commit afd7ef0376
25 changed files with 897 additions and 12 deletions
--- a/README.md
+++ b/README.md
@ -58,6 +58,8 @@ A working example can be found at [openzeppelin-contracts, here.][35]
 | solcoverjs | `--solcoverjs ./../.solcover.js` | Relative path from working directory to config. Useful for monorepo packages that share settings. (Path must be "./" prefixed) |
 | network    | `--network development` | Use network settings defined in the Hardhat config |
 | temp[<sup>*</sup>][14]       | `--temp build`   | :warning: **Caution** :warning:  Path to a *disposable* folder to store compilation artifacts in. Useful when your test setup scripts include hard-coded paths to a build directory. [More...][14] |
 | matrix   | `--matrix` | Generate a JSON object that maps which mocha tests hit which lines of code. (Useful
 as an input for some fuzzing, mutation testing and fault-localization algorithms.) [More...][39]|
 [<sup>*</sup> Advanced use][14]
@ -82,6 +84,7 @@ module.exports = {
 | measureStatementCoverage | *boolean* | `true` | Computes statement (in addition to line) coverage. [More...][34] |
 | measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] |
 | measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] |
 | matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39] |
 | istanbulFolder | *String* | `./coverage` |  Folder location for Istanbul coverage reports. |
 | istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2]  |
 | mocha | *Object* | `{ }` | [Mocha options][3] to merge into existing mocha config. `grep` and `invert` are useful for skipping certain tests under coverage using tags in the test descriptions.|
@ -212,6 +215,7 @@ $ yarn
 [36]: https://hardhat.org/
 [37]: https://github.com/sc-forks/solidity-coverage/blob/master/HARDHAT_README.md
 [38]: https://github.com/sindresorhus/globby#globbing-patterns
 [39]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#generating-a-test-matrix
 [1001]: https://docs.soliditylang.org/en/v0.8.0/using-the-compiler.html#input-description
 [1002]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/faq.md#running-out-of-stack
--- a/docs/advanced.md
+++ b/docs/advanced.md
@ -106,3 +106,18 @@ Setting the `measureStatementCoverage` and/or `measureFunctionCoverage` options
 improve performance, lower the cost of execution and minimize complications that arise from `solc`'s
 limits on how large the compilation payload can be.
 ## Generating a test matrix
 Some advanced testing strategies benefit from knowing which tests in a suite hit a
 specific line of code. Examples include:
 + [mutation testing][22], where this data lets you select the correct subset of tests to check
 a mutation with.
 + [fault localization techniques][23], where the complete data set is a key input to algorithms that try
 to guess where bugs might exist in a given codebase.
 Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable
 test names to each line of code to a file named `testMatrix.json` in your project's root.
 [22]: https://github.com/JoranHonig/vertigo#vertigo
 [23]: http://spideruci.org/papers/jones05.pdf
 [25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md
--- a/docs/matrix.md
+++ b/docs/matrix.md
@ -0,0 +1,421 @@
 ### Test Matrix Example
 An example of output written to the file `./testMatrix.json` when coverage
 is run with the `--matrix` cli flag. (Source project: [sc-forks/hardhat-e2e][1])
 [1]: https://github.com/sc-forks/hardhat-e2e
 ```js
 // Paths are relative to the project root directory
 {
 // Solidity file name
 "contracts/EtherRouter/EtherRouter.sol": {
  // Line number
  "23": [
   {
    // Grep-able mocha test title
    "title": "Resolves methods routed through an EtherRouter proxy",
    // Selectable mocha test file
    "file": "test/etherrouter.js"
   }
  ],
  "42": [
   {
    "title": "Resolves methods routed through an EtherRouter proxy",
    "file": "test/etherrouter.js"
   }
  ],
  "45": [
   {
    "title": "Resolves methods routed through an EtherRouter proxy",
    "file": "test/etherrouter.js"
   }
  ],
  "61": [
   {
    "title": "Resolves methods routed through an EtherRouter proxy",
    "file": "test/etherrouter.js"
   }
  ]
 },
 "contracts/EtherRouter/Factory.sol": {
  "19": [
   {
    "title": "Resolves methods routed through an EtherRouter proxy",
    "file": "test/etherrouter.js"
   }
  ]
 },
 "contracts/EtherRouter/Resolver.sol": {
  "22": [
   {
    "title": "Resolves methods routed through an EtherRouter proxy",
    "file": "test/etherrouter.js"
   }
  ],
  "26": [
   {
    "title": "Resolves methods routed through an EtherRouter proxy",
    "file": "test/etherrouter.js"
   }
  ],
  "30": [
   {
    "title": "Resolves methods routed through an EtherRouter proxy",
    "file": "test/etherrouter.js"
   }
  ]
 },
 "contracts/MetaCoin.sol": {
  "16": [
   {
    "title": "should put 10000 MetaCoin in the first account",
    "file": "test/metacoin.js"
   },
   {
    "title": "should call a function that depends on a linked library",
    "file": "test/metacoin.js"
   },
   {
    "title": "should send coin correctly",
    "file": "test/metacoin.js"
   },
   {
    "title": "a and b",
    "file": "test/multicontract.js"
   }
  ],
  "20": [
   {
    "title": "should send coin correctly",
    "file": "test/metacoin.js"
   }
  ],
  "21": [
   {
    "title": "should send coin correctly",
    "file": "test/metacoin.js"
   }
  ],
  "22": [
   {
    "title": "should send coin correctly",
    "file": "test/metacoin.js"
   }
  ],
  "23": [
   {
    "title": "should send coin correctly",
    "file": "test/metacoin.js"
   }
  ],
  "24": [
   {
    "title": "should send coin correctly",
    "file": "test/metacoin.js"
   }
  ],
  "28": [
   {
    "title": "should call a function that depends on a linked library",
    "file": "test/metacoin.js"
   }
  ],
  "32": [
   {
    "title": "should put 10000 MetaCoin in the first account",
    "file": "test/metacoin.js"
   },
   {
    "title": "should call a function that depends on a linked library",
    "file": "test/metacoin.js"
   },
   {
    "title": "should send coin correctly",
    "file": "test/metacoin.js"
   }
  ]
 },
 "contracts/ConvertLib.sol": {
  "6": [
   {
    "title": "should call a function that depends on a linked library",
    "file": "test/metacoin.js"
   }
  ]
 },
 "contracts/MultiContractFile.sol": {
  "7": [
   {
    "title": "a and b",
    "file": "test/multicontract.js"
   },
   {
    "title": "methods that call methods in other contracts",
    "file": "test/variablecosts.js"
   }
  ],
  "15": [
   {
    "title": "a and b",
    "file": "test/multicontract.js"
   }
  ]
 },
 "contracts/VariableConstructor.sol": {
  "8": [
   {
    "title": "should should initialize with a short string",
    "file": "test/variableconstructor.js"
   },
   {
    "title": "should should initialize with a medium length string",
    "file": "test/variableconstructor.js"
   },
   {
    "title": "should should initialize with a long string",
    "file": "test/variableconstructor.js"
   },
   {
    "title": "should should initialize with a random length string",
    "file": "test/variableconstructor.js"
   }
  ]
 },
 "contracts/VariableCosts.sol": {
  "13": [
   {
    "title": "should should initialize with a short string",
    "file": "test/variableconstructor.js"
   },
   {
    "title": "should should initialize with a medium length string",
    "file": "test/variableconstructor.js"
   },
   {
    "title": "should should initialize with a long string",
    "file": "test/variableconstructor.js"
   },
   {
    "title": "should should initialize with a random length string",
    "file": "test/variableconstructor.js"
   },
   {
    "title": "should add one",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add three",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add even 5!",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should delete one",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should delete three",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should delete five",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add five and delete one",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should set a random length string",
    "file": "test/variablecosts.js"
   },
   {
    "title": "methods that do not throw",
    "file": "test/variablecosts.js"
   },
   {
    "title": "methods that throw",
    "file": "test/variablecosts.js"
   },
   {
    "title": "methods that call methods in other contracts",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should allow contracts to have identically named methods",
    "file": "test/variablecosts.js"
   }
  ],
  "29": [
   {
    "title": "should add one",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add three",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add even 5!",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add five and delete one",
    "file": "test/variablecosts.js"
   }
  ],
  "30": [
   {
    "title": "should add one",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add three",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add even 5!",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add five and delete one",
    "file": "test/variablecosts.js"
   }
  ],
  "34": [
   {
    "title": "should delete one",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should delete three",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should delete five",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add five and delete one",
    "file": "test/variablecosts.js"
   }
  ],
  "35": [
   {
    "title": "should delete one",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should delete three",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should delete five",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should add five and delete one",
    "file": "test/variablecosts.js"
   }
  ],
  "43": [
   {
    "title": "should set a random length string",
    "file": "test/variablecosts.js"
   }
  ],
  "47": [
   {
    "title": "methods that do not throw",
    "file": "test/variablecosts.js"
   },
   {
    "title": "methods that throw",
    "file": "test/variablecosts.js"
   }
  ],
  "48": [
   {
    "title": "methods that do not throw",
    "file": "test/variablecosts.js"
   }
  ],
  "52": [
   {
    "title": "methods that call methods in other contracts",
    "file": "test/variablecosts.js"
   }
  ],
  "53": [
   {
    "title": "methods that call methods in other contracts",
    "file": "test/variablecosts.js"
   }
  ],
  "54": [
   {
    "title": "methods that call methods in other contracts",
    "file": "test/variablecosts.js"
   }
  ]
 },
 "contracts/Wallets/Wallet.sol": {
  "8": [
   {
    "title": "should allow contracts to have identically named methods",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should should allow transfers and sends",
    "file": "test/wallet.js"
   }
  ],
  "12": [
   {
    "title": "should allow contracts to have identically named methods",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should should allow transfers and sends",
    "file": "test/wallet.js"
   }
  ],
  "17": [
   {
    "title": "should allow contracts to have identically named methods",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should should allow transfers and sends",
    "file": "test/wallet.js"
   }
  ],
  "22": [
   {
    "title": "should allow contracts to have identically named methods",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should should allow transfers and sends",
    "file": "test/wallet.js"
   }
  ],
  "23": [
   {
    "title": "should allow contracts to have identically named methods",
    "file": "test/variablecosts.js"
   },
   {
    "title": "should should allow transfers and sends",
    "file": "test/wallet.js"
   }
  ]
 }
 }
 ```
--- a/lib/api.js
+++ b/lib/api.js
@ -20,6 +20,7 @@ class API {
  constructor(config={}) {
    this.validator = new ConfigValidator()
    this.config = config || {};
    this.testMatrix = {};
    // Validate
    this.validator.validate(this.config);
@ -30,6 +31,8 @@ class API {
    this.testsErrored = false;
    this.cwd = config.cwd || process.cwd();
    this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json";
    this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js"
    this.defaultHook = () => {};
    this.onServerReady = config.onServerReady           || this.defaultHook;
@ -296,6 +299,52 @@ class API {
    }
  }*/
  // ==========================
  // Test Matrix Data Collector
  // ==========================
  /**
   * @param  {Object} testInfo Mocha object passed to reporter 'test end' event
   */
  collectTestMatrixData(testInfo){
    const hashes = Object.keys(this.instrumenter.instrumentationData);
    const title = testInfo.title;
    const file = path.relative(this.cwd, testInfo.file);
    for (const hash of hashes){
      const {
        contractPath,
        hits,
        type,
        id
      } = this.instrumenter.instrumentationData[hash];
      if (type === 'line' && hits > 0){
        if (!this.testMatrix[contractPath]){
          this.testMatrix[contractPath] = {};
        }
        if (!this.testMatrix[contractPath][id]){
          this.testMatrix[contractPath][id] = [];
        }
        // Search for and exclude duplicate entries
        let duplicate = false;
        for (const item of this.testMatrix[contractPath][id]){
          if (item.title === title && item.file === file){
            duplicate = true;
            break;
          }
        }
        if (!duplicate) {
          this.testMatrix[contractPath][id].push({title, file});
        }
        // Reset line data
        this.instrumenter.instrumentationData[hash].hits = 0;
      }
    }
  }
  // ========
  // File I/O
  // ========
@ -305,6 +354,12 @@ class API {
    fs.writeFileSync(covPath, JSON.stringify(data));
  }
  saveTestMatrix(){
    const matrixPath = path.join(this.cwd, this.matrixOutputPath);
    const mapping = this.makeKeysRelative(this.testMatrix, this.cwd);
    fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' '));
  }
  // =====
  // Paths
  // =====
--- a/package.json
+++ b/package.json
@ -12,7 +12,7 @@
  "scripts": {
    "nyc": "SILENT=true  nyc --exclude '**/sc_temp/**' --exclude '**/test/**'",
    "test": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
-    "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
+    "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' --exclude 'plugins/resources/matrix.js' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
    "test:debug": "node --max-old-space-size=4096 ./node_modules/.bin/mocha test/units/* --timeout 100000 --no-warnings --exit",
    "netlify": "./scripts/run-netlify.sh"
  },
@ -35,6 +35,7 @@
    "globby": "^10.0.1",
    "jsonschema": "^1.2.4",
    "lodash": "^4.17.15",
    "mocha": "7.1.2",
    "node-emoji": "^1.10.0",
    "pify": "^4.0.1",
    "recursive-readdir": "^2.2.2",
@ -60,7 +61,6 @@
    "ganache-cli": "6.12.2",
    "hardhat": "^2.9.3",
    "hardhat-gas-reporter": "^1.0.1",
    "mocha": "5.2.0",
    "nyc": "^14.1.1",
    "solc": "^0.7.5",
    "truffle": "5.1.43",
--- a/plugins/hardhat.plugin.js
+++ b/plugins/hardhat.plugin.js
@ -97,6 +97,7 @@ task("coverage", "Generates a code coverage report for tests")
  .addOptionalParam("testfiles",  ui.flags.file,       "", types.string)
  .addOptionalParam("solcoverjs", ui.flags.solcoverjs, "", types.string)
  .addOptionalParam('temp',       ui.flags.temp,       "", types.string)
  .addFlag('matrix', ui.flags.testMatrix)
  .setAction(async function(args, env){
  const API = require('./../lib/api');
@ -232,6 +233,9 @@ task("coverage", "Generates a code coverage report for tests")
      ? nomiclabsUtils.getTestFilePaths(args.testfiles)
      : [];
    // Optionally collect tests-per-line-of-code data
    nomiclabsUtils.collectTestMatrixData(args, env, api);
    try {
      failedTests = await env.run(TASK_TEST, {testFiles: testfiles})
    } catch (e) {
@ -239,10 +243,13 @@ task("coverage", "Generates a code coverage report for tests")
    }
    await api.onTestsComplete(config);
-    // ========
+    // =================================
-    // Istanbul
+    // Output (Istanbul or Test Matrix)
-    // ========
+    // =================================
-    await api.report();
+    (args.matrix)
      ? await api.saveTestMatrix()
      : await api.report();
    await api.onIstanbulComplete(config);
  } catch(e) {
--- a/plugins/resources/matrix.js
+++ b/plugins/resources/matrix.js
@ -0,0 +1,71 @@
 const mocha = require("mocha");
 const inherits = require("util").inherits;
 const Spec = mocha.reporters.Spec;
 /**
 * This file adapted from mocha's stats-collector
 * https://github.com/mochajs/mocha/blob/54475eb4ca35a2c9044a1b8c59a60f09c73e6c01/lib/stats-collector.js#L1-L83
 */
 const Date = global.Date;
 /**
 * Provides stats such as test duration, number of tests passed / failed etc., by
 * listening for events emitted by `runner`.
 */
 function mochaStats(runner) {
  const stats = {
    suites: 0,
    tests: 0,
    passes: 0,
    pending: 0,
    failures: 0
  };
  if (!runner) throw new Error("Missing runner argument");
  runner.stats = stats;
  runner.on("pass", () => stats.passes++);
  runner.on("fail", () => stats.failures++);
  runner.on("pending", () => stats.pending++);
  runner.on("test end", () => stats.tests++);
  runner.once("start", () => (stats.start = new Date()));
  runner.once("end", function() {
    stats.end = new Date();
    stats.duration = stats.end - stats.start;
  });
 }
 /**
 * Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run
 * and collects data about which tests hit which lines of code.
 * This "test matrix" can be used as an input to
 *
 *
 * @param {Object} runner  mocha's runner
 * @param {Object} options reporter.options (see README example usage)
 */
 function Matrix(runner, options) {
  // Spec reporter
  Spec.call(this, runner, options);
  // Initialize stats for Mocha 6+ epilogue
  if (!runner.stats) {
    mochaStats(runner);
    this.stats = runner.stats;
  }
  runner.on("test end", (info) => {
    options.reporterOptions.collectTestMatrixData(info);
  });
 }
 /**
 * Inherit from `Base.prototype`.
 */
 inherits(Matrix, Spec);
 module.exports = Matrix;
--- a/plugins/resources/nomiclabs.ui.js
+++ b/plugins/resources/nomiclabs.ui.js
@ -8,7 +8,9 @@ class PluginUI extends UI {
    super(log);
    this.flags = {
-      file:       `Path (or glob) defining a subset of tests to run`,
+      testfiles:  `Path (or glob) defining a subset of tests to run`,
      testMatrix: `Generate a json object which maps which unit tests hit which lines of code.`,
      solcoverjs: `Relative path from working directory to config. ` +
                  `Useful for monorepo packages that share settings.`,
@ -16,7 +18,6 @@ class PluginUI extends UI {
      temp:       `Path to a disposable folder to store compilation artifacts in. ` +
                  `Useful when your test setup scripts include hard-coded paths to ` +
                  `a build directory.`,
    }
  }
--- a/plugins/resources/nomiclabs.utils.js
+++ b/plugins/resources/nomiclabs.utils.js
@ -37,6 +37,7 @@ function normalizeConfig(config, args={}){
  config.logger = config.logger ? config.logger : {log: null};
  config.solcoverjs = args.solcoverjs
  config.gasReporter = { enabled: false }
  config.matrix = args.matrix;
  try {
    const hardhatPackage = require('hardhat/package.json');
@ -165,6 +166,21 @@ function configureHttpProvider(networkConfig, api, ui){
  networkConfig.url = `http://${api.host}:${api.port}`;
 }
 /**
 * Configures mocha to generate a json object which maps which tests
 * hit which lines of code.
 */
 function collectTestMatrixData(args, env, api){
  if (args.matrix){
    mochaConfig = env.config.mocha || {};
    mochaConfig.reporter = api.matrixReporterPath;
    mochaConfig.reporterOptions = {
      collectTestMatrixData: api.collectTestMatrixData.bind(api)
    }
    env.config.mocha = mochaConfig;
  }
 }
 /**
 * Sets the default `from` account field in the network that will be used.
 * This needs to be done after accounts are fetched from the launched client.
@ -216,6 +232,7 @@ module.exports = {
  setupBuidlerNetwork,
  setupHardhatNetwork,
  getTestFilePaths,
-  setNetworkFrom
+  setNetworkFrom,
  collectTestMatrixData
 }
--- a/plugins/resources/plugin.utils.js
+++ b/plugins/resources/plugin.utils.js
@ -222,6 +222,13 @@ function loadSolcoverJS(config={}){
  coverageConfig.cwd = config.workingDir;
  coverageConfig.originalContractsDir = config.contractsDir;
  if (config.matrix){
    coverageConfig.measureBranchCoverage = false;
    coverageConfig.measureFunctionCoverage = false;
    coverageConfig.measureModifierCoverage = false;
    coverageConfig.measureStatementCoverage = false;
  }
  // Solidity-Coverage writes to Truffle config
  config.mocha = config.mocha || {};
--- a/plugins/resources/truffle.utils.js
+++ b/plugins/resources/truffle.utils.js
@ -196,6 +196,20 @@ function normalizeConfig(config){
  return config;
 }
 /**
 * Configures mocha to generate a json object which maps which tests
 * hit which lines of code.
 */
 function collectTestMatrixData(config, api){
  if (config.matrix){
    config.mocha = config.mocha || {};
    config.mocha.reporter = api.matrixReporterPath;
    config.mocha.reporterOptions = {
      collectTestMatrixData: api.collectTestMatrixData.bind(api)
    }
  }
 }
 /**
 * Replacement logger which filters out compilation warnings triggered by injected trace
 * function definitions.
@ -220,5 +234,6 @@ module.exports = {
  setNetworkFrom,
  loadLibrary,
  normalizeConfig,
-  filteredLogger
+  filteredLogger,
  collectTestMatrixData
 }
--- a/plugins/truffle.plugin.js
+++ b/plugins/truffle.plugin.js
@ -110,6 +110,7 @@ async function plugin(config){
    await api.onCompileComplete(config);
    config.test_files = await truffleUtils.getTestFilePaths(config);
    truffleUtils.collectTestMatrixData(config, api);
    // Run tests
    try {
      failures = await truffle.test.run(config)
@ -118,8 +119,13 @@ async function plugin(config){
    }
    await api.onTestsComplete(config);
-    // Run Istanbul
+    // =================================
-    await api.report();
+    // Output (Istanbul or Test Matrix)
    // =================================
    (config.matrix)
      ? await api.saveTestMatrix()
      : await api.report();
    await api.onIstanbulComplete(config);
  } catch(e){
--- a/scripts/run-metacoin.sh
+++ b/scripts/run-metacoin.sh
@ -53,3 +53,13 @@ if [ ! -d "coverage" ]; then
  exit 1
 fi
 npx truffle run coverage --matrix
 # Test that coverage/ was generated
 if [ ! -f "testMatrix.json" ]; then
  echo "ERROR: no matrix file was created."
  exit 1
 fi
 cat testMatrix.json
--- a/scripts/run-nomiclabs.sh
+++ b/scripts/run-nomiclabs.sh
@ -13,6 +13,13 @@ function verifyCoverageExists {
  fi
 }
 function verifyMatrixExists {
  if [ ! -f "testMatrix.json" ]; then
    echo "ERROR: no matrix file was created."
    exit 1
  fi
 }
 # Get rid of any caches
 sudo rm -rf node_modules
 echo "NVM CURRENT >>>>>" && nvm current
@ -47,6 +54,12 @@ npx hardhat coverage
 verifyCoverageExists
 npx hardhat coverage --matrix
 verifyMatrixExists
 cat testMatrix.json
 echo ""
 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
 echo "wighawag/hardhat-deploy                "
--- a/test/integration/projects/matrix/.solcover.js
+++ b/test/integration/projects/matrix/.solcover.js
@ -0,0 +1,16 @@
 // Testing hooks
 const fn = (msg, config) => config.logger.log(msg);
 const reporterPath = (process.env.TRUFFLE_TEST)
  ? "./plugins/resources/matrix.js"
  : "../plugins/resources/matrix.js";
 module.exports = {
  // This is loaded directly from `./plugins` during unit tests. The default val is
  // "solidity-coverage/plugins/resources/matrix.js"
  matrixReporterPath: reporterPath,
  matrixOutputPath: "alternateTestMatrix.json",
  skipFiles: ['Migrations.sol'],
  silent: process.env.SILENT ? true : false,
  istanbulReporter: ['json-summary', 'text'],
 }
--- a/test/integration/projects/matrix/contracts/MatrixA.sol
+++ b/test/integration/projects/matrix/contracts/MatrixA.sol
@ -0,0 +1,17 @@
 pragma solidity ^0.7.0;
 contract MatrixA {
  uint x;
  constructor() public {
  }
  function sendFn() public {
    x = 5;
  }
  function callFn() public pure returns (uint){
    uint y = 5;
    return y;
  }
 }
--- a/test/integration/projects/matrix/contracts/MatrixB.sol
+++ b/test/integration/projects/matrix/contracts/MatrixB.sol
@ -0,0 +1,17 @@
 pragma solidity ^0.7.0;
 contract MatrixB {
  uint x;
  constructor() public {
  }
  function sendFn() public {
    x = 5;
  }
  function callFn() public pure returns (uint){
    uint y = 5;
    return y;
  }
 }
--- a/test/integration/projects/matrix/expectedTestMatrixHardhat.json
+++ b/test/integration/projects/matrix/expectedTestMatrixHardhat.json
@ -0,0 +1,46 @@
 {
 "contracts/MatrixA.sol": {
  "10": [
   {
    "title": "sends to A",
    "file": "test/matrix_a_b.js"
   },
   {
    "title": "sends",
    "file": "test/matrix_a.js"
   }
  ],
  "14": [
   {
    "title": "calls",
    "file": "test/matrix_a.js"
   }
  ],
  "15": [
   {
    "title": "calls",
    "file": "test/matrix_a.js"
   }
  ]
 },
 "contracts/MatrixB.sol": {
  "10": [
   {
    "title": "sends to B",
    "file": "test/matrix_a_b.js"
   }
  ],
  "14": [
   {
    "title": "calls B",
    "file": "test/matrix_a_b.js"
   }
  ],
  "15": [
   {
    "title": "calls B",
    "file": "test/matrix_a_b.js"
   }
  ]
 }
 }
--- a/test/integration/projects/matrix/expectedTestMatrixTruffle.json
+++ b/test/integration/projects/matrix/expectedTestMatrixTruffle.json
@ -0,0 +1,46 @@
 {
 "contracts/MatrixA.sol": {
  "10": [
   {
    "title": "sends",
    "file": "test/matrix_a.js"
   },
   {
    "title": "sends to A",
    "file": "test/matrix_a_b.js"
   }
  ],
  "14": [
   {
    "title": "calls",
    "file": "test/matrix_a.js"
   }
  ],
  "15": [
   {
    "title": "calls",
    "file": "test/matrix_a.js"
   }
  ]
 },
 "contracts/MatrixB.sol": {
  "10": [
   {
    "title": "sends to B",
    "file": "test/matrix_a_b.js"
   }
  ],
  "14": [
   {
    "title": "calls B",
    "file": "test/matrix_a_b.js"
   }
  ],
  "15": [
   {
    "title": "calls B",
    "file": "test/matrix_a_b.js"
   }
  ]
 }
 }
--- a/test/integration/projects/matrix/hardhat.config.js
+++ b/test/integration/projects/matrix/hardhat.config.js
@ -0,0 +1,9 @@
 require("@nomiclabs/hardhat-truffle5");
 require(__dirname + "/../plugins/nomiclabs.plugin");
 module.exports={
  solidity: {
    version: "0.7.3"
  },
  logger: process.env.SILENT ? { log: () => {} } : console,
 };
--- a/test/integration/projects/matrix/test/matrix_a.js
+++ b/test/integration/projects/matrix/test/matrix_a.js
@ -0,0 +1,15 @@
 const MatrixA = artifacts.require("MatrixA");
 contract("MatrixA", function(accounts) {
  let instance;
  before(async () => instance = await MatrixA.new())
  it('sends', async function(){
    await instance.sendFn();
  });
  it('calls', async function(){
    await instance.callFn();
  })
 });
--- a/test/integration/projects/matrix/test/matrix_a_b.js
+++ b/test/integration/projects/matrix/test/matrix_a_b.js
@ -0,0 +1,30 @@
 const MatrixA = artifacts.require("MatrixA");
 const MatrixB = artifacts.require("MatrixB");
 contract("Matrix A and B", function(accounts) {
  let instanceA;
  let instanceB;
  before(async () => {
    instanceA = await MatrixA.new();
    instanceB = await MatrixB.new();
  })
  it('sends to A', async function(){
    await instanceA.sendFn();
  });
  // Duplicate test title and file should *not* be duplicated in the output
  it('sends to A', async function(){
    await instanceA.sendFn();
  })
  it('calls B', async function(){
    await instanceB.callFn();
  })
  it('sends to B', async function(){
    await instanceB.sendFn();
  });
 });
--- a/test/integration/projects/matrix/truffle-config.js
+++ b/test/integration/projects/matrix/truffle-config.js
@ -0,0 +1,10 @@
 module.exports = {
  networks: {},
  mocha: {},
  compilers: {
    solc: {
      version: "0.7.3"
    }
  },
  logger: process.env.SILENT ? { log: () => {} } : console,
 }
--- a/test/units/hardhat/flags.js
+++ b/test/units/hardhat/flags.js
@ -172,5 +172,24 @@ describe('Hardhat Plugin: command line options', function() {
    verify.lineCoverage(expected);
    shell.rm('.solcover.js');
  });
  it('--matrix', async function(){
    const taskArgs = {
      matrix: true
    }
    mock.installFullProject('matrix');
    mock.hardhatSetupEnv(this);
    await this.env.run("coverage", taskArgs);
    // Integration test checks output path configurabililty
    const altPath = path.join(process.cwd(), './alternateTestMatrix.json');
    const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
    const producedMatrix = require(altPath)
    const expectedMatrix = require(expPath);
    assert.deepEqual(producedMatrix, expectedMatrix);
  });
 });
--- a/test/units/truffle/flags.js
+++ b/test/units/truffle/flags.js
@ -257,5 +257,23 @@ describe('Truffle Plugin: command line options', function() {
      `Should have used default coverage port 8545: ${mock.loggerOutput.val}`
    );
  });
  it('--matrix', async function(){
    process.env.TRUFFLE_TEST = true; // Path to reporter differs btw HH and Truffle
    truffleConfig.matrix = true;
    mock.installFullProject('matrix');
    await plugin(truffleConfig);
    // Integration test checks output path configurabililty
    const altPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json'));
    const expPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json'));
    const producedMatrix = require(altPath)
    const expectedMatrix = require(expPath);
    assert.deepEqual(producedMatrix, expectedMatrix);
    process.env.TRUFFLE_TEST = false;
  });
 });