From afd7ef03763bc7901f0c021975461d2d974f3350 Mon Sep 17 00:00:00 2001
From: cgewecke <christophergewecke@gmail.com>
Date: Mon, 4 Jan 2021 17:07:58 -0800
Subject: [PATCH] Add test matrix generator (#593)

---
 README.md                                     |   4 +
 docs/advanced.md                              |  15 +
 docs/matrix.md                                | 421 ++++++++++++++++++
 lib/api.js                                    |  55 +++
 package.json                                  |   4 +-
 plugins/hardhat.plugin.js                     |  15 +-
 plugins/resources/matrix.js                   |  71 +++
 plugins/resources/nomiclabs.ui.js             |   5 +-
 plugins/resources/nomiclabs.utils.js          |  19 +-
 plugins/resources/plugin.utils.js             |   7 +
 plugins/resources/truffle.utils.js            |  17 +-
 plugins/truffle.plugin.js                     |  10 +-
 scripts/run-metacoin.sh                       |  10 +
 scripts/run-nomiclabs.sh                      |  13 +
 test/integration/projects/matrix/.solcover.js |  16 +
 .../projects/matrix/contracts/MatrixA.sol     |  17 +
 .../projects/matrix/contracts/MatrixB.sol     |  17 +
 .../matrix/expectedTestMatrixHardhat.json     |  46 ++
 .../matrix/expectedTestMatrixTruffle.json     |  46 ++
 .../projects/matrix/hardhat.config.js         |   9 +
 .../projects/matrix/test/matrix_a.js          |  15 +
 .../projects/matrix/test/matrix_a_b.js        |  30 ++
 .../projects/matrix/truffle-config.js         |  10 +
 test/units/hardhat/flags.js                   |  19 +
 test/units/truffle/flags.js                   |  18 +
 25 files changed, 897 insertions(+), 12 deletions(-)
 create mode 100644 docs/matrix.md
 create mode 100644 plugins/resources/matrix.js
 create mode 100644 test/integration/projects/matrix/.solcover.js
 create mode 100644 test/integration/projects/matrix/contracts/MatrixA.sol
 create mode 100644 test/integration/projects/matrix/contracts/MatrixB.sol
 create mode 100644 test/integration/projects/matrix/expectedTestMatrixHardhat.json
 create mode 100644 test/integration/projects/matrix/expectedTestMatrixTruffle.json
 create mode 100644 test/integration/projects/matrix/hardhat.config.js
 create mode 100644 test/integration/projects/matrix/test/matrix_a.js
 create mode 100644 test/integration/projects/matrix/test/matrix_a_b.js
 create mode 100644 test/integration/projects/matrix/truffle-config.js

diff --git a/README.md b/README.md
index e57572a..f16a9d8 100644
--- a/README.md
+++ b/README.md
@@ -58,6 +58,8 @@ A working example can be found at [openzeppelin-contracts, here.][35]
 | solcoverjs | `--solcoverjs ./../.solcover.js` | Relative path from working directory to config. Useful for monorepo packages that share settings. (Path must be "./" prefixed) |
 | network    | `--network development` | Use network settings defined in the Hardhat config |
 | temp[<sup>*</sup>][14]       | `--temp build`   | :warning: **Caution** :warning:  Path to a *disposable* folder to store compilation artifacts in. Useful when your test setup scripts include hard-coded paths to a build directory. [More...][14] |
+| matrix   | `--matrix` | Generate a JSON object that maps which mocha tests hit which lines of code. (Useful
+as an input for some fuzzing, mutation testing and fault-localization algorithms.) [More...][39]|
 
 [<sup>*</sup> Advanced use][14]
 
@@ -82,6 +84,7 @@ module.exports = {
 | measureStatementCoverage | *boolean* | `true` | Computes statement (in addition to line) coverage. [More...][34] |
 | measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] |
 | measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] |
+| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39] |
 | istanbulFolder | *String* | `./coverage` |  Folder location for Istanbul coverage reports. |
 | istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2]  |
 | mocha | *Object* | `{ }` | [Mocha options][3] to merge into existing mocha config. `grep` and `invert` are useful for skipping certain tests under coverage using tags in the test descriptions.|
@@ -212,6 +215,7 @@ $ yarn
 [36]: https://hardhat.org/
 [37]: https://github.com/sc-forks/solidity-coverage/blob/master/HARDHAT_README.md
 [38]: https://github.com/sindresorhus/globby#globbing-patterns
+[39]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#generating-a-test-matrix
 [1001]: https://docs.soliditylang.org/en/v0.8.0/using-the-compiler.html#input-description
 [1002]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/faq.md#running-out-of-stack
 
diff --git a/docs/advanced.md b/docs/advanced.md
index 9d309ef..0e8c4da 100644
--- a/docs/advanced.md
+++ b/docs/advanced.md
@@ -106,3 +106,18 @@ Setting the `measureStatementCoverage` and/or `measureFunctionCoverage` options
 improve performance, lower the cost of execution and minimize complications that arise from `solc`'s
 limits on how large the compilation payload can be.
 
+## Generating a test matrix
+
+Some advanced testing strategies benefit from knowing which tests in a suite hit a
+specific line of code. Examples include:
++ [mutation testing][22], where this data lets you select the correct subset of tests to check
+a mutation with.
++ [fault localization techniques][23], where the complete data set is a key input to algorithms that try
+to guess where bugs might exist in a given codebase.
+
+Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable
+test names to each line of code to a file named `testMatrix.json` in your project's root.
+
+[22]: https://github.com/JoranHonig/vertigo#vertigo
+[23]: http://spideruci.org/papers/jones05.pdf
+[25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md
diff --git a/docs/matrix.md b/docs/matrix.md
new file mode 100644
index 0000000..585186c
--- /dev/null
+++ b/docs/matrix.md
@@ -0,0 +1,421 @@
+### Test Matrix Example
+
+An example of output written to the file `./testMatrix.json` when coverage
+is run with the `--matrix` cli flag. (Source project: [sc-forks/hardhat-e2e][1])
+
+[1]: https://github.com/sc-forks/hardhat-e2e
+
+
+```js
+// Paths are relative to the project root directory
+{
+ // Solidity file name
+ "contracts/EtherRouter/EtherRouter.sol": {
+
+  // Line number
+  "23": [
+   {
+    // Grep-able mocha test title
+    "title": "Resolves methods routed through an EtherRouter proxy",
+
+    // Selectable mocha test file
+    "file": "test/etherrouter.js"
+   }
+  ],
+  "42": [
+   {
+    "title": "Resolves methods routed through an EtherRouter proxy",
+    "file": "test/etherrouter.js"
+   }
+  ],
+  "45": [
+   {
+    "title": "Resolves methods routed through an EtherRouter proxy",
+    "file": "test/etherrouter.js"
+   }
+  ],
+  "61": [
+   {
+    "title": "Resolves methods routed through an EtherRouter proxy",
+    "file": "test/etherrouter.js"
+   }
+  ]
+ },
+ "contracts/EtherRouter/Factory.sol": {
+  "19": [
+   {
+    "title": "Resolves methods routed through an EtherRouter proxy",
+    "file": "test/etherrouter.js"
+   }
+  ]
+ },
+ "contracts/EtherRouter/Resolver.sol": {
+  "22": [
+   {
+    "title": "Resolves methods routed through an EtherRouter proxy",
+    "file": "test/etherrouter.js"
+   }
+  ],
+  "26": [
+   {
+    "title": "Resolves methods routed through an EtherRouter proxy",
+    "file": "test/etherrouter.js"
+   }
+  ],
+  "30": [
+   {
+    "title": "Resolves methods routed through an EtherRouter proxy",
+    "file": "test/etherrouter.js"
+   }
+  ]
+ },
+ "contracts/MetaCoin.sol": {
+  "16": [
+   {
+    "title": "should put 10000 MetaCoin in the first account",
+    "file": "test/metacoin.js"
+   },
+   {
+    "title": "should call a function that depends on a linked library",
+    "file": "test/metacoin.js"
+   },
+   {
+    "title": "should send coin correctly",
+    "file": "test/metacoin.js"
+   },
+   {
+    "title": "a and b",
+    "file": "test/multicontract.js"
+   }
+  ],
+  "20": [
+   {
+    "title": "should send coin correctly",
+    "file": "test/metacoin.js"
+   }
+  ],
+  "21": [
+   {
+    "title": "should send coin correctly",
+    "file": "test/metacoin.js"
+   }
+  ],
+  "22": [
+   {
+    "title": "should send coin correctly",
+    "file": "test/metacoin.js"
+   }
+  ],
+  "23": [
+   {
+    "title": "should send coin correctly",
+    "file": "test/metacoin.js"
+   }
+  ],
+  "24": [
+   {
+    "title": "should send coin correctly",
+    "file": "test/metacoin.js"
+   }
+  ],
+  "28": [
+   {
+    "title": "should call a function that depends on a linked library",
+    "file": "test/metacoin.js"
+   }
+  ],
+  "32": [
+   {
+    "title": "should put 10000 MetaCoin in the first account",
+    "file": "test/metacoin.js"
+   },
+   {
+    "title": "should call a function that depends on a linked library",
+    "file": "test/metacoin.js"
+   },
+   {
+    "title": "should send coin correctly",
+    "file": "test/metacoin.js"
+   }
+  ]
+ },
+ "contracts/ConvertLib.sol": {
+  "6": [
+   {
+    "title": "should call a function that depends on a linked library",
+    "file": "test/metacoin.js"
+   }
+  ]
+ },
+ "contracts/MultiContractFile.sol": {
+  "7": [
+   {
+    "title": "a and b",
+    "file": "test/multicontract.js"
+   },
+   {
+    "title": "methods that call methods in other contracts",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "15": [
+   {
+    "title": "a and b",
+    "file": "test/multicontract.js"
+   }
+  ]
+ },
+ "contracts/VariableConstructor.sol": {
+  "8": [
+   {
+    "title": "should should initialize with a short string",
+    "file": "test/variableconstructor.js"
+   },
+   {
+    "title": "should should initialize with a medium length string",
+    "file": "test/variableconstructor.js"
+   },
+   {
+    "title": "should should initialize with a long string",
+    "file": "test/variableconstructor.js"
+   },
+   {
+    "title": "should should initialize with a random length string",
+    "file": "test/variableconstructor.js"
+   }
+  ]
+ },
+ "contracts/VariableCosts.sol": {
+  "13": [
+   {
+    "title": "should should initialize with a short string",
+    "file": "test/variableconstructor.js"
+   },
+   {
+    "title": "should should initialize with a medium length string",
+    "file": "test/variableconstructor.js"
+   },
+   {
+    "title": "should should initialize with a long string",
+    "file": "test/variableconstructor.js"
+   },
+   {
+    "title": "should should initialize with a random length string",
+    "file": "test/variableconstructor.js"
+   },
+   {
+    "title": "should add one",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add three",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add even 5!",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should delete one",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should delete three",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should delete five",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add five and delete one",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should set a random length string",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "methods that do not throw",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "methods that throw",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "methods that call methods in other contracts",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should allow contracts to have identically named methods",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "29": [
+   {
+    "title": "should add one",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add three",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add even 5!",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add five and delete one",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "30": [
+   {
+    "title": "should add one",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add three",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add even 5!",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add five and delete one",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "34": [
+   {
+    "title": "should delete one",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should delete three",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should delete five",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add five and delete one",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "35": [
+   {
+    "title": "should delete one",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should delete three",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should delete five",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should add five and delete one",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "43": [
+   {
+    "title": "should set a random length string",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "47": [
+   {
+    "title": "methods that do not throw",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "methods that throw",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "48": [
+   {
+    "title": "methods that do not throw",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "52": [
+   {
+    "title": "methods that call methods in other contracts",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "53": [
+   {
+    "title": "methods that call methods in other contracts",
+    "file": "test/variablecosts.js"
+   }
+  ],
+  "54": [
+   {
+    "title": "methods that call methods in other contracts",
+    "file": "test/variablecosts.js"
+   }
+  ]
+ },
+ "contracts/Wallets/Wallet.sol": {
+  "8": [
+   {
+    "title": "should allow contracts to have identically named methods",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should should allow transfers and sends",
+    "file": "test/wallet.js"
+   }
+  ],
+  "12": [
+   {
+    "title": "should allow contracts to have identically named methods",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should should allow transfers and sends",
+    "file": "test/wallet.js"
+   }
+  ],
+  "17": [
+   {
+    "title": "should allow contracts to have identically named methods",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should should allow transfers and sends",
+    "file": "test/wallet.js"
+   }
+  ],
+  "22": [
+   {
+    "title": "should allow contracts to have identically named methods",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should should allow transfers and sends",
+    "file": "test/wallet.js"
+   }
+  ],
+  "23": [
+   {
+    "title": "should allow contracts to have identically named methods",
+    "file": "test/variablecosts.js"
+   },
+   {
+    "title": "should should allow transfers and sends",
+    "file": "test/wallet.js"
+   }
+  ]
+ }
+}
+```
\ No newline at end of file
diff --git a/lib/api.js b/lib/api.js
index 33ce2b5..d0f89c8 100644
--- a/lib/api.js
+++ b/lib/api.js
@@ -20,6 +20,7 @@ class API {
   constructor(config={}) {
     this.validator = new ConfigValidator()
     this.config = config || {};
+    this.testMatrix = {};
 
     // Validate
     this.validator.validate(this.config);
@@ -30,6 +31,8 @@ class API {
     this.testsErrored = false;
 
     this.cwd = config.cwd || process.cwd();
+    this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json";
+    this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js"
 
     this.defaultHook = () => {};
     this.onServerReady = config.onServerReady           || this.defaultHook;
@@ -296,6 +299,52 @@ class API {
     }
   }*/
 
+  // ==========================
+  // Test Matrix Data Collector
+  // ==========================
+  /**
+   * @param  {Object} testInfo Mocha object passed to reporter 'test end' event
+   */
+  collectTestMatrixData(testInfo){
+    const hashes = Object.keys(this.instrumenter.instrumentationData);
+    const title = testInfo.title;
+    const file = path.relative(this.cwd, testInfo.file);
+
+    for (const hash of hashes){
+      const {
+        contractPath,
+        hits,
+        type,
+        id
+      } = this.instrumenter.instrumentationData[hash];
+
+      if (type === 'line' && hits > 0){
+        if (!this.testMatrix[contractPath]){
+          this.testMatrix[contractPath] = {};
+        }
+        if (!this.testMatrix[contractPath][id]){
+          this.testMatrix[contractPath][id] = [];
+        }
+
+        // Search for and exclude duplicate entries
+        let duplicate = false;
+        for (const item of this.testMatrix[contractPath][id]){
+          if (item.title === title && item.file === file){
+            duplicate = true;
+            break;
+          }
+        }
+
+        if (!duplicate) {
+          this.testMatrix[contractPath][id].push({title, file});
+        }
+
+        // Reset line data
+        this.instrumenter.instrumentationData[hash].hits = 0;
+      }
+    }
+  }
+
   // ========
   // File I/O
   // ========
@@ -305,6 +354,12 @@ class API {
     fs.writeFileSync(covPath, JSON.stringify(data));
   }
 
+  saveTestMatrix(){
+    const matrixPath = path.join(this.cwd, this.matrixOutputPath);
+    const mapping = this.makeKeysRelative(this.testMatrix, this.cwd);
+    fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' '));
+  }
+
   // =====
   // Paths
   // =====
diff --git a/package.json b/package.json
index a8555d1..df695f7 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,7 @@
   "scripts": {
     "nyc": "SILENT=true  nyc --exclude '**/sc_temp/**' --exclude '**/test/**'",
     "test": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
-    "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
+    "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' --exclude 'plugins/resources/matrix.js' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
     "test:debug": "node --max-old-space-size=4096 ./node_modules/.bin/mocha test/units/* --timeout 100000 --no-warnings --exit",
     "netlify": "./scripts/run-netlify.sh"
   },
@@ -35,6 +35,7 @@
     "globby": "^10.0.1",
     "jsonschema": "^1.2.4",
     "lodash": "^4.17.15",
+    "mocha": "7.1.2",
     "node-emoji": "^1.10.0",
     "pify": "^4.0.1",
     "recursive-readdir": "^2.2.2",
@@ -60,7 +61,6 @@
     "ganache-cli": "6.12.2",
     "hardhat": "^2.9.3",
     "hardhat-gas-reporter": "^1.0.1",
-    "mocha": "5.2.0",
     "nyc": "^14.1.1",
     "solc": "^0.7.5",
     "truffle": "5.1.43",
diff --git a/plugins/hardhat.plugin.js b/plugins/hardhat.plugin.js
index d695be3..6342570 100644
--- a/plugins/hardhat.plugin.js
+++ b/plugins/hardhat.plugin.js
@@ -97,6 +97,7 @@ task("coverage", "Generates a code coverage report for tests")
   .addOptionalParam("testfiles",  ui.flags.file,       "", types.string)
   .addOptionalParam("solcoverjs", ui.flags.solcoverjs, "", types.string)
   .addOptionalParam('temp',       ui.flags.temp,       "", types.string)
+  .addFlag('matrix', ui.flags.testMatrix)
   .setAction(async function(args, env){
 
   const API = require('./../lib/api');
@@ -232,6 +233,9 @@ task("coverage", "Generates a code coverage report for tests")
       ? nomiclabsUtils.getTestFilePaths(args.testfiles)
       : [];
 
+    // Optionally collect tests-per-line-of-code data
+    nomiclabsUtils.collectTestMatrixData(args, env, api);
+
     try {
       failedTests = await env.run(TASK_TEST, {testFiles: testfiles})
     } catch (e) {
@@ -239,10 +243,13 @@ task("coverage", "Generates a code coverage report for tests")
     }
     await api.onTestsComplete(config);
 
-    // ========
-    // Istanbul
-    // ========
-    await api.report();
+    // =================================
+    // Output (Istanbul or Test Matrix)
+    // =================================
+    (args.matrix)
+      ? await api.saveTestMatrix()
+      : await api.report();
+
     await api.onIstanbulComplete(config);
 
   } catch(e) {
diff --git a/plugins/resources/matrix.js b/plugins/resources/matrix.js
new file mode 100644
index 0000000..8701e53
--- /dev/null
+++ b/plugins/resources/matrix.js
@@ -0,0 +1,71 @@
+const mocha = require("mocha");
+const inherits = require("util").inherits;
+const Spec = mocha.reporters.Spec;
+
+
+/**
+ * This file adapted from mocha's stats-collector
+ * https://github.com/mochajs/mocha/blob/54475eb4ca35a2c9044a1b8c59a60f09c73e6c01/lib/stats-collector.js#L1-L83
+ */
+const Date = global.Date;
+
+/**
+ * Provides stats such as test duration, number of tests passed / failed etc., by
+ * listening for events emitted by `runner`.
+ */
+function mochaStats(runner) {
+  const stats = {
+    suites: 0,
+    tests: 0,
+    passes: 0,
+    pending: 0,
+    failures: 0
+  };
+
+  if (!runner) throw new Error("Missing runner argument");
+
+  runner.stats = stats;
+
+  runner.on("pass", () => stats.passes++);
+  runner.on("fail", () => stats.failures++);
+  runner.on("pending", () => stats.pending++);
+  runner.on("test end", () => stats.tests++);
+
+  runner.once("start", () => (stats.start = new Date()));
+
+  runner.once("end", function() {
+    stats.end = new Date();
+    stats.duration = stats.end - stats.start;
+  });
+}
+
+/**
+ * Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run
+ * and collects data about which tests hit which lines of code.
+ * This "test matrix" can be used as an input to
+ *
+ *
+ * @param {Object} runner  mocha's runner
+ * @param {Object} options reporter.options (see README example usage)
+ */
+function Matrix(runner, options) {
+  // Spec reporter
+  Spec.call(this, runner, options);
+
+  // Initialize stats for Mocha 6+ epilogue
+  if (!runner.stats) {
+    mochaStats(runner);
+    this.stats = runner.stats;
+  }
+
+  runner.on("test end", (info) => {
+    options.reporterOptions.collectTestMatrixData(info);
+  });
+}
+
+/**
+ * Inherit from `Base.prototype`.
+ */
+inherits(Matrix, Spec);
+
+module.exports = Matrix;
\ No newline at end of file
diff --git a/plugins/resources/nomiclabs.ui.js b/plugins/resources/nomiclabs.ui.js
index 770fcba..7027095 100644
--- a/plugins/resources/nomiclabs.ui.js
+++ b/plugins/resources/nomiclabs.ui.js
@@ -8,7 +8,9 @@ class PluginUI extends UI {
     super(log);
 
     this.flags = {
-      file:       `Path (or glob) defining a subset of tests to run`,
+      testfiles:  `Path (or glob) defining a subset of tests to run`,
+
+      testMatrix: `Generate a json object which maps which unit tests hit which lines of code.`,
 
       solcoverjs: `Relative path from working directory to config. ` +
                   `Useful for monorepo packages that share settings.`,
@@ -16,7 +18,6 @@ class PluginUI extends UI {
       temp:       `Path to a disposable folder to store compilation artifacts in. ` +
                   `Useful when your test setup scripts include hard-coded paths to ` +
                   `a build directory.`,
-
     }
   }
 
diff --git a/plugins/resources/nomiclabs.utils.js b/plugins/resources/nomiclabs.utils.js
index fde8c30..4f2ed1c 100644
--- a/plugins/resources/nomiclabs.utils.js
+++ b/plugins/resources/nomiclabs.utils.js
@@ -37,6 +37,7 @@ function normalizeConfig(config, args={}){
   config.logger = config.logger ? config.logger : {log: null};
   config.solcoverjs = args.solcoverjs
   config.gasReporter = { enabled: false }
+  config.matrix = args.matrix;
 
   try {
     const hardhatPackage = require('hardhat/package.json');
@@ -165,6 +166,21 @@ function configureHttpProvider(networkConfig, api, ui){
   networkConfig.url = `http://${api.host}:${api.port}`;
 }
 
+/**
+ * Configures mocha to generate a json object which maps which tests
+ * hit which lines of code.
+ */
+function collectTestMatrixData(args, env, api){
+  if (args.matrix){
+    mochaConfig = env.config.mocha || {};
+    mochaConfig.reporter = api.matrixReporterPath;
+    mochaConfig.reporterOptions = {
+      collectTestMatrixData: api.collectTestMatrixData.bind(api)
+    }
+    env.config.mocha = mochaConfig;
+  }
+}
+
 /**
  * Sets the default `from` account field in the network that will be used.
  * This needs to be done after accounts are fetched from the launched client.
@@ -216,6 +232,7 @@ module.exports = {
   setupBuidlerNetwork,
   setupHardhatNetwork,
   getTestFilePaths,
-  setNetworkFrom
+  setNetworkFrom,
+  collectTestMatrixData
 }
 
diff --git a/plugins/resources/plugin.utils.js b/plugins/resources/plugin.utils.js
index 8ef7310..7bdeaa9 100644
--- a/plugins/resources/plugin.utils.js
+++ b/plugins/resources/plugin.utils.js
@@ -222,6 +222,13 @@ function loadSolcoverJS(config={}){
   coverageConfig.cwd = config.workingDir;
   coverageConfig.originalContractsDir = config.contractsDir;
 
+  if (config.matrix){
+    coverageConfig.measureBranchCoverage = false;
+    coverageConfig.measureFunctionCoverage = false;
+    coverageConfig.measureModifierCoverage = false;
+    coverageConfig.measureStatementCoverage = false;
+  }
+
   // Solidity-Coverage writes to Truffle config
   config.mocha = config.mocha || {};
 
diff --git a/plugins/resources/truffle.utils.js b/plugins/resources/truffle.utils.js
index 8e2cc70..5aaa21c 100644
--- a/plugins/resources/truffle.utils.js
+++ b/plugins/resources/truffle.utils.js
@@ -196,6 +196,20 @@ function normalizeConfig(config){
   return config;
 }
 
+/**
+ * Configures mocha to generate a json object which maps which tests
+ * hit which lines of code.
+ */
+function collectTestMatrixData(config, api){
+  if (config.matrix){
+    config.mocha = config.mocha || {};
+    config.mocha.reporter = api.matrixReporterPath;
+    config.mocha.reporterOptions = {
+      collectTestMatrixData: api.collectTestMatrixData.bind(api)
+    }
+  }
+}
+
 /**
  * Replacement logger which filters out compilation warnings triggered by injected trace
  * function definitions.
@@ -220,5 +234,6 @@ module.exports = {
   setNetworkFrom,
   loadLibrary,
   normalizeConfig,
-  filteredLogger
+  filteredLogger,
+  collectTestMatrixData
 }
diff --git a/plugins/truffle.plugin.js b/plugins/truffle.plugin.js
index c1850df..949077d 100644
--- a/plugins/truffle.plugin.js
+++ b/plugins/truffle.plugin.js
@@ -110,6 +110,7 @@ async function plugin(config){
     await api.onCompileComplete(config);
 
     config.test_files = await truffleUtils.getTestFilePaths(config);
+    truffleUtils.collectTestMatrixData(config, api);
     // Run tests
     try {
       failures = await truffle.test.run(config)
@@ -118,8 +119,13 @@ async function plugin(config){
     }
     await api.onTestsComplete(config);
 
-    // Run Istanbul
-    await api.report();
+    // =================================
+    // Output (Istanbul or Test Matrix)
+    // =================================
+    (config.matrix)
+      ? await api.saveTestMatrix()
+      : await api.report();
+
     await api.onIstanbulComplete(config);
 
   } catch(e){
diff --git a/scripts/run-metacoin.sh b/scripts/run-metacoin.sh
index 0ed037c..a489a62 100755
--- a/scripts/run-metacoin.sh
+++ b/scripts/run-metacoin.sh
@@ -53,3 +53,13 @@ if [ ! -d "coverage" ]; then
   exit 1
 fi
 
+npx truffle run coverage --matrix
+
+# Test that coverage/ was generated
+if [ ! -f "testMatrix.json" ]; then
+  echo "ERROR: no matrix file was created."
+  exit 1
+fi
+
+cat testMatrix.json
+
diff --git a/scripts/run-nomiclabs.sh b/scripts/run-nomiclabs.sh
index aed4c3e..3d62b77 100755
--- a/scripts/run-nomiclabs.sh
+++ b/scripts/run-nomiclabs.sh
@@ -13,6 +13,13 @@ function verifyCoverageExists {
   fi
 }
 
+function verifyMatrixExists {
+  if [ ! -f "testMatrix.json" ]; then
+    echo "ERROR: no matrix file was created."
+    exit 1
+  fi
+}
+
 # Get rid of any caches
 sudo rm -rf node_modules
 echo "NVM CURRENT >>>>>" && nvm current
@@ -47,6 +54,12 @@ npx hardhat coverage
 
 verifyCoverageExists
 
+npx hardhat coverage --matrix
+
+verifyMatrixExists
+
+cat testMatrix.json
+
 echo ""
 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
 echo "wighawag/hardhat-deploy                "
diff --git a/test/integration/projects/matrix/.solcover.js b/test/integration/projects/matrix/.solcover.js
new file mode 100644
index 0000000..992f82b
--- /dev/null
+++ b/test/integration/projects/matrix/.solcover.js
@@ -0,0 +1,16 @@
+// Testing hooks
+const fn = (msg, config) => config.logger.log(msg);
+const reporterPath = (process.env.TRUFFLE_TEST)
+  ? "./plugins/resources/matrix.js"
+  : "../plugins/resources/matrix.js";
+
+module.exports = {
+  // This is loaded directly from `./plugins` during unit tests. The default val is
+  // "solidity-coverage/plugins/resources/matrix.js"
+  matrixReporterPath: reporterPath,
+  matrixOutputPath: "alternateTestMatrix.json",
+
+  skipFiles: ['Migrations.sol'],
+  silent: process.env.SILENT ? true : false,
+  istanbulReporter: ['json-summary', 'text'],
+}
diff --git a/test/integration/projects/matrix/contracts/MatrixA.sol b/test/integration/projects/matrix/contracts/MatrixA.sol
new file mode 100644
index 0000000..aeac9bc
--- /dev/null
+++ b/test/integration/projects/matrix/contracts/MatrixA.sol
@@ -0,0 +1,17 @@
+pragma solidity ^0.7.0;
+
+
+contract MatrixA {
+  uint x;
+  constructor() public {
+  }
+
+  function sendFn() public {
+    x = 5;
+  }
+
+  function callFn() public pure returns (uint){
+    uint y = 5;
+    return y;
+  }
+}
diff --git a/test/integration/projects/matrix/contracts/MatrixB.sol b/test/integration/projects/matrix/contracts/MatrixB.sol
new file mode 100644
index 0000000..b1981c2
--- /dev/null
+++ b/test/integration/projects/matrix/contracts/MatrixB.sol
@@ -0,0 +1,17 @@
+pragma solidity ^0.7.0;
+
+
+contract MatrixB {
+  uint x;
+  constructor() public {
+  }
+
+  function sendFn() public {
+    x = 5;
+  }
+
+  function callFn() public pure returns (uint){
+    uint y = 5;
+    return y;
+  }
+}
diff --git a/test/integration/projects/matrix/expectedTestMatrixHardhat.json b/test/integration/projects/matrix/expectedTestMatrixHardhat.json
new file mode 100644
index 0000000..5afc260
--- /dev/null
+++ b/test/integration/projects/matrix/expectedTestMatrixHardhat.json
@@ -0,0 +1,46 @@
+{
+ "contracts/MatrixA.sol": {
+  "10": [
+   {
+    "title": "sends to A",
+    "file": "test/matrix_a_b.js"
+   },
+   {
+    "title": "sends",
+    "file": "test/matrix_a.js"
+   }
+  ],
+  "14": [
+   {
+    "title": "calls",
+    "file": "test/matrix_a.js"
+   }
+  ],
+  "15": [
+   {
+    "title": "calls",
+    "file": "test/matrix_a.js"
+   }
+  ]
+ },
+ "contracts/MatrixB.sol": {
+  "10": [
+   {
+    "title": "sends to B",
+    "file": "test/matrix_a_b.js"
+   }
+  ],
+  "14": [
+   {
+    "title": "calls B",
+    "file": "test/matrix_a_b.js"
+   }
+  ],
+  "15": [
+   {
+    "title": "calls B",
+    "file": "test/matrix_a_b.js"
+   }
+  ]
+ }
+}
\ No newline at end of file
diff --git a/test/integration/projects/matrix/expectedTestMatrixTruffle.json b/test/integration/projects/matrix/expectedTestMatrixTruffle.json
new file mode 100644
index 0000000..f57487a
--- /dev/null
+++ b/test/integration/projects/matrix/expectedTestMatrixTruffle.json
@@ -0,0 +1,46 @@
+{
+ "contracts/MatrixA.sol": {
+  "10": [
+   {
+    "title": "sends",
+    "file": "test/matrix_a.js"
+   },
+   {
+    "title": "sends to A",
+    "file": "test/matrix_a_b.js"
+   }
+  ],
+  "14": [
+   {
+    "title": "calls",
+    "file": "test/matrix_a.js"
+   }
+  ],
+  "15": [
+   {
+    "title": "calls",
+    "file": "test/matrix_a.js"
+   }
+  ]
+ },
+ "contracts/MatrixB.sol": {
+  "10": [
+   {
+    "title": "sends to B",
+    "file": "test/matrix_a_b.js"
+   }
+  ],
+  "14": [
+   {
+    "title": "calls B",
+    "file": "test/matrix_a_b.js"
+   }
+  ],
+  "15": [
+   {
+    "title": "calls B",
+    "file": "test/matrix_a_b.js"
+   }
+  ]
+ }
+}
\ No newline at end of file
diff --git a/test/integration/projects/matrix/hardhat.config.js b/test/integration/projects/matrix/hardhat.config.js
new file mode 100644
index 0000000..b402a97
--- /dev/null
+++ b/test/integration/projects/matrix/hardhat.config.js
@@ -0,0 +1,9 @@
+require("@nomiclabs/hardhat-truffle5");
+require(__dirname + "/../plugins/nomiclabs.plugin");
+
+module.exports={
+  solidity: {
+    version: "0.7.3"
+  },
+  logger: process.env.SILENT ? { log: () => {} } : console,
+};
diff --git a/test/integration/projects/matrix/test/matrix_a.js b/test/integration/projects/matrix/test/matrix_a.js
new file mode 100644
index 0000000..d1cde11
--- /dev/null
+++ b/test/integration/projects/matrix/test/matrix_a.js
@@ -0,0 +1,15 @@
+const MatrixA = artifacts.require("MatrixA");
+
+contract("MatrixA", function(accounts) {
+  let instance;
+
+  before(async () => instance = await MatrixA.new())
+
+  it('sends', async function(){
+    await instance.sendFn();
+  });
+
+  it('calls', async function(){
+    await instance.callFn();
+  })
+});
diff --git a/test/integration/projects/matrix/test/matrix_a_b.js b/test/integration/projects/matrix/test/matrix_a_b.js
new file mode 100644
index 0000000..6e37de9
--- /dev/null
+++ b/test/integration/projects/matrix/test/matrix_a_b.js
@@ -0,0 +1,30 @@
+const MatrixA = artifacts.require("MatrixA");
+const MatrixB = artifacts.require("MatrixB");
+
+contract("Matrix A and B", function(accounts) {
+  let instanceA;
+  let instanceB;
+
+  before(async () => {
+    instanceA = await MatrixA.new();
+    instanceB = await MatrixB.new();
+  })
+
+  it('sends to A', async function(){
+    await instanceA.sendFn();
+  });
+
+  // Duplicate test title and file should *not* be duplicated in the output
+  it('sends to A', async function(){
+    await instanceA.sendFn();
+  })
+
+  it('calls B', async function(){
+    await instanceB.callFn();
+  })
+
+  it('sends to B', async function(){
+    await instanceB.sendFn();
+  });
+
+});
diff --git a/test/integration/projects/matrix/truffle-config.js b/test/integration/projects/matrix/truffle-config.js
new file mode 100644
index 0000000..bac9791
--- /dev/null
+++ b/test/integration/projects/matrix/truffle-config.js
@@ -0,0 +1,10 @@
+module.exports = {
+  networks: {},
+  mocha: {},
+  compilers: {
+    solc: {
+      version: "0.7.3"
+    }
+  },
+  logger: process.env.SILENT ? { log: () => {} } : console,
+}
diff --git a/test/units/hardhat/flags.js b/test/units/hardhat/flags.js
index 5af5949..6127e1a 100644
--- a/test/units/hardhat/flags.js
+++ b/test/units/hardhat/flags.js
@@ -172,5 +172,24 @@ describe('Hardhat Plugin: command line options', function() {
     verify.lineCoverage(expected);
     shell.rm('.solcover.js');
   });
+
+  it('--matrix', async function(){
+    const taskArgs = {
+      matrix: true
+    }
+
+    mock.installFullProject('matrix');
+    mock.hardhatSetupEnv(this);
+
+    await this.env.run("coverage", taskArgs);
+
+    // Integration test checks output path configurabililty
+    const altPath = path.join(process.cwd(), './alternateTestMatrix.json');
+    const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
+    const producedMatrix = require(altPath)
+    const expectedMatrix = require(expPath);
+
+    assert.deepEqual(producedMatrix, expectedMatrix);
+  });
 });
 
diff --git a/test/units/truffle/flags.js b/test/units/truffle/flags.js
index 2365041..e073fc3 100644
--- a/test/units/truffle/flags.js
+++ b/test/units/truffle/flags.js
@@ -257,5 +257,23 @@ describe('Truffle Plugin: command line options', function() {
       `Should have used default coverage port 8545: ${mock.loggerOutput.val}`
     );
   });
+
+  it('--matrix', async function(){
+    process.env.TRUFFLE_TEST = true; // Path to reporter differs btw HH and Truffle
+    truffleConfig.matrix = true;
+
+    mock.installFullProject('matrix');
+    await plugin(truffleConfig);
+
+    // Integration test checks output path configurabililty
+    const altPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json'));
+    const expPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json'));
+
+    const producedMatrix = require(altPath)
+    const expectedMatrix = require(expPath);
+
+    assert.deepEqual(producedMatrix, expectedMatrix);
+    process.env.TRUFFLE_TEST = false;
+  });
 });