diff --git a/README.md b/README.md
index e57572a..f16a9d8 100644
--- a/README.md
+++ b/README.md
@@ -58,6 +58,8 @@ A working example can be found at [openzeppelin-contracts, here.][35]
| solcoverjs | `--solcoverjs ./../.solcover.js` | Relative path from working directory to config. Useful for monorepo packages that share settings. (Path must be "./" prefixed) |
| network | `--network development` | Use network settings defined in the Hardhat config |
| temp[*][14] | `--temp build` | :warning: **Caution** :warning: Path to a *disposable* folder to store compilation artifacts in. Useful when your test setup scripts include hard-coded paths to a build directory. [More...][14] |
+| matrix | `--matrix` | Generate a JSON object that maps which mocha tests hit which lines of code. (Useful
+as an input for some fuzzing, mutation testing and fault-localization algorithms.) [More...][39]|
[* Advanced use][14]
@@ -82,6 +84,7 @@ module.exports = {
| measureStatementCoverage | *boolean* | `true` | Computes statement (in addition to line) coverage. [More...][34] |
| measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] |
| measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] |
+| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][39] |
| istanbulFolder | *String* | `./coverage` | Folder location for Istanbul coverage reports. |
| istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2] |
| mocha | *Object* | `{ }` | [Mocha options][3] to merge into existing mocha config. `grep` and `invert` are useful for skipping certain tests under coverage using tags in the test descriptions.|
@@ -212,6 +215,7 @@ $ yarn
[36]: https://hardhat.org/
[37]: https://github.com/sc-forks/solidity-coverage/blob/master/HARDHAT_README.md
[38]: https://github.com/sindresorhus/globby#globbing-patterns
+[39]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#generating-a-test-matrix
[1001]: https://docs.soliditylang.org/en/v0.8.0/using-the-compiler.html#input-description
[1002]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/faq.md#running-out-of-stack
diff --git a/docs/advanced.md b/docs/advanced.md
index 9d309ef..0e8c4da 100644
--- a/docs/advanced.md
+++ b/docs/advanced.md
@@ -106,3 +106,18 @@ Setting the `measureStatementCoverage` and/or `measureFunctionCoverage` options
improve performance, lower the cost of execution and minimize complications that arise from `solc`'s
limits on how large the compilation payload can be.
+## Generating a test matrix
+
+Some advanced testing strategies benefit from knowing which tests in a suite hit a
+specific line of code. Examples include:
++ [mutation testing][22], where this data lets you select the correct subset of tests to check
+a mutation with.
++ [fault localization techniques][23], where the complete data set is a key input to algorithms that try
+to guess where bugs might exist in a given codebase.
+
+Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable
+test names to each line of code to a file named `testMatrix.json` in your project's root.
+
+[22]: https://github.com/JoranHonig/vertigo#vertigo
+[23]: http://spideruci.org/papers/jones05.pdf
+[25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md
diff --git a/docs/matrix.md b/docs/matrix.md
new file mode 100644
index 0000000..585186c
--- /dev/null
+++ b/docs/matrix.md
@@ -0,0 +1,421 @@
+### Test Matrix Example
+
+An example of output written to the file `./testMatrix.json` when coverage
+is run with the `--matrix` cli flag. (Source project: [sc-forks/hardhat-e2e][1])
+
+[1]: https://github.com/sc-forks/hardhat-e2e
+
+
+```js
+// Paths are relative to the project root directory
+{
+ // Solidity file name
+ "contracts/EtherRouter/EtherRouter.sol": {
+
+ // Line number
+ "23": [
+ {
+ // Grep-able mocha test title
+ "title": "Resolves methods routed through an EtherRouter proxy",
+
+ // Selectable mocha test file
+ "file": "test/etherrouter.js"
+ }
+ ],
+ "42": [
+ {
+ "title": "Resolves methods routed through an EtherRouter proxy",
+ "file": "test/etherrouter.js"
+ }
+ ],
+ "45": [
+ {
+ "title": "Resolves methods routed through an EtherRouter proxy",
+ "file": "test/etherrouter.js"
+ }
+ ],
+ "61": [
+ {
+ "title": "Resolves methods routed through an EtherRouter proxy",
+ "file": "test/etherrouter.js"
+ }
+ ]
+ },
+ "contracts/EtherRouter/Factory.sol": {
+ "19": [
+ {
+ "title": "Resolves methods routed through an EtherRouter proxy",
+ "file": "test/etherrouter.js"
+ }
+ ]
+ },
+ "contracts/EtherRouter/Resolver.sol": {
+ "22": [
+ {
+ "title": "Resolves methods routed through an EtherRouter proxy",
+ "file": "test/etherrouter.js"
+ }
+ ],
+ "26": [
+ {
+ "title": "Resolves methods routed through an EtherRouter proxy",
+ "file": "test/etherrouter.js"
+ }
+ ],
+ "30": [
+ {
+ "title": "Resolves methods routed through an EtherRouter proxy",
+ "file": "test/etherrouter.js"
+ }
+ ]
+ },
+ "contracts/MetaCoin.sol": {
+ "16": [
+ {
+ "title": "should put 10000 MetaCoin in the first account",
+ "file": "test/metacoin.js"
+ },
+ {
+ "title": "should call a function that depends on a linked library",
+ "file": "test/metacoin.js"
+ },
+ {
+ "title": "should send coin correctly",
+ "file": "test/metacoin.js"
+ },
+ {
+ "title": "a and b",
+ "file": "test/multicontract.js"
+ }
+ ],
+ "20": [
+ {
+ "title": "should send coin correctly",
+ "file": "test/metacoin.js"
+ }
+ ],
+ "21": [
+ {
+ "title": "should send coin correctly",
+ "file": "test/metacoin.js"
+ }
+ ],
+ "22": [
+ {
+ "title": "should send coin correctly",
+ "file": "test/metacoin.js"
+ }
+ ],
+ "23": [
+ {
+ "title": "should send coin correctly",
+ "file": "test/metacoin.js"
+ }
+ ],
+ "24": [
+ {
+ "title": "should send coin correctly",
+ "file": "test/metacoin.js"
+ }
+ ],
+ "28": [
+ {
+ "title": "should call a function that depends on a linked library",
+ "file": "test/metacoin.js"
+ }
+ ],
+ "32": [
+ {
+ "title": "should put 10000 MetaCoin in the first account",
+ "file": "test/metacoin.js"
+ },
+ {
+ "title": "should call a function that depends on a linked library",
+ "file": "test/metacoin.js"
+ },
+ {
+ "title": "should send coin correctly",
+ "file": "test/metacoin.js"
+ }
+ ]
+ },
+ "contracts/ConvertLib.sol": {
+ "6": [
+ {
+ "title": "should call a function that depends on a linked library",
+ "file": "test/metacoin.js"
+ }
+ ]
+ },
+ "contracts/MultiContractFile.sol": {
+ "7": [
+ {
+ "title": "a and b",
+ "file": "test/multicontract.js"
+ },
+ {
+ "title": "methods that call methods in other contracts",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "15": [
+ {
+ "title": "a and b",
+ "file": "test/multicontract.js"
+ }
+ ]
+ },
+ "contracts/VariableConstructor.sol": {
+ "8": [
+ {
+ "title": "should should initialize with a short string",
+ "file": "test/variableconstructor.js"
+ },
+ {
+ "title": "should should initialize with a medium length string",
+ "file": "test/variableconstructor.js"
+ },
+ {
+ "title": "should should initialize with a long string",
+ "file": "test/variableconstructor.js"
+ },
+ {
+ "title": "should should initialize with a random length string",
+ "file": "test/variableconstructor.js"
+ }
+ ]
+ },
+ "contracts/VariableCosts.sol": {
+ "13": [
+ {
+ "title": "should should initialize with a short string",
+ "file": "test/variableconstructor.js"
+ },
+ {
+ "title": "should should initialize with a medium length string",
+ "file": "test/variableconstructor.js"
+ },
+ {
+ "title": "should should initialize with a long string",
+ "file": "test/variableconstructor.js"
+ },
+ {
+ "title": "should should initialize with a random length string",
+ "file": "test/variableconstructor.js"
+ },
+ {
+ "title": "should add one",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add three",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add even 5!",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should delete one",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should delete three",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should delete five",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add five and delete one",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should set a random length string",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "methods that do not throw",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "methods that throw",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "methods that call methods in other contracts",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should allow contracts to have identically named methods",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "29": [
+ {
+ "title": "should add one",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add three",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add even 5!",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add five and delete one",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "30": [
+ {
+ "title": "should add one",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add three",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add even 5!",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add five and delete one",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "34": [
+ {
+ "title": "should delete one",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should delete three",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should delete five",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add five and delete one",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "35": [
+ {
+ "title": "should delete one",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should delete three",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should delete five",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should add five and delete one",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "43": [
+ {
+ "title": "should set a random length string",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "47": [
+ {
+ "title": "methods that do not throw",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "methods that throw",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "48": [
+ {
+ "title": "methods that do not throw",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "52": [
+ {
+ "title": "methods that call methods in other contracts",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "53": [
+ {
+ "title": "methods that call methods in other contracts",
+ "file": "test/variablecosts.js"
+ }
+ ],
+ "54": [
+ {
+ "title": "methods that call methods in other contracts",
+ "file": "test/variablecosts.js"
+ }
+ ]
+ },
+ "contracts/Wallets/Wallet.sol": {
+ "8": [
+ {
+ "title": "should allow contracts to have identically named methods",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should should allow transfers and sends",
+ "file": "test/wallet.js"
+ }
+ ],
+ "12": [
+ {
+ "title": "should allow contracts to have identically named methods",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should should allow transfers and sends",
+ "file": "test/wallet.js"
+ }
+ ],
+ "17": [
+ {
+ "title": "should allow contracts to have identically named methods",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should should allow transfers and sends",
+ "file": "test/wallet.js"
+ }
+ ],
+ "22": [
+ {
+ "title": "should allow contracts to have identically named methods",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should should allow transfers and sends",
+ "file": "test/wallet.js"
+ }
+ ],
+ "23": [
+ {
+ "title": "should allow contracts to have identically named methods",
+ "file": "test/variablecosts.js"
+ },
+ {
+ "title": "should should allow transfers and sends",
+ "file": "test/wallet.js"
+ }
+ ]
+ }
+}
+```
\ No newline at end of file
diff --git a/lib/api.js b/lib/api.js
index 33ce2b5..d0f89c8 100644
--- a/lib/api.js
+++ b/lib/api.js
@@ -20,6 +20,7 @@ class API {
constructor(config={}) {
this.validator = new ConfigValidator()
this.config = config || {};
+ this.testMatrix = {};
// Validate
this.validator.validate(this.config);
@@ -30,6 +31,8 @@ class API {
this.testsErrored = false;
this.cwd = config.cwd || process.cwd();
+ this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json";
+ this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js"
this.defaultHook = () => {};
this.onServerReady = config.onServerReady || this.defaultHook;
@@ -296,6 +299,52 @@ class API {
}
}*/
+ // ==========================
+ // Test Matrix Data Collector
+ // ==========================
+ /**
+ * @param {Object} testInfo Mocha object passed to reporter 'test end' event
+ */
+ collectTestMatrixData(testInfo){
+ const hashes = Object.keys(this.instrumenter.instrumentationData);
+ const title = testInfo.title;
+ const file = path.relative(this.cwd, testInfo.file);
+
+ for (const hash of hashes){
+ const {
+ contractPath,
+ hits,
+ type,
+ id
+ } = this.instrumenter.instrumentationData[hash];
+
+ if (type === 'line' && hits > 0){
+ if (!this.testMatrix[contractPath]){
+ this.testMatrix[contractPath] = {};
+ }
+ if (!this.testMatrix[contractPath][id]){
+ this.testMatrix[contractPath][id] = [];
+ }
+
+ // Search for and exclude duplicate entries
+ let duplicate = false;
+ for (const item of this.testMatrix[contractPath][id]){
+ if (item.title === title && item.file === file){
+ duplicate = true;
+ break;
+ }
+ }
+
+ if (!duplicate) {
+ this.testMatrix[contractPath][id].push({title, file});
+ }
+
+ // Reset line data
+ this.instrumenter.instrumentationData[hash].hits = 0;
+ }
+ }
+ }
+
// ========
// File I/O
// ========
@@ -305,6 +354,12 @@ class API {
fs.writeFileSync(covPath, JSON.stringify(data));
}
+ saveTestMatrix(){
+ const matrixPath = path.join(this.cwd, this.matrixOutputPath);
+ const mapping = this.makeKeysRelative(this.testMatrix, this.cwd);
+ fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' '));
+ }
+
// =====
// Paths
// =====
diff --git a/package.json b/package.json
index a8555d1..df695f7 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,7 @@
"scripts": {
"nyc": "SILENT=true nyc --exclude '**/sc_temp/**' --exclude '**/test/**'",
"test": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
- "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
+ "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' --exclude 'plugins/resources/matrix.js' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
"test:debug": "node --max-old-space-size=4096 ./node_modules/.bin/mocha test/units/* --timeout 100000 --no-warnings --exit",
"netlify": "./scripts/run-netlify.sh"
},
@@ -35,6 +35,7 @@
"globby": "^10.0.1",
"jsonschema": "^1.2.4",
"lodash": "^4.17.15",
+ "mocha": "7.1.2",
"node-emoji": "^1.10.0",
"pify": "^4.0.1",
"recursive-readdir": "^2.2.2",
@@ -60,7 +61,6 @@
"ganache-cli": "6.12.2",
"hardhat": "^2.9.3",
"hardhat-gas-reporter": "^1.0.1",
- "mocha": "5.2.0",
"nyc": "^14.1.1",
"solc": "^0.7.5",
"truffle": "5.1.43",
diff --git a/plugins/hardhat.plugin.js b/plugins/hardhat.plugin.js
index d695be3..6342570 100644
--- a/plugins/hardhat.plugin.js
+++ b/plugins/hardhat.plugin.js
@@ -97,6 +97,7 @@ task("coverage", "Generates a code coverage report for tests")
.addOptionalParam("testfiles", ui.flags.file, "", types.string)
.addOptionalParam("solcoverjs", ui.flags.solcoverjs, "", types.string)
.addOptionalParam('temp', ui.flags.temp, "", types.string)
+ .addFlag('matrix', ui.flags.testMatrix)
.setAction(async function(args, env){
const API = require('./../lib/api');
@@ -232,6 +233,9 @@ task("coverage", "Generates a code coverage report for tests")
? nomiclabsUtils.getTestFilePaths(args.testfiles)
: [];
+ // Optionally collect tests-per-line-of-code data
+ nomiclabsUtils.collectTestMatrixData(args, env, api);
+
try {
failedTests = await env.run(TASK_TEST, {testFiles: testfiles})
} catch (e) {
@@ -239,10 +243,13 @@ task("coverage", "Generates a code coverage report for tests")
}
await api.onTestsComplete(config);
- // ========
- // Istanbul
- // ========
- await api.report();
+ // =================================
+ // Output (Istanbul or Test Matrix)
+ // =================================
+ (args.matrix)
+ ? await api.saveTestMatrix()
+ : await api.report();
+
await api.onIstanbulComplete(config);
} catch(e) {
diff --git a/plugins/resources/matrix.js b/plugins/resources/matrix.js
new file mode 100644
index 0000000..8701e53
--- /dev/null
+++ b/plugins/resources/matrix.js
@@ -0,0 +1,71 @@
+const mocha = require("mocha");
+const inherits = require("util").inherits;
+const Spec = mocha.reporters.Spec;
+
+
+/**
+ * This file adapted from mocha's stats-collector
+ * https://github.com/mochajs/mocha/blob/54475eb4ca35a2c9044a1b8c59a60f09c73e6c01/lib/stats-collector.js#L1-L83
+ */
+const Date = global.Date;
+
+/**
+ * Provides stats such as test duration, number of tests passed / failed etc., by
+ * listening for events emitted by `runner`.
+ */
+function mochaStats(runner) {
+ const stats = {
+ suites: 0,
+ tests: 0,
+ passes: 0,
+ pending: 0,
+ failures: 0
+ };
+
+ if (!runner) throw new Error("Missing runner argument");
+
+ runner.stats = stats;
+
+ runner.on("pass", () => stats.passes++);
+ runner.on("fail", () => stats.failures++);
+ runner.on("pending", () => stats.pending++);
+ runner.on("test end", () => stats.tests++);
+
+ runner.once("start", () => (stats.start = new Date()));
+
+ runner.once("end", function() {
+ stats.end = new Date();
+ stats.duration = stats.end - stats.start;
+ });
+}
+
+/**
+ * Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run
+ * and collects data about which tests hit which lines of code.
+ * This "test matrix" can be used as an input to
+ *
+ *
+ * @param {Object} runner mocha's runner
+ * @param {Object} options reporter.options (see README example usage)
+ */
+function Matrix(runner, options) {
+ // Spec reporter
+ Spec.call(this, runner, options);
+
+ // Initialize stats for Mocha 6+ epilogue
+ if (!runner.stats) {
+ mochaStats(runner);
+ this.stats = runner.stats;
+ }
+
+ runner.on("test end", (info) => {
+ options.reporterOptions.collectTestMatrixData(info);
+ });
+}
+
+/**
+ * Inherit from `Base.prototype`.
+ */
+inherits(Matrix, Spec);
+
+module.exports = Matrix;
\ No newline at end of file
diff --git a/plugins/resources/nomiclabs.ui.js b/plugins/resources/nomiclabs.ui.js
index 770fcba..7027095 100644
--- a/plugins/resources/nomiclabs.ui.js
+++ b/plugins/resources/nomiclabs.ui.js
@@ -8,7 +8,9 @@ class PluginUI extends UI {
super(log);
this.flags = {
- file: `Path (or glob) defining a subset of tests to run`,
+ testfiles: `Path (or glob) defining a subset of tests to run`,
+
+ testMatrix: `Generate a json object which maps which unit tests hit which lines of code.`,
solcoverjs: `Relative path from working directory to config. ` +
`Useful for monorepo packages that share settings.`,
@@ -16,7 +18,6 @@ class PluginUI extends UI {
temp: `Path to a disposable folder to store compilation artifacts in. ` +
`Useful when your test setup scripts include hard-coded paths to ` +
`a build directory.`,
-
}
}
diff --git a/plugins/resources/nomiclabs.utils.js b/plugins/resources/nomiclabs.utils.js
index fde8c30..4f2ed1c 100644
--- a/plugins/resources/nomiclabs.utils.js
+++ b/plugins/resources/nomiclabs.utils.js
@@ -37,6 +37,7 @@ function normalizeConfig(config, args={}){
config.logger = config.logger ? config.logger : {log: null};
config.solcoverjs = args.solcoverjs
config.gasReporter = { enabled: false }
+ config.matrix = args.matrix;
try {
const hardhatPackage = require('hardhat/package.json');
@@ -165,6 +166,21 @@ function configureHttpProvider(networkConfig, api, ui){
networkConfig.url = `http://${api.host}:${api.port}`;
}
+/**
+ * Configures mocha to generate a json object which maps which tests
+ * hit which lines of code.
+ */
+function collectTestMatrixData(args, env, api){
+ if (args.matrix){
+ mochaConfig = env.config.mocha || {};
+ mochaConfig.reporter = api.matrixReporterPath;
+ mochaConfig.reporterOptions = {
+ collectTestMatrixData: api.collectTestMatrixData.bind(api)
+ }
+ env.config.mocha = mochaConfig;
+ }
+}
+
/**
* Sets the default `from` account field in the network that will be used.
* This needs to be done after accounts are fetched from the launched client.
@@ -216,6 +232,7 @@ module.exports = {
setupBuidlerNetwork,
setupHardhatNetwork,
getTestFilePaths,
- setNetworkFrom
+ setNetworkFrom,
+ collectTestMatrixData
}
diff --git a/plugins/resources/plugin.utils.js b/plugins/resources/plugin.utils.js
index 8ef7310..7bdeaa9 100644
--- a/plugins/resources/plugin.utils.js
+++ b/plugins/resources/plugin.utils.js
@@ -222,6 +222,13 @@ function loadSolcoverJS(config={}){
coverageConfig.cwd = config.workingDir;
coverageConfig.originalContractsDir = config.contractsDir;
+ if (config.matrix){
+ coverageConfig.measureBranchCoverage = false;
+ coverageConfig.measureFunctionCoverage = false;
+ coverageConfig.measureModifierCoverage = false;
+ coverageConfig.measureStatementCoverage = false;
+ }
+
// Solidity-Coverage writes to Truffle config
config.mocha = config.mocha || {};
diff --git a/plugins/resources/truffle.utils.js b/plugins/resources/truffle.utils.js
index 8e2cc70..5aaa21c 100644
--- a/plugins/resources/truffle.utils.js
+++ b/plugins/resources/truffle.utils.js
@@ -196,6 +196,20 @@ function normalizeConfig(config){
return config;
}
+/**
+ * Configures mocha to generate a json object which maps which tests
+ * hit which lines of code.
+ */
+function collectTestMatrixData(config, api){
+ if (config.matrix){
+ config.mocha = config.mocha || {};
+ config.mocha.reporter = api.matrixReporterPath;
+ config.mocha.reporterOptions = {
+ collectTestMatrixData: api.collectTestMatrixData.bind(api)
+ }
+ }
+}
+
/**
* Replacement logger which filters out compilation warnings triggered by injected trace
* function definitions.
@@ -220,5 +234,6 @@ module.exports = {
setNetworkFrom,
loadLibrary,
normalizeConfig,
- filteredLogger
+ filteredLogger,
+ collectTestMatrixData
}
diff --git a/plugins/truffle.plugin.js b/plugins/truffle.plugin.js
index c1850df..949077d 100644
--- a/plugins/truffle.plugin.js
+++ b/plugins/truffle.plugin.js
@@ -110,6 +110,7 @@ async function plugin(config){
await api.onCompileComplete(config);
config.test_files = await truffleUtils.getTestFilePaths(config);
+ truffleUtils.collectTestMatrixData(config, api);
// Run tests
try {
failures = await truffle.test.run(config)
@@ -118,8 +119,13 @@ async function plugin(config){
}
await api.onTestsComplete(config);
- // Run Istanbul
- await api.report();
+ // =================================
+ // Output (Istanbul or Test Matrix)
+ // =================================
+ (config.matrix)
+ ? await api.saveTestMatrix()
+ : await api.report();
+
await api.onIstanbulComplete(config);
} catch(e){
diff --git a/scripts/run-metacoin.sh b/scripts/run-metacoin.sh
index 0ed037c..a489a62 100755
--- a/scripts/run-metacoin.sh
+++ b/scripts/run-metacoin.sh
@@ -53,3 +53,13 @@ if [ ! -d "coverage" ]; then
exit 1
fi
+npx truffle run coverage --matrix
+
+# Test that coverage/ was generated
+if [ ! -f "testMatrix.json" ]; then
+ echo "ERROR: no matrix file was created."
+ exit 1
+fi
+
+cat testMatrix.json
+
diff --git a/scripts/run-nomiclabs.sh b/scripts/run-nomiclabs.sh
index aed4c3e..3d62b77 100755
--- a/scripts/run-nomiclabs.sh
+++ b/scripts/run-nomiclabs.sh
@@ -13,6 +13,13 @@ function verifyCoverageExists {
fi
}
+function verifyMatrixExists {
+ if [ ! -f "testMatrix.json" ]; then
+ echo "ERROR: no matrix file was created."
+ exit 1
+ fi
+}
+
# Get rid of any caches
sudo rm -rf node_modules
echo "NVM CURRENT >>>>>" && nvm current
@@ -47,6 +54,12 @@ npx hardhat coverage
verifyCoverageExists
+npx hardhat coverage --matrix
+
+verifyMatrixExists
+
+cat testMatrix.json
+
echo ""
echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
echo "wighawag/hardhat-deploy "
diff --git a/test/integration/projects/matrix/.solcover.js b/test/integration/projects/matrix/.solcover.js
new file mode 100644
index 0000000..992f82b
--- /dev/null
+++ b/test/integration/projects/matrix/.solcover.js
@@ -0,0 +1,16 @@
+// Testing hooks
+const fn = (msg, config) => config.logger.log(msg);
+const reporterPath = (process.env.TRUFFLE_TEST)
+ ? "./plugins/resources/matrix.js"
+ : "../plugins/resources/matrix.js";
+
+module.exports = {
+ // This is loaded directly from `./plugins` during unit tests. The default val is
+ // "solidity-coverage/plugins/resources/matrix.js"
+ matrixReporterPath: reporterPath,
+ matrixOutputPath: "alternateTestMatrix.json",
+
+ skipFiles: ['Migrations.sol'],
+ silent: process.env.SILENT ? true : false,
+ istanbulReporter: ['json-summary', 'text'],
+}
diff --git a/test/integration/projects/matrix/contracts/MatrixA.sol b/test/integration/projects/matrix/contracts/MatrixA.sol
new file mode 100644
index 0000000..aeac9bc
--- /dev/null
+++ b/test/integration/projects/matrix/contracts/MatrixA.sol
@@ -0,0 +1,17 @@
+pragma solidity ^0.7.0;
+
+
+contract MatrixA {
+ uint x;
+ constructor() public {
+ }
+
+ function sendFn() public {
+ x = 5;
+ }
+
+ function callFn() public pure returns (uint){
+ uint y = 5;
+ return y;
+ }
+}
diff --git a/test/integration/projects/matrix/contracts/MatrixB.sol b/test/integration/projects/matrix/contracts/MatrixB.sol
new file mode 100644
index 0000000..b1981c2
--- /dev/null
+++ b/test/integration/projects/matrix/contracts/MatrixB.sol
@@ -0,0 +1,17 @@
+pragma solidity ^0.7.0;
+
+
+contract MatrixB {
+ uint x;
+ constructor() public {
+ }
+
+ function sendFn() public {
+ x = 5;
+ }
+
+ function callFn() public pure returns (uint){
+ uint y = 5;
+ return y;
+ }
+}
diff --git a/test/integration/projects/matrix/expectedTestMatrixHardhat.json b/test/integration/projects/matrix/expectedTestMatrixHardhat.json
new file mode 100644
index 0000000..5afc260
--- /dev/null
+++ b/test/integration/projects/matrix/expectedTestMatrixHardhat.json
@@ -0,0 +1,46 @@
+{
+ "contracts/MatrixA.sol": {
+ "10": [
+ {
+ "title": "sends to A",
+ "file": "test/matrix_a_b.js"
+ },
+ {
+ "title": "sends",
+ "file": "test/matrix_a.js"
+ }
+ ],
+ "14": [
+ {
+ "title": "calls",
+ "file": "test/matrix_a.js"
+ }
+ ],
+ "15": [
+ {
+ "title": "calls",
+ "file": "test/matrix_a.js"
+ }
+ ]
+ },
+ "contracts/MatrixB.sol": {
+ "10": [
+ {
+ "title": "sends to B",
+ "file": "test/matrix_a_b.js"
+ }
+ ],
+ "14": [
+ {
+ "title": "calls B",
+ "file": "test/matrix_a_b.js"
+ }
+ ],
+ "15": [
+ {
+ "title": "calls B",
+ "file": "test/matrix_a_b.js"
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/test/integration/projects/matrix/expectedTestMatrixTruffle.json b/test/integration/projects/matrix/expectedTestMatrixTruffle.json
new file mode 100644
index 0000000..f57487a
--- /dev/null
+++ b/test/integration/projects/matrix/expectedTestMatrixTruffle.json
@@ -0,0 +1,46 @@
+{
+ "contracts/MatrixA.sol": {
+ "10": [
+ {
+ "title": "sends",
+ "file": "test/matrix_a.js"
+ },
+ {
+ "title": "sends to A",
+ "file": "test/matrix_a_b.js"
+ }
+ ],
+ "14": [
+ {
+ "title": "calls",
+ "file": "test/matrix_a.js"
+ }
+ ],
+ "15": [
+ {
+ "title": "calls",
+ "file": "test/matrix_a.js"
+ }
+ ]
+ },
+ "contracts/MatrixB.sol": {
+ "10": [
+ {
+ "title": "sends to B",
+ "file": "test/matrix_a_b.js"
+ }
+ ],
+ "14": [
+ {
+ "title": "calls B",
+ "file": "test/matrix_a_b.js"
+ }
+ ],
+ "15": [
+ {
+ "title": "calls B",
+ "file": "test/matrix_a_b.js"
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/test/integration/projects/matrix/hardhat.config.js b/test/integration/projects/matrix/hardhat.config.js
new file mode 100644
index 0000000..b402a97
--- /dev/null
+++ b/test/integration/projects/matrix/hardhat.config.js
@@ -0,0 +1,9 @@
+require("@nomiclabs/hardhat-truffle5");
+require(__dirname + "/../plugins/nomiclabs.plugin");
+
+module.exports={
+ solidity: {
+ version: "0.7.3"
+ },
+ logger: process.env.SILENT ? { log: () => {} } : console,
+};
diff --git a/test/integration/projects/matrix/test/matrix_a.js b/test/integration/projects/matrix/test/matrix_a.js
new file mode 100644
index 0000000..d1cde11
--- /dev/null
+++ b/test/integration/projects/matrix/test/matrix_a.js
@@ -0,0 +1,15 @@
+const MatrixA = artifacts.require("MatrixA");
+
+contract("MatrixA", function(accounts) {
+ let instance;
+
+ before(async () => instance = await MatrixA.new())
+
+ it('sends', async function(){
+ await instance.sendFn();
+ });
+
+ it('calls', async function(){
+ await instance.callFn();
+ })
+});
diff --git a/test/integration/projects/matrix/test/matrix_a_b.js b/test/integration/projects/matrix/test/matrix_a_b.js
new file mode 100644
index 0000000..6e37de9
--- /dev/null
+++ b/test/integration/projects/matrix/test/matrix_a_b.js
@@ -0,0 +1,30 @@
+const MatrixA = artifacts.require("MatrixA");
+const MatrixB = artifacts.require("MatrixB");
+
+contract("Matrix A and B", function(accounts) {
+ let instanceA;
+ let instanceB;
+
+ before(async () => {
+ instanceA = await MatrixA.new();
+ instanceB = await MatrixB.new();
+ })
+
+ it('sends to A', async function(){
+ await instanceA.sendFn();
+ });
+
+ // Duplicate test title and file should *not* be duplicated in the output
+ it('sends to A', async function(){
+ await instanceA.sendFn();
+ })
+
+ it('calls B', async function(){
+ await instanceB.callFn();
+ })
+
+ it('sends to B', async function(){
+ await instanceB.sendFn();
+ });
+
+});
diff --git a/test/integration/projects/matrix/truffle-config.js b/test/integration/projects/matrix/truffle-config.js
new file mode 100644
index 0000000..bac9791
--- /dev/null
+++ b/test/integration/projects/matrix/truffle-config.js
@@ -0,0 +1,10 @@
+module.exports = {
+ networks: {},
+ mocha: {},
+ compilers: {
+ solc: {
+ version: "0.7.3"
+ }
+ },
+ logger: process.env.SILENT ? { log: () => {} } : console,
+}
diff --git a/test/units/hardhat/flags.js b/test/units/hardhat/flags.js
index 5af5949..6127e1a 100644
--- a/test/units/hardhat/flags.js
+++ b/test/units/hardhat/flags.js
@@ -172,5 +172,24 @@ describe('Hardhat Plugin: command line options', function() {
verify.lineCoverage(expected);
shell.rm('.solcover.js');
});
+
+ it('--matrix', async function(){
+ const taskArgs = {
+ matrix: true
+ }
+
+ mock.installFullProject('matrix');
+ mock.hardhatSetupEnv(this);
+
+ await this.env.run("coverage", taskArgs);
+
+ // Integration test checks output path configurabililty
+ const altPath = path.join(process.cwd(), './alternateTestMatrix.json');
+ const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
+ const producedMatrix = require(altPath)
+ const expectedMatrix = require(expPath);
+
+ assert.deepEqual(producedMatrix, expectedMatrix);
+ });
});
diff --git a/test/units/truffle/flags.js b/test/units/truffle/flags.js
index 2365041..e073fc3 100644
--- a/test/units/truffle/flags.js
+++ b/test/units/truffle/flags.js
@@ -257,5 +257,23 @@ describe('Truffle Plugin: command line options', function() {
`Should have used default coverage port 8545: ${mock.loggerOutput.val}`
);
});
+
+ it('--matrix', async function(){
+ process.env.TRUFFLE_TEST = true; // Path to reporter differs btw HH and Truffle
+ truffleConfig.matrix = true;
+
+ mock.installFullProject('matrix');
+ await plugin(truffleConfig);
+
+ // Integration test checks output path configurabililty
+ const altPath = path.join(process.cwd(), mock.pathToTemp('./alternateTestMatrix.json'));
+ const expPath = path.join(process.cwd(), mock.pathToTemp('./expectedTestMatrixHardhat.json'));
+
+ const producedMatrix = require(altPath)
+ const expectedMatrix = require(expPath);
+
+ assert.deepEqual(producedMatrix, expectedMatrix);
+ process.env.TRUFFLE_TEST = false;
+ });
});