Add test matrix generator (#593)
cgewecke
4 years ago
parent
e76a165b63
commit
afd7ef0376
@ -0,0 +1,421 @@ |
||||
### Test Matrix Example |
||||
|
||||
An example of output written to the file `./testMatrix.json` when coverage |
||||
is run with the `--matrix` cli flag. (Source project: [sc-forks/hardhat-e2e][1]) |
||||
|
||||
[1]: https://github.com/sc-forks/hardhat-e2e |
||||
|
||||
|
||||
```js |
||||
// Paths are relative to the project root directory |
||||
{ |
||||
// Solidity file name |
||||
"contracts/EtherRouter/EtherRouter.sol": { |
||||
|
||||
// Line number |
||||
"23": [ |
||||
{ |
||||
// Grep-able mocha test title |
||||
"title": "Resolves methods routed through an EtherRouter proxy", |
||||
|
||||
// Selectable mocha test file |
||||
"file": "test/etherrouter.js" |
||||
} |
||||
], |
||||
"42": [ |
||||
{ |
||||
"title": "Resolves methods routed through an EtherRouter proxy", |
||||
"file": "test/etherrouter.js" |
||||
} |
||||
], |
||||
"45": [ |
||||
{ |
||||
"title": "Resolves methods routed through an EtherRouter proxy", |
||||
"file": "test/etherrouter.js" |
||||
} |
||||
], |
||||
"61": [ |
||||
{ |
||||
"title": "Resolves methods routed through an EtherRouter proxy", |
||||
"file": "test/etherrouter.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/EtherRouter/Factory.sol": { |
||||
"19": [ |
||||
{ |
||||
"title": "Resolves methods routed through an EtherRouter proxy", |
||||
"file": "test/etherrouter.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/EtherRouter/Resolver.sol": { |
||||
"22": [ |
||||
{ |
||||
"title": "Resolves methods routed through an EtherRouter proxy", |
||||
"file": "test/etherrouter.js" |
||||
} |
||||
], |
||||
"26": [ |
||||
{ |
||||
"title": "Resolves methods routed through an EtherRouter proxy", |
||||
"file": "test/etherrouter.js" |
||||
} |
||||
], |
||||
"30": [ |
||||
{ |
||||
"title": "Resolves methods routed through an EtherRouter proxy", |
||||
"file": "test/etherrouter.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/MetaCoin.sol": { |
||||
"16": [ |
||||
{ |
||||
"title": "should put 10000 MetaCoin in the first account", |
||||
"file": "test/metacoin.js" |
||||
}, |
||||
{ |
||||
"title": "should call a function that depends on a linked library", |
||||
"file": "test/metacoin.js" |
||||
}, |
||||
{ |
||||
"title": "should send coin correctly", |
||||
"file": "test/metacoin.js" |
||||
}, |
||||
{ |
||||
"title": "a and b", |
||||
"file": "test/multicontract.js" |
||||
} |
||||
], |
||||
"20": [ |
||||
{ |
||||
"title": "should send coin correctly", |
||||
"file": "test/metacoin.js" |
||||
} |
||||
], |
||||
"21": [ |
||||
{ |
||||
"title": "should send coin correctly", |
||||
"file": "test/metacoin.js" |
||||
} |
||||
], |
||||
"22": [ |
||||
{ |
||||
"title": "should send coin correctly", |
||||
"file": "test/metacoin.js" |
||||
} |
||||
], |
||||
"23": [ |
||||
{ |
||||
"title": "should send coin correctly", |
||||
"file": "test/metacoin.js" |
||||
} |
||||
], |
||||
"24": [ |
||||
{ |
||||
"title": "should send coin correctly", |
||||
"file": "test/metacoin.js" |
||||
} |
||||
], |
||||
"28": [ |
||||
{ |
||||
"title": "should call a function that depends on a linked library", |
||||
"file": "test/metacoin.js" |
||||
} |
||||
], |
||||
"32": [ |
||||
{ |
||||
"title": "should put 10000 MetaCoin in the first account", |
||||
"file": "test/metacoin.js" |
||||
}, |
||||
{ |
||||
"title": "should call a function that depends on a linked library", |
||||
"file": "test/metacoin.js" |
||||
}, |
||||
{ |
||||
"title": "should send coin correctly", |
||||
"file": "test/metacoin.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/ConvertLib.sol": { |
||||
"6": [ |
||||
{ |
||||
"title": "should call a function that depends on a linked library", |
||||
"file": "test/metacoin.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/MultiContractFile.sol": { |
||||
"7": [ |
||||
{ |
||||
"title": "a and b", |
||||
"file": "test/multicontract.js" |
||||
}, |
||||
{ |
||||
"title": "methods that call methods in other contracts", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"15": [ |
||||
{ |
||||
"title": "a and b", |
||||
"file": "test/multicontract.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/VariableConstructor.sol": { |
||||
"8": [ |
||||
{ |
||||
"title": "should should initialize with a short string", |
||||
"file": "test/variableconstructor.js" |
||||
}, |
||||
{ |
||||
"title": "should should initialize with a medium length string", |
||||
"file": "test/variableconstructor.js" |
||||
}, |
||||
{ |
||||
"title": "should should initialize with a long string", |
||||
"file": "test/variableconstructor.js" |
||||
}, |
||||
{ |
||||
"title": "should should initialize with a random length string", |
||||
"file": "test/variableconstructor.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/VariableCosts.sol": { |
||||
"13": [ |
||||
{ |
||||
"title": "should should initialize with a short string", |
||||
"file": "test/variableconstructor.js" |
||||
}, |
||||
{ |
||||
"title": "should should initialize with a medium length string", |
||||
"file": "test/variableconstructor.js" |
||||
}, |
||||
{ |
||||
"title": "should should initialize with a long string", |
||||
"file": "test/variableconstructor.js" |
||||
}, |
||||
{ |
||||
"title": "should should initialize with a random length string", |
||||
"file": "test/variableconstructor.js" |
||||
}, |
||||
{ |
||||
"title": "should add one", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add three", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add even 5!", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should delete one", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should delete three", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should delete five", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add five and delete one", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should set a random length string", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "methods that do not throw", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "methods that throw", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "methods that call methods in other contracts", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should allow contracts to have identically named methods", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"29": [ |
||||
{ |
||||
"title": "should add one", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add three", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add even 5!", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add five and delete one", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"30": [ |
||||
{ |
||||
"title": "should add one", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add three", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add even 5!", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add five and delete one", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"34": [ |
||||
{ |
||||
"title": "should delete one", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should delete three", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should delete five", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add five and delete one", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"35": [ |
||||
{ |
||||
"title": "should delete one", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should delete three", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should delete five", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should add five and delete one", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"43": [ |
||||
{ |
||||
"title": "should set a random length string", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"47": [ |
||||
{ |
||||
"title": "methods that do not throw", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "methods that throw", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"48": [ |
||||
{ |
||||
"title": "methods that do not throw", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"52": [ |
||||
{ |
||||
"title": "methods that call methods in other contracts", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"53": [ |
||||
{ |
||||
"title": "methods that call methods in other contracts", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
], |
||||
"54": [ |
||||
{ |
||||
"title": "methods that call methods in other contracts", |
||||
"file": "test/variablecosts.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/Wallets/Wallet.sol": { |
||||
"8": [ |
||||
{ |
||||
"title": "should allow contracts to have identically named methods", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should should allow transfers and sends", |
||||
"file": "test/wallet.js" |
||||
} |
||||
], |
||||
"12": [ |
||||
{ |
||||
"title": "should allow contracts to have identically named methods", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should should allow transfers and sends", |
||||
"file": "test/wallet.js" |
||||
} |
||||
], |
||||
"17": [ |
||||
{ |
||||
"title": "should allow contracts to have identically named methods", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should should allow transfers and sends", |
||||
"file": "test/wallet.js" |
||||
} |
||||
], |
||||
"22": [ |
||||
{ |
||||
"title": "should allow contracts to have identically named methods", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should should allow transfers and sends", |
||||
"file": "test/wallet.js" |
||||
} |
||||
], |
||||
"23": [ |
||||
{ |
||||
"title": "should allow contracts to have identically named methods", |
||||
"file": "test/variablecosts.js" |
||||
}, |
||||
{ |
||||
"title": "should should allow transfers and sends", |
||||
"file": "test/wallet.js" |
||||
} |
||||
] |
||||
} |
||||
} |
||||
``` |
||||
@ -0,0 +1,71 @@ |
||||
const mocha = require("mocha"); |
||||
const inherits = require("util").inherits; |
||||
const Spec = mocha.reporters.Spec; |
||||
|
||||
|
||||
/** |
||||
* This file adapted from mocha's stats-collector |
||||
* https://github.com/mochajs/mocha/blob/54475eb4ca35a2c9044a1b8c59a60f09c73e6c01/lib/stats-collector.js#L1-L83
|
||||
*/ |
||||
const Date = global.Date; |
||||
|
||||
/** |
||||
* Provides stats such as test duration, number of tests passed / failed etc., by |
||||
* listening for events emitted by `runner`. |
||||
*/ |
||||
function mochaStats(runner) { |
||||
const stats = { |
||||
suites: 0, |
||||
tests: 0, |
||||
passes: 0, |
||||
pending: 0, |
||||
failures: 0 |
||||
}; |
||||
|
||||
if (!runner) throw new Error("Missing runner argument"); |
||||
|
||||
runner.stats = stats; |
||||
|
||||
runner.on("pass", () => stats.passes++); |
||||
runner.on("fail", () => stats.failures++); |
||||
runner.on("pending", () => stats.pending++); |
||||
runner.on("test end", () => stats.tests++); |
||||
|
||||
runner.once("start", () => (stats.start = new Date())); |
||||
|
||||
runner.once("end", function() { |
||||
stats.end = new Date(); |
||||
stats.duration = stats.end - stats.start; |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run |
||||
* and collects data about which tests hit which lines of code. |
||||
* This "test matrix" can be used as an input to |
||||
* |
||||
* |
||||
* @param {Object} runner mocha's runner |
||||
* @param {Object} options reporter.options (see README example usage) |
||||
*/ |
||||
function Matrix(runner, options) { |
||||
// Spec reporter
|
||||
Spec.call(this, runner, options); |
||||
|
||||
// Initialize stats for Mocha 6+ epilogue
|
||||
if (!runner.stats) { |
||||
mochaStats(runner); |
||||
this.stats = runner.stats; |
||||
} |
||||
|
||||
runner.on("test end", (info) => { |
||||
options.reporterOptions.collectTestMatrixData(info); |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Inherit from `Base.prototype`. |
||||
*/ |
||||
inherits(Matrix, Spec); |
||||
|
||||
module.exports = Matrix; |
||||
@ -0,0 +1,16 @@ |
||||
// Testing hooks
|
||||
const fn = (msg, config) => config.logger.log(msg); |
||||
const reporterPath = (process.env.TRUFFLE_TEST) |
||||
? "./plugins/resources/matrix.js" |
||||
: "../plugins/resources/matrix.js"; |
||||
|
||||
module.exports = { |
||||
// This is loaded directly from `./plugins` during unit tests. The default val is
|
||||
// "solidity-coverage/plugins/resources/matrix.js"
|
||||
matrixReporterPath: reporterPath, |
||||
matrixOutputPath: "alternateTestMatrix.json", |
||||
|
||||
skipFiles: ['Migrations.sol'], |
||||
silent: process.env.SILENT ? true : false, |
||||
istanbulReporter: ['json-summary', 'text'], |
||||
} |
||||
@ -0,0 +1,17 @@ |
||||
pragma solidity ^0.7.0; |
||||
|
||||
|
||||
contract MatrixA { |
||||
uint x; |
||||
constructor() public { |
||||
} |
||||
|
||||
function sendFn() public { |
||||
x = 5; |
||||
} |
||||
|
||||
function callFn() public pure returns (uint){ |
||||
uint y = 5; |
||||
return y; |
||||
} |
||||
} |
||||
@ -0,0 +1,17 @@ |
||||
pragma solidity ^0.7.0; |
||||
|
||||
|
||||
contract MatrixB { |
||||
uint x; |
||||
constructor() public { |
||||
} |
||||
|
||||
function sendFn() public { |
||||
x = 5; |
||||
} |
||||
|
||||
function callFn() public pure returns (uint){ |
||||
uint y = 5; |
||||
return y; |
||||
} |
||||
} |
||||
@ -0,0 +1,46 @@ |
||||
{ |
||||
"contracts/MatrixA.sol": { |
||||
"10": [ |
||||
{ |
||||
"title": "sends to A", |
||||
"file": "test/matrix_a_b.js" |
||||
}, |
||||
{ |
||||
"title": "sends", |
||||
"file": "test/matrix_a.js" |
||||
} |
||||
], |
||||
"14": [ |
||||
{ |
||||
"title": "calls", |
||||
"file": "test/matrix_a.js" |
||||
} |
||||
], |
||||
"15": [ |
||||
{ |
||||
"title": "calls", |
||||
"file": "test/matrix_a.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/MatrixB.sol": { |
||||
"10": [ |
||||
{ |
||||
"title": "sends to B", |
||||
"file": "test/matrix_a_b.js" |
||||
} |
||||
], |
||||
"14": [ |
||||
{ |
||||
"title": "calls B", |
||||
"file": "test/matrix_a_b.js" |
||||
} |
||||
], |
||||
"15": [ |
||||
{ |
||||
"title": "calls B", |
||||
"file": "test/matrix_a_b.js" |
||||
} |
||||
] |
||||
} |
||||
} |
||||
@ -0,0 +1,46 @@ |
||||
{ |
||||
"contracts/MatrixA.sol": { |
||||
"10": [ |
||||
{ |
||||
"title": "sends", |
||||
"file": "test/matrix_a.js" |
||||
}, |
||||
{ |
||||
"title": "sends to A", |
||||
"file": "test/matrix_a_b.js" |
||||
} |
||||
], |
||||
"14": [ |
||||
{ |
||||
"title": "calls", |
||||
"file": "test/matrix_a.js" |
||||
} |
||||
], |
||||
"15": [ |
||||
{ |
||||
"title": "calls", |
||||
"file": "test/matrix_a.js" |
||||
} |
||||
] |
||||
}, |
||||
"contracts/MatrixB.sol": { |
||||
"10": [ |
||||
{ |
||||
"title": "sends to B", |
||||
"file": "test/matrix_a_b.js" |
||||
} |
||||
], |
||||
"14": [ |
||||
{ |
||||
"title": "calls B", |
||||
"file": "test/matrix_a_b.js" |
||||
} |
||||
], |
||||
"15": [ |
||||
{ |
||||
"title": "calls B", |
||||
"file": "test/matrix_a_b.js" |
||||
} |
||||
] |
||||
} |
||||
} |
||||
@ -0,0 +1,9 @@ |
||||
require("@nomiclabs/hardhat-truffle5"); |
||||
require(__dirname + "/../plugins/nomiclabs.plugin"); |
||||
|
||||
module.exports={ |
||||
solidity: { |
||||
version: "0.7.3" |
||||
}, |
||||
logger: process.env.SILENT ? { log: () => {} } : console, |
||||
}; |
||||
@ -0,0 +1,15 @@ |
||||
const MatrixA = artifacts.require("MatrixA"); |
||||
|
||||
contract("MatrixA", function(accounts) { |
||||
let instance; |
||||
|
||||
before(async () => instance = await MatrixA.new()) |
||||
|
||||
it('sends', async function(){ |
||||
await instance.sendFn(); |
||||
}); |
||||
|
||||
it('calls', async function(){ |
||||
await instance.callFn(); |
||||
}) |
||||
}); |
||||
@ -0,0 +1,30 @@ |
||||
const MatrixA = artifacts.require("MatrixA"); |
||||
const MatrixB = artifacts.require("MatrixB"); |
||||
|
||||
contract("Matrix A and B", function(accounts) { |
||||
let instanceA; |
||||
let instanceB; |
||||
|
||||
before(async () => { |
||||
instanceA = await MatrixA.new(); |
||||
instanceB = await MatrixB.new(); |
||||
}) |
||||
|
||||
it('sends to A', async function(){ |
||||
await instanceA.sendFn(); |
||||
}); |
||||
|
||||
// Duplicate test title and file should *not* be duplicated in the output
|
||||
it('sends to A', async function(){ |
||||
await instanceA.sendFn(); |
||||
}) |
||||
|
||||
it('calls B', async function(){ |
||||
await instanceB.callFn(); |
||||
}) |
||||
|
||||
it('sends to B', async function(){ |
||||
await instanceB.sendFn(); |
||||
}); |
||||
|
||||
}); |
||||
@ -0,0 +1,10 @@ |
||||
module.exports = { |
||||
networks: {}, |
||||
mocha: {}, |
||||
compilers: { |
||||
solc: { |
||||
version: "0.7.3" |
||||
} |
||||
}, |
||||
logger: process.env.SILENT ? { log: () => {} } : console, |
||||
} |
||||
Loading…
Reference in new issue