TheDude
/
solidity-coverage
mirror of https://github.com/sc-forks/solidity-coverage
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

exec.js test suite updated for truffle 3

Browse Source
pull/1/head
cgewecke 8 years ago
parent b1fa110d89
commit f1de0f42df
17 changed files with 424 additions and 69 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 17
      exec.js
  2. 61
      runCoveredTests.js
  3. 168
      test/run.js
  4. 6
      test/run/block-gas-limit.js
  5. 6
      test/run/empty.js
  6. 10
      test/run/only-call.js
  7. 14
      test/run/simple.js
  8. 16
      test/run/sol-parse-fail.js
  9. 10
      test/run/truffle-crash.js
  10. 15
      test/run/truffle-test-fail.js
  11. 4
      test/sources/run/Empty.sol
  12. 15
      test/sources/run/Expensive.sol
  13. 20
      test/sources/run/Migrations.sol
  14. 11
      test/sources/run/OnlyCall.sol
  15. 13
      test/sources/run/Simple.sol
  16. 14
      test/sources/run/SimpleError.sol
  17. 93
      test/util/mockTruffle.js

17
exec.js
Unescape View File

@ -1,5 +1,3 @@
const shell = require('shelljs');
const fs = require('fs');
const path = require('path');
@ -9,21 +7,24 @@ const SolidityCoder = require('web3/lib/solidity/coder.js');
const getInstrumentedVersion = require('./instrumentSolidity.js');
const CoverageMap = require('./coverageMap.js');
const gasLimit = '0xfffffffffff';
const coverage = new CoverageMap();
const coverageDir = './coverageEnv';
//const _MODULES_ = 'node_modules/solcover/node_modules';
const _MODULES_ = 'node_modules';
let log = () => {};
let workingDir = './..';
const gasLimit = '0xfffffffffff';
let port = 8555;
let testrpcProcess = null;
let events = null;
let silence = '';
// --------------------------------------- Script --------------------------------------------------
if (argv.dir) workingDir = argv.dir; // Working dir relative to solcover
if (argv.port) port = argv.port; // Testrpc port
if (argv.silent) { // Log level
if (argv.silent) { // Log level
silence = '> /dev/null 2>&1';
} else {
log = console.log;
@ -33,13 +34,13 @@ if (argv.silent) { // Log level
// on (hopefully) unused port. Changes here should be also be added to the before() block
// of test/run.js
if (!argv.norpc) {
if (!shell.test('-e', './node_modules/ethereumjs-vm/lib/opFns.js.orig')) {
if (!shell.test('-e', `./${_MODULES_}/ethereumjs-vm/lib/opFns.js.orig`)) {
log('Patch local testrpc...');
shell.exec('patch -b ./node_modules/ethereumjs-vm/lib/opFns.js ./hookIntoEvents.patch');
shell.exec(`patch -b ./${_MODULES_}/ethereumjs-vm/lib/opFns.js ./hookIntoEvents.patch`);
}
log(`Launching testrpc on port ${port}`);
try {
const command = `./node_modules/ethereumjs-testrpc/bin/testrpc --gasLimit ${gasLimit} --port ${port}`;
const command = `./${_MODULES_}/ethereumjs-testrpc/bin/testrpc --gasLimit ${gasLimit} --port ${port}`;
testrpcProcess = childprocess.exec(command);
} catch (err) {
const msg = `There was a problem launching testrpc: ${err}`;
@ -109,7 +110,7 @@ try {
try {
coverage.generate(events, `${coverageDir}/contracts/`);
fs.writeFileSync('./coverage.json', JSON.stringify(coverage.coverage));
shell.exec(`./node_modules/istanbul/lib/cli.js report lcov ${silence}`);
shell.exec(`./${_MODULES_}/istanbul/lib/cli.js report lcov ${silence}`);
} catch (err) {
const msg = 'There was a problem generating producing the coverage map / running Istanbul.\n';
cleanUp(msg + err);

61
runCoveredTests.js
Unescape View File

@ -1,61 +0,0 @@
const Web3 = require('web3');
const shell = require('shelljs');
const SolidityCoder = require('web3/lib/solidity/coder.js');
const getInstrumentedVersion = require('./instrumentSolidity.js');
const fs = require('fs');
const path = require('path');
const CoverageMap = require('./coverageMap.js');
const mkdirp = require('mkdirp');
const childprocess = require('child_process');
const coverage = new CoverageMap();
const web3 = new Web3(new Web3.providers.HttpProvider('http://localhost:8545'));
// Patch our local testrpc if necessary
if (!shell.test('-e', './node_modules/ethereumjs-vm/lib/opFns.js.orig')) {
console.log('patch local testrpc...');
shell.exec('patch -b ./node_modules/ethereumjs-vm/lib/opFns.js ./hookIntoEvents.patch');
}
// Run the modified testrpc with large block limit
const testrpcProcess = childprocess.exec('./node_modules/ethereumjs-testrpc/bin/testrpc --gasLimit 0xfffffffffffff --gasPrice 0x1');
if (shell.test('-d', '../originalContracts')) {
console.log('There is already an "originalContracts" directory in your truffle directory.\n' +
'This is probably due to a previous solcover failure.\n' +
'Please make sure the ./contracts/ directory contains your contracts (perhaps by copying them from originalContracts), ' +
'and then delete the originalContracts directory.');
process.exit(1);
}
shell.mv('./../contracts/', './../originalContracts/');
shell.mkdir('./../contracts/');
// For each contract in originalContracts, get the instrumented version
shell.ls('./../originalContracts/**/*.sol').forEach(file => {
if (file !== 'originalContracts/Migrations.sol') {
const canonicalContractPath = path.resolve(file);
console.log('instrumenting ', canonicalContractPath);
const contract = fs.readFileSync(canonicalContractPath).toString();
const instrumentedContractInfo = getInstrumentedVersion(contract, canonicalContractPath);
mkdirp.sync(path.dirname(canonicalContractPath.replace('originalContracts', 'contracts')));
fs.writeFileSync(canonicalContractPath.replace('originalContracts', 'contracts'), instrumentedContractInfo.contract);
coverage.addContract(instrumentedContractInfo, canonicalContractPath);
}
});
shell.cp('./../originalContracts/Migrations.sol', './../contracts/Migrations.sol');
shell.rm('./allFiredEvents'); // Delete previous results
shell.exec('truffle test --network test');
const events = fs.readFileSync('./allFiredEvents').toString().split('\n');
events.pop();
// The pop here isn't a bug - there is an empty line at the end of this file, so we
// don't want to include it as an event.
coverage.generate(events, './../originalContracts/');
fs.writeFileSync('./coverage.json', JSON.stringify(coverage.coverage));
shell.exec('./node_modules/istanbul/lib/cli.js report lcov');
testrpcProcess.kill();
shell.rm('-rf', './../contracts');
shell.mv('./../originalContracts', './../contracts');

168
test/run.js
Unescape View File

@ -0,0 +1,168 @@
const assert = require('assert');
const shell = require('shelljs');
const fs = require('fs');
const childprocess = require('child_process');
const mock = require('./util/mockTruffle.js');
let launchTestRpc = false;
// shell.test alias for legibility
function pathExists(path) { return shell.test('-e', path); }
// tests run out of memory in CI without this
function collectGarbage() {
if (global.gc) { global.gc(); }
}
describe('run', () => {
let port = 8555;
let testrpcProcess = null;
let script = `node ./exec.js --dir "./mock" --port ${port} --silent`; //
before(() => {
mock.protectCoverage();
});
beforeEach(() => {
// CI (Ubuntu) doesn't seem to be freeing server resources until the parent process of these
// tests exits so there are errors launching testrpc repeatedly on the same port. Tests #2 through
// #last will use this instance of testrpc (port 8557). Test #1 uses the instance launched by
// the run script (which also installs the patch). This allows us to end run CI container issues
// AND verify that the script actually works.
if (launchTestRpc) {
port = 8557;
script = `node ./exec.js --dir "./mock" --port ${port} --norpc --silent`; //
const command = `./node_modules/ethereumjs-testrpc/bin/testrpc --gasLimit 0xfffffffffff --port ${port}`;
testrpcProcess = childprocess.exec(command);
launchTestRpc = false;
}
});
after(() => {
mock.restoreCoverage();
testrpcProcess.kill();
});
afterEach(() => {
mock.remove();
});
// This pre-test flushes the suite. There's some kind of sequencing issue here in development
// - the first test always fails unless there is a fresh testrpc install.
// Should not be a problem on CI but results in false positives @home.
it('flush tests', () => {
mock.install('Simple.sol', 'simple.js');
shell.exec(script);
collectGarbage();
});
// This test should be positioned first in the suite because of the way testrpc is
// launched for these tests.
it('simple contract: should generate coverage, cleanup & exit(0)', () => {
// Directory should be clean
assert(pathExists('./coverage') === false, 'should start without: coverage');
assert(pathExists('./coverage.json') === false, 'should start without: coverage.json');
// Run script (exits 0);
mock.install('Simple.sol', 'simple.js');
shell.exec(script);
assert(shell.error() === null, 'script should not error');
// Directory should have coverage report
assert(pathExists('./coverage') === true, 'script should gen coverage folder');
assert(pathExists('./coverage.json') === true, 'script should gen coverage.json');
// Coverage should be real.
// This test is tightly bound to the function names in Simple.sol
const produced = JSON.parse(fs.readFileSync('./coverage.json', 'utf8'));
const path = Object.keys(produced)[0];
assert(produced[path].fnMap['1'].name === 'test', 'coverage.json should map "test"');
assert(produced[path].fnMap['2'].name === 'getX', 'coverage.json should map "getX"');
collectGarbage();
launchTestRpc = true; // Toggle flag to launch a single testrpc instance for rest of tests.
});
it('contract only uses .call: should generate coverage, cleanup & exit(0)', () => {
// Run against contract that only uses method.call.
assert(pathExists('./coverage') === false, 'should start without: coverage');
assert(pathExists('./coverage.json') === false, 'should start without: coverage.json');
mock.install('OnlyCall.sol', 'only-call.js');
shell.exec(script);
assert(shell.error() === null, 'script should not error');
assert(pathExists('./coverage') === true, 'script should gen coverage folder');
assert(pathExists('./coverage.json') === true, 'script should gen coverage.json');
const produced = JSON.parse(fs.readFileSync('./coverage.json', 'utf8'));
const path = Object.keys(produced)[0];
assert(produced[path].fnMap['1'].name === 'getFive', 'coverage.json should map "getFive"');
collectGarbage();
});
it('truffle tests failing: should generate coverage, cleanup & exit(0)', () => {
assert(pathExists('./coverage') === false, 'should start without: coverage');
assert(pathExists('./coverage.json') === false, 'should start without: coverage.json');
// Run with Simple.sol and a failing assertion in a truffle test
mock.install('Simple.sol', 'truffle-test-fail.js');
shell.exec(script);
assert(shell.error() === null, 'script should not error');
assert(pathExists('./coverage') === true, 'script should gen coverage folder');
assert(pathExists('./coverage.json') === true, 'script should gen coverage.json');
const produced = JSON.parse(fs.readFileSync('./coverage.json', 'utf8'));
const path = Object.keys(produced)[0];
assert(produced[path].fnMap['1'].name === 'test', 'coverage.json should map "test"');
assert(produced[path].fnMap['2'].name === 'getX', 'coverage.json should map "getX"');
collectGarbage();
});
it('deployment cost > block gasLimit: should generate coverage, cleanup & exit(0)', () => {
// Just making sure Expensive.sol compiles and deploys here.
mock.install('Expensive.sol', 'block-gas-limit.js');
shell.exec(script);
assert(shell.error() === null, 'script should not error');
collectGarbage();
});
it('truffle crashes: should generate NO coverage, cleanup and exit(1)', () => {
assert(pathExists('./coverage') === false, 'should start without: coverage');
assert(pathExists('./coverage.json') === false, 'should start without: coverage.json');
// Run with Simple.sol and a syntax error in the truffle test
mock.install('Simple.sol', 'truffle-crash.js');
shell.exec(script);
assert(shell.error() !== null, 'script should error');
assert(pathExists('./coverage') !== true, 'script should NOT gen coverage folder');
assert(pathExists('./coverage.json') !== true, 'script should NOT gen coverage.json');
collectGarbage();
});
it('instrumentation errors: should generate NO coverage, cleanup and exit(1)', () => {
assert(pathExists('./coverage') === false, 'should start without: coverage');
assert(pathExists('./coverage.json') === false, 'should start without: coverage.json');
// Run with SimpleError.sol (has syntax error) and working truffle test
mock.install('SimpleError.sol', 'simple.js');
shell.exec(script);
assert(shell.error() !== null, 'script should error');
assert(pathExists('./coverage') !== true, 'script should NOT gen coverage folder');
assert(pathExists('./coverage.json') !== true, 'script should NOT gen coverage.json');
collectGarbage();
});
it('no events log produced: should generate NO coverage, cleanup and exit(1)', () => {
// Run contract and test that pass but fire no events
assert(pathExists('./coverage') === false, 'should start without: coverage');
assert(pathExists('./coverage.json') === false, 'should start without: coverage.json');
mock.install('Empty.sol', 'empty.js');
shell.exec(script);
assert(shell.error() !== null, 'script should error');
assert(pathExists('./coverage') !== true, 'script should NOT gen coverage folder');
assert(pathExists('./coverage.json') !== true, 'script should NOT gen coverage.json');
collectGarbage();
});
});

6
test/run/block-gas-limit.js
Unescape View File

@ -0,0 +1,6 @@
const Expensive = artifacts.require('./Expensive.sol');
contract('Expensive', accounts => {
it('should deploy', () => Expensive.deployed());
});

6
test/run/empty.js
Unescape View File

@ -0,0 +1,6 @@
const Empty = artifacts.require('./Empty.sol');
contract('Empty', accounts => {
it('should deploy', () => Empty.deployed());
});

10
test/run/only-call.js
Unescape View File

@ -0,0 +1,10 @@
const OnlyCall = artifacts.require('./OnlyCall.sol');
contract('OnlyCall', accounts => {
it('should return 5', () =>
OnlyCall.deployed()
.then(instance => instance.getFive.call())
.then(val => assert.equal(val, 5))
);
});

14
test/run/simple.js
Unescape View File

@ -0,0 +1,14 @@
const Simple = artifacts.require('./Simple.sol');
contract('Simple', accounts => {
it('should set x to 5', () => {
let simple;
return Simple.deployed().then(instance => {
simple = instance;
return simple.test(5);
})
.then(val => simple.getX.call())
.then(val => assert.equal(val.toNumber(), 5));
});
});

16
test/run/sol-parse-fail.js
Unescape View File

@ -0,0 +1,16 @@
const Simple = artifacts.require('./Simple.sol');
// This test is constructed correctly but the SimpleError.sol has a syntax error
contract('SimpleError', accounts => {
it('should set x to 5', () => {
let simple;
return Simple.deployed().then(instance => {
simple = instance;
return simple.test(5);
})
.then(val => simple.getX.call())
.then(val => assert.equal(val, 5));
});
});

10
test/run/truffle-crash.js
Unescape View File

@ -0,0 +1,10 @@
'use strict'
var Simple = artifacts.require('./Simple.sol');
// This test should break truffle because it has a syntax error.
contract('Simple', function(accounts){
it('should crash', function(){
return Simple.deployed().then.why.
})
})

15
test/run/truffle-test-fail.js
Unescape View File

@ -0,0 +1,15 @@
const Simple = artifacts.require('./Simple.sol');
contract('Simple', accounts => {
it('should set x to 5', () => {
let simple;
return Simple.deployed().then(instance => {
simple = instance;
return simple.test(5);
})
.then(val => simple.getX.call())
.then(val => assert.equal(val.toNumber(), 4)); // <-- Wrong result: test fails
});
});

4
test/sources/run/Empty.sol
Unescape View File

@ -0,0 +1,4 @@
pragma solidity ^0.4.3;
contract Empty {
}

15
test/sources/run/Expensive.sol
Unescape View File

@ -0,0 +1,15 @@
// Cost to deploy Expensive: 0x4e042f
// Block gas limit is: 0x47e7c4
// Should throw out of gas on unmodified truffle
// Should pass solcover truffle
pragma solidity ^0.4.2;
contract Expensive {
mapping (uint => address) map;
function Expensive(){
for(uint i = 0; i < 250; i++ ){
map[i] = address(this);
}
}
}

20
test/sources/run/Migrations.sol
Unescape View File

@ -0,0 +1,20 @@
pragma solidity ^0.4.4;
contract Migrations {
address public owner;
uint public last_completed_migration;
modifier restricted() {if (msg.sender == owner) _;}
function Migrations() {
owner = msg.sender;
}
function setCompleted(uint completed) restricted {
last_completed_migration = completed;
}
function upgrade(address new_address) restricted {
Migrations upgraded = Migrations(new_address);
upgraded.setCompleted(last_completed_migration);
}
}

11
test/sources/run/OnlyCall.sol
Unescape View File

@ -0,0 +1,11 @@
/**
* This contract contains a single function that is accessed using method.call
* With an unpatched testrpc it should not generate any events.
*/
pragma solidity ^0.4.3;
contract OnlyCall {
function getFive() returns (uint){
return 5;
}
}

13
test/sources/run/Simple.sol
Unescape View File

@ -0,0 +1,13 @@
pragma solidity ^0.4.3;
contract Simple {
uint x = 0;
function test(uint val) {
x = x + val;
}
function getX() returns (uint){
return x;
}
}

14
test/sources/run/SimpleError.sol
Unescape View File

@ -0,0 +1,14 @@
// This contract should throw a parse error in instrumentSolidity.js
pragma solidity ^0.4.3;
contract SimpleError {
uint x = 0;
function test(uint val) {
x = x + val // <-- no semi-colon
}
function getX() returns (uint){
return x;
}
}

93
test/util/mockTruffle.js
Unescape View File

@ -0,0 +1,93 @@
/*
This file contains utilities for generating a mock truffle project to test solcover's
run script against.
*/
const assert = require('assert');
const fs = require('fs');
const shell = require('shelljs');
/**
* Moves existing coverage reports into a safe place while testing run script which
* would overwrite them. Silences shell complaints about non-existent files.
*/
module.exports.protectCoverage = function () {
shell.config.silent = true;
shell.rm('-Rf', './safe');
shell.mkdir('./safe');
shell.mv('./coverage', './safe/coverage');
shell.mv('./coverage.json', './safe/coverage.json');
shell.config.silent = false;
};
/**
* Restores pre-existing coverage reports after testing run script.
* Silences shell complaints about non-existent files.
*/
module.exports.restoreCoverage = function () {
shell.config.silent = true;
shell.mv('./safe/coverage', './coverage');
shell.mv('./safe/coverage.json', './coverage.json');
shell.rm('-Rf', './safe');
shell.config.silent = false;
};
/**
* Installs mock truffle project at ./mock with a single contract
* and test specified by the params.
* @param {String} contract <contractName.sol> located in /test/sources/run/
* @param {[type]} test <testName.js> located in /test/run/
*/
module.exports.install = function (contract, test) {
shell.mkdir('./mock');
shell.mkdir('./mock/contracts');
shell.mkdir('./mock/migrations');
shell.mkdir('./mock/test');
// Mock contracts
shell.cp(`./test/sources/run/${contract}`, `./mock/contracts/${contract}`);
shell.cp('./test/sources/run/Migrations.sol', './mock/contracts/Migrations.sol');
// Mock migrations
const initialMigration = `
let Migrations = artifacts.require('Migrations.sol');
module.exports = function(deployer) {
deployer.deploy(Migrations);
};`;
const contractLocation = './' + contract;
const deployContracts = `
var contract = artifacts.require('${contractLocation}');
module.exports = function(deployer) {
deployer.deploy(contract);
};`;
fs.writeFileSync('./mock/migrations/1_initial_migration.js', initialMigration);
fs.writeFileSync('./mock/migrations/2_deploy_contracts.js', deployContracts);
// Mock test
shell.cp(`./test/run/${test}`, `./mock/test/${test}`);
// Mock truffle.js
const trufflejs = `module.exports = {
networks: {
development: {
host: "localhost",
port: 8545,
network_id: "*"
}}};`
;
fs.writeFileSync('./mock/truffle.js', trufflejs);
};
/**
* Removes mock truffle project and coverage reports generated by runCovered tests
*/
module.exports.remove = function () {
shell.config.silent = true;
shell.rm('-Rf', 'mock');
shell.rm('-Rf', 'coverage');
shell.rm('coverage.json');
shell.config.silent = false;
};
Write Preview
Loading…
Cancel
Save