Prepare API for parallelization (#436)

README.md

@@ -68,6 +68,7 @@ module.exports = {
 | client | *Object* | `require("ganache-core")` | Useful if you need a specific ganache version. |
 | providerOptions | *Object* | `{ }` | [ganache-core options][1]  |
 | skipFiles | *Array* | `['Migrations.sol']` | Array of contracts or folders (with paths expressed relative to the `contracts` directory) that should be skipped when doing instrumentation. |
+| istanbulFolder | *String* | `./coverage` |  Folder location for Istanbul coverage reports. |
 | istanbulReporter | *Array* | `['html', 'lcov', 'text']` | [Istanbul coverage reporters][2]  |
 | mocha | *Object* | `{ }` | [Mocha options][3] to merge into existing mocha config. `grep` and `invert` are useful for skipping certain tests under coverage using tags in the test descriptions.|
 | onServerReady[<sup>*</sup>][14] | *Function* |   | Hook run *after* server is launched, *before* the tests execute. Useful if you need to use the Oraclize bridge or have setup scripts which rely on the server's availability. [More...][23] |

dist/plugin-assets/buidler.ui.js

@@ -1,7 +1,7 @@
 const UI = require('./../../lib/ui').UI;
 
 /**
- * Truffle Plugin logging
+ * Buidler Plugin logging
  */
 class PluginUI extends UI {
   constructor(log){

dist/plugin-assets/buidler.utils.js

@@ -7,7 +7,7 @@ const { createProvider } = require("@nomiclabs/buidler/internal/core/providers/c
 
 
 // =============================
-// Buidler Specific Plugin Utils
+// Buidler Plugin Utils
 // =============================
 
 /**

dist/plugin-assets/plugin.utils.js

@@ -1,9 +1,6 @@
 /**
  * A collection of utilities for common tasks plugins will need in the course
  * of composing a workflow using the solidity-coverage API
- *
- * TODO: Sweep back through here and make all `config.truffle_variable` plugin
- * platform neutral...
  */
 
 const PluginUI = require('./truffle.ui');

lib/api.js

@@ -6,6 +6,7 @@ const istanbul = require('istanbul');
 const util = require('util');
 const assert = require('assert');
 const detect = require('detect-port');
+const _ = require('lodash/lang');
 
 const ConfigValidator = require('./validator');
 const Instrumenter = require('./instrumenter');
@@ -17,7 +18,7 @@ const AppUI = require('./ui').AppUI;
  * Coverage Runner
  */
 class API {
-  constructor(config) {
+  constructor(config={}) {
     this.coverage = new Coverage();
     this.instrumenter = new Instrumenter();
     this.validator = new ConfigValidator()
@@ -54,7 +55,8 @@ class API {
     this.gasLimitString = "0xfffffffffff";   // block gas limit for ganache (higher than "gas sent")
     this.gasPrice = 0x01;
 
-    this.istanbulReporter = config.istanbulReporter || ['html', 'lcov', 'text'];
+    this.istanbulFolder = config.istanbulFolder || false;
+    this.istanbulReporter = config.istanbulReporter || ['html', 'lcov', 'text', 'json'];
 
     this.setLoggingLevel(config.silent);
     this.ui = new AppUI(this.log);
@@ -65,21 +67,12 @@ class API {
    * Instruments a set of sources to prepare them for running under coverage
    * @param  {Object[]}  targets (see below)
    * @return {Object[]}          (see below)
-   * @example:
+   * @example of input/output array:
-   *
+   * [{
-   *  targets:
+   *   source:         (required) <solidity-source>,
-   *  [{
+   *   canonicalPath:  (required) <absolute path to source file>
-   *    canonicalPath: <absolute-path>
+   *   relativePath:   (optional) <rel path to source file for logging>
-   *    relativePath: <relative-path>
+   * }]
-   *    source: <source-file>
-   *
-   *  },...]
-   *
-   *  outputs:
-   *  [{
-   *    canonicalPath: <path>
-   *    source: <instrumented-source-file>
-   *  }...]
    */
   instrument(targets=[]) {
     let currentFile;      // Keep track of filename in case we crash...
@@ -95,7 +88,7 @@ class API {
           this.ui.report('instr-start');
         }
 
-        this.ui.report('instr-item', [target.relativePath]);
+        this.ui.report('instr-item', [currentFile]);
 
         const instrumented = this.instrumenter.instrument(
           target.source,
@@ -119,22 +112,35 @@ class API {
     return outputs;
   }
 
+  /**
+   * Returns a copy of the hit map created during instrumentation.
+   * Useful if you'd like to delegate coverage collection to multiple processes.
+   * @return {Object} instrumentationData
+   */
+  getInstrumentationData(){
+    return _.cloneDeep(this.instrumenter.instrumentationData)
+  }
+
+  /**
+   * Sets the hit map object generated during instrumentation. Useful if you'd like
+   * to collect data for a pre-existing instrumentation.
+   * @param {Object} data
+   */
+  setInstrumentationData(data={}){
+    this.instrumenter.instrumentationData = _.cloneDeep(data);
+  }
+
   /**
    * Launches an in-process ethereum client server, hooking the DataCollector to its VM.
    * @param  {Object} client ganache client
    * @return {String}        address of server to connect to
    */
   async ganache(client){
-    let retry = false;
-    let address = `http://${this.host}:${this.port}`;
-
     // Check for port-in-use
     if (await detect(this.port) !== this.port){
       throw new Error(this.ui.generate('server-fail', [this.port]))
     }
 
-    if(!this.client) this.client = client; // Prefer client from options
-
     this.collector = new DataCollector(this.instrumenter.instrumentationData);
 
     this.providerOptions.gasLimit = this.gasLimitString;
@@ -143,16 +149,17 @@ class API {
     // Launch server and attach to vm step of supplied client
     try {
       if (this.config.forceBackupServer) throw new Error()
-      await this.attachToVM()
+      await this.attachToVM(client)
     }
 
-    // Fallback to ganache-core-sc (eq: ganache-core 2.7.0)
+    // Fallback to ganache-cli)
     catch(err) {
-      this.ui.report('vm-fail', []);
+      const _ganache = require('ganache-cli');
-      this.client = require('ganache-core-sc');
+      this.ui.report('vm-fail', [_ganache.version]);
-      await this.attachToVM();
+      await this.attachToVM(_ganache);
     }
 
+    const address = `http://${this.host}:${this.port}`;
     this.ui.report('server', [address]);
     return address;
   }
@@ -162,7 +169,7 @@ class API {
    */
   async report() {
     const collector = new istanbul.Collector();
-    const reporter = new istanbul.Reporter();
+    const reporter = new istanbul.Reporter(false, this.istanbulFolder);
 
     return new Promise((resolve, reject) => {
       try {
@@ -177,7 +184,8 @@ class API {
 
         // Pify doesn't like this one...
         reporter.write(collector, true, (err) => {
-          if (err) throw err;
+          if (err) return reject(err);
+
           this.ui.report('istanbul');
           resolve();
         });
@@ -204,9 +212,11 @@ class API {
   // ========
   // Provider
   // ========
-  async attachToVM(){
+  async attachToVM(client){
     const self = this;
 
+    // Prefer client from options
+    if(!this.client) this.client = client;
     this.server = this.client.server(this.providerOptions);
 
     this.assertHasBlockchain(this.server.provider);
@@ -225,7 +235,6 @@ class API {
       return vm;
     }
 
-    // NB: EADDRINUSE errors are uncatch-able?
     await pify(this.server.listen)(this.port);
   }
 

lib/ui.js

@@ -56,9 +56,9 @@ class AppUI extends UI {
     const w = ":warning:";
 
     const kinds = {
-      'vm-fail': `${w}  ${c.red('There was a problem attaching to the ganache-core VM.')} `+
+      'vm-fail': `${w}  ${c.red('There was a problem attaching to the ganache VM.')}\n` +
-                       `${c.red('Check the provider option syntax in solidity-coverage docs.')}\n`+
+                 `${w}  ${c.red('For help, see the "client" & "providerOptions" syntax in solidity-coverage docs.')}\n`+
-                 `${w}  ${c.red('Using ganache-core-sc (eq. core v2.7.0) instead.')}\n`,
+                 `${w}  ${c.red(`Using ganache-cli (v${args[0]}) instead.`)}\n`,
 
 
       'instr-start': `\n${c.bold('Instrumenting for coverage...')}` +

package.json

@@ -12,9 +12,9 @@
   },
   "scripts": {
     "nyc": "SILENT=true  nyc --exclude '**/sc_temp/**' --exclude '**/test/**'",
-    "test": "npm run nyc -- mocha test/units/* --timeout 100000 --no-warnings --exit",
+    "test": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc -- mocha test/units/* --timeout 100000 --no-warnings --exit",
-    "test:ci": "SILENT=true node --max-old-space-size=3072 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
+    "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit",
-    "test:debug": "mocha test/units/* --timeout 100000 --no-warnings --exit"
+    "test:debug": "node --max-old-space-size=4096 ./node_modules/.bin/mocha test/units/* --timeout 100000 --no-warnings --exit"
   },
   "homepage": "https://github.com/sc-forks/solidity-coverage",
   "repository": {
@@ -35,6 +35,7 @@
     "globby": "^10.0.1",
     "istanbul": "^0.4.5",
     "jsonschema": "^1.2.4",
+    "lodash": "^4.17.15",
     "node-dir": "^0.1.17",
     "node-emoji": "^1.10.0",
     "pify": "^4.0.1",

test/integration/projects/ganache-solcoverjs/.gitignore

@@ -0,0 +1 @@
+!node_modules

test/integration/projects/ganache-solcoverjs/.solcover.js

@@ -0,0 +1,5 @@
+module.exports = {
+  client: require('ganache-cli'),
+  silent: process.env.SILENT ? true : false,
+  istanbulReporter: ['json-summary', 'text'],
+}

test/integration/projects/ganache-solcoverjs/buidler.config.js

@@ -0,0 +1,8 @@
+const { loadPluginFile } = require("@nomiclabs/buidler/plugins-testing");
+loadPluginFile(__dirname + "/../dist/buidler.plugin");
+usePlugin("@nomiclabs/buidler-truffle5");
+
+module.exports={
+  defaultNetwork: "buidlerevm",
+  logger: process.env.SILENT ? { log: () => {} } : console,
+};

test/integration/projects/ganache-solcoverjs/contracts/ContractA.sol

@@ -0,0 +1,17 @@
+pragma solidity ^0.5.0;
+
+
+contract ContractA {
+  uint x;
+  constructor() public {
+  }
+
+  function sendFn() public {
+    x = 5;
+  }
+
+  function callFn() public pure returns (uint){
+    uint y = 5;
+    return y;
+  }
+}

test/integration/projects/ganache-solcoverjs/contracts/ContractB.sol

@@ -0,0 +1,17 @@
+pragma solidity ^0.5.0;
+
+
+contract ContractB {
+  uint x;
+  constructor() public {
+  }
+
+  function sendFn() public {
+    x = 5;
+  }
+
+  function callFn() public pure returns (uint){
+    uint y = 5;
+    return y;
+  }
+}

test/integration/projects/ganache-solcoverjs/contracts/ContractC.sol

@@ -0,0 +1,17 @@
+pragma solidity ^0.5.0;
+
+
+contract ContractC {
+  uint x;
+  constructor() public {
+  }
+
+  function sendFn() public {
+    x = 5;
+  }
+
+  function callFn() public pure returns (uint){
+    uint y = 5;
+    return y;
+  }
+}

test/integration/projects/ganache-solcoverjs/contracts/Migrations.sol

@@ -0,0 +1,23 @@
+pragma solidity >=0.4.21 <0.6.0;
+
+contract Migrations {
+  address public owner;
+  uint public last_completed_migration;
+
+  constructor() public {
+    owner = msg.sender;
+  }
+
+  modifier restricted() {
+    if (msg.sender == owner) _;
+  }
+
+  function setCompleted(uint completed) public restricted {
+    last_completed_migration = completed;
+  }
+
+  function upgrade(address new_address) public restricted {
+    Migrations upgraded = Migrations(new_address);
+    upgraded.setCompleted(last_completed_migration);
+  }
+}

test/integration/projects/ganache-solcoverjs/test/contracta.js

@@ -0,0 +1,15 @@
+const ContractA = artifacts.require("ContractA");
+
+contract("contracta", function(accounts) {
+  let instance;
+
+  before(async () => instance = await ContractA.new())
+
+  it('sends [ @skipForCoverage ]', async function(){
+    await instance.sendFn();
+  });
+
+  it('calls [ @skipForCoverage ]', async function(){
+    await instance.callFn();
+  })
+});

test/integration/projects/ganache-solcoverjs/test/contractb.js

@@ -0,0 +1,15 @@
+const ContractB = artifacts.require("ContractB");
+
+contract("contractB [ @skipForCoverage ]", function(accounts) {
+  let instance;
+
+  before(async () => instance = await ContractB.new())
+
+  it('sends', async function(){
+    await instance.sendFn();
+  });
+
+  it('calls', async function(){
+    await instance.callFn();
+  })
+});

test/integration/projects/ganache-solcoverjs/test/contractc.js

@@ -0,0 +1,20 @@
+const ContractC = artifacts.require("ContractC");
+
+contract("contractc", function(accounts) {
+  let instance;
+
+  before(async () => instance = await ContractC.new())
+
+  it('sends', async function(){
+    await instance.sendFn();
+  });
+
+  it('calls', async function(){
+    await instance.callFn();
+  })
+
+  it('sends', async function(){
+    await instance.sendFn();
+  });
+
+});

test/integration/projects/ganache-solcoverjs/truffle-config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  networks: {},
+  mocha: {},
+  compilers: {
+    solc: {}
+  }
+}

test/units/api.js

@@ -0,0 +1,54 @@
+const assert = require('assert');
+const util = require('./../util/util.js');
+const API = require('./../../lib/api.js');
+
+describe('api', () => {
+  const opts = {silent: true};
+
+  it('getInstrumentationData', function(){
+    const api = new API(opts);
+    const canonicalPath = 'statements/single.sol'
+    const source = util.getCode(canonicalPath);
+
+    api.instrument([{
+      source: source,
+      canonicalPath: canonicalPath
+    }]);
+
+    const data = api.getInstrumentationData();
+
+    const hash = Object.keys(data)[0];
+    assert(data[hash].hits === 0);
+  });
+
+  it('setInstrumentationData', function(){
+    let api = new API(opts);
+
+    const canonicalPath = 'statements/single.sol'
+    const source = util.getCode(canonicalPath);
+
+    api.instrument([{
+      source: source,
+      canonicalPath: canonicalPath
+    }]);
+
+    const cloneA = api.getInstrumentationData();
+    const hash = Object.keys(cloneA)[0];
+
+    // Verify cloning
+    cloneA[hash].hits = 5;
+    const cloneB = api.getInstrumentationData();
+    assert(cloneB[hash].hits === 0);
+
+    // Verify setting
+    api = new API(opts);
+    api.instrument([{
+      source: source,
+      canonicalPath: canonicalPath
+    }]);
+
+    api.setInstrumentationData(cloneA);
+    const cloneC = api.getInstrumentationData();
+    assert(cloneC[hash].hits === 5);
+  });
+})

test/units/buidler/flags.js

@@ -10,9 +10,6 @@ const plugin = require('../../../dist/buidler.plugin');
 // =======================
 // CLI Options / Flags
 // =======================
-async function delay(){
-  return new Promise(res  => setTimeout(() => res(), 1000))
-}
 
 describe('Buidler Plugin: command line options', function() {
   let buidlerConfig;

test/units/truffle/flags.js

@@ -22,6 +22,7 @@ describe('Truffle Plugin: command line options', function() {
     solcoverConfig = {};
     truffleConfig = mock.getDefaultTruffleConfig();
     verify.cleanInitialState();
+
   })
 
   afterEach(() => mock.clean());

test/units/truffle/standard.js

@@ -47,7 +47,7 @@ describe('Truffle Plugin: standard use cases', function() {
   });
 
   // Instrumentation speed is fine - but this takes solc almost a minute to compile
-  // so annoying. Unskip whenever modifying the instrumentation files though.....
+  // Unskip whenever modifying the instrumentation files though.....
   it.skip('with many unbracketed statements (time check)', async function() {
     truffleConfig.compilers.solc.version = "0.4.24";
 
@@ -156,7 +156,10 @@ describe('Truffle Plugin: standard use cases', function() {
   });
 
   // Truffle test asserts deployment cost is greater than 20,000,000 gas
-  it.skip('deployment cost > block gasLimit', async function() {
+  // Test times out on CircleCI @ 100000 ms. Fine locally though.
+  it('deployment cost > block gasLimit', async function() {
+    if (process.env.CI) return;
+
     mock.install('Expensive', 'block-gas-limit.js', solcoverConfig);
     await plugin(truffleConfig);
   });
@@ -181,6 +184,24 @@ describe('Truffle Plugin: standard use cases', function() {
     assert(output[path].fnMap['2'].name === 'getX', 'cov missing "getX"');
   });
 
+  // This test tightly coupled to the ganache version in truffle dev dep
+  it('uses the server from truffle by default', async function(){
+    truffleConfig.logger = mock.testLogger;
+    truffleConfig.version = true;
+
+    // Baseline inequality check
+    const truffleClientVersion = "v2.5.7";
+
+    // Truffle client
+    mock.install('Simple', 'simple.js', solcoverConfig);
+    await plugin(truffleConfig);
+
+    assert(
+      mock.loggerOutput.val.includes(truffleClientVersion),
+      `Should use truffles ganache: ${mock.loggerOutput.val}`
+    );
+  });
+
   it('uses the fallback server', async function(){
     truffleConfig.logger = mock.testLogger;
     solcoverConfig.forceBackupServer = true;
@@ -189,11 +210,39 @@ describe('Truffle Plugin: standard use cases', function() {
     await plugin(truffleConfig);
 
     assert(
-      mock.loggerOutput.val.includes("Using ganache-core-sc"),
+      mock.loggerOutput.val.includes("Using ganache-cli"),
       `Should notify about backup server module: ${mock.loggerOutput.val}`
     );
   });
 
+  // This test tightly coupled to the ganache version in production deps
+  // "test-files" project solcoverjs includes `client: require('ganache-cli')`
+  it('config: client', async function(){
+    truffleConfig.logger = mock.testLogger;
+    truffleConfig.version = true;
+
+    const configClientVersion = "v2.8.0";
+
+    // Config client
+    mock.installFullProject('ganache-solcoverjs');
+    await plugin(truffleConfig);
+
+    assert(
+      mock.loggerOutput.val.includes(configClientVersion),
+      `Should use solcover provided ganache: ${mock.loggerOutput.val}`
+    );
+  });
+
+  it('config: istanbulFolder', async function(){
+    solcoverConfig.istanbulFolder = mock.pathToTemp('specialFolder');
+
+    // Truffle client
+    mock.install('Simple', 'simple.js', solcoverConfig);
+    await plugin(truffleConfig);
+
+    assert(verify.pathExists(solcoverConfig.istanbulFolder));
+  });
+
   // This project has [ @skipForCoverage ] tags in the test descriptions
   // at selected 'contract' and 'it' blocks.
   it('config: mocha options', async function() {

test/util/integration.js

@@ -55,6 +55,10 @@ function pathToContract(config, file) {
   return path.join('contracts', file);
 }
 
+function pathToTemp(_path) {
+  return path.join(temp, _path);
+}
+
 function getOutput(config){
   const workingDir = config.working_directory || config.paths.root;
   const jsonPath = path.join(workingDir, "coverage.json");
@@ -303,6 +307,7 @@ const testLogger = {
 
 
 module.exports = {
+  pathToTemp: pathToTemp,
   testLogger: testLogger,
   loggerOutput: loggerOutput,
   getDefaultTruffleConfig: getDefaultTruffleConfig,

test/util/util.js

@@ -118,11 +118,11 @@ function initializeProvider(ganache){
 }
 
 module.exports = {
+  getCode: getCode,
   pathPrefix: pathPrefix,
   filePath: filePath,
   report: report,
   instrumentAndCompile: instrumentAndCompile,
   bootstrapCoverage: bootstrapCoverage,
   initializeProvider: initializeProvider,
-
 }

test/util/verifiers.js

@@ -45,6 +45,7 @@ function coverageNotGenerated(config){
 }
 
 module.exports = {
+  pathExists: pathExists,
   lineCoverage: lineCoverage,
   coverageMissing: coverageMissing,
   cleanInitialState: cleanInitialState,