Add autoLauchServer option / update config validator (#437)

README.md

@@ -41,8 +41,8 @@ truffle run coverage [command-options]
 ## Command Options
 | Option <img width=200/> | Example <img width=750/>| Description <img width=1000/> |
 |--------------|------------------------------------|--------------------------------|
-| file  (Truffle)   | `--file="test/registry/*.js"`    | Filename or glob describing a subset of JS tests to run. (Globs must be enclosed by quotes.)|
-| testFiles  (Buidler)   | `--testFiles test/file.js` | JS test file(s) to run.|
+| file | `--file="test/registry/*.js"`    | (Truffle) Filename or glob describing a subset of JS tests to run. (Globs must be enclosed by quotes.)|
+| testFiles  | `--testFiles test/file.js` | (Buidler) JS test file(s) to run.|
 | solcoverjs | `--solcoverjs ./../.solcover.js` | Relative path from working directory to config. Useful for monorepo packages that share settings. (Path must be "./" prefixed) |
 | network    | `--network development` | Use network settings defined in the Truffle or Buidler config |
 | temp[<sup>*</sup>][14]       | `--temp build`   | :warning: **Caution** :warning:  Path to a *disposable* folder to store compilation artifacts in. Useful when your test setup scripts include hard-coded paths to a build directory. [More...][14] |
@@ -69,12 +69,12 @@ module.exports = {
 | providerOptions | *Object* | `{ }` | [ganache-core options][1]  |
 | skipFiles | *Array* | `['Migrations.sol']` | Array of contracts or folders (with paths expressed relative to the `contracts` directory) that should be skipped when doing instrumentation. |
 | istanbulFolder | *String* | `./coverage` |  Folder location for Istanbul coverage reports. |
-| istanbulReporter | *Array* | `['html', 'lcov', 'text']` | [Istanbul coverage reporters][2]  |
+| istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2]  |
 | mocha | *Object* | `{ }` | [Mocha options][3] to merge into existing mocha config. `grep` and `invert` are useful for skipping certain tests under coverage using tags in the test descriptions.|
 | onServerReady[<sup>*</sup>][14] | *Function* |   | Hook run *after* server is launched, *before* the tests execute. Useful if you need to use the Oraclize bridge or have setup scripts which rely on the server's availability. [More...][23] |
 | onCompileComplete[<sup>*</sup>][14] | *Function* |  | Hook run *after* compilation completes, *before* tests are run. Useful if you have secondary compilation steps or need to modify built artifacts. [More...][23]|
-| onTestsComplete[<sup>*</sup>][14] | *Function* |  | Hook run *after* the tests complete, *before* Istanbul reports are generated.|
-| onIstanbulComplete[<sup>*</sup>][14] | *Function* |  | Hook run *after* the Istanbul reports are generated, *before* the ganache server is shut down. Useful if you need to clean resources up.|
+| onTestsComplete[<sup>*</sup>][14] | *Function* |  | Hook run *after* the tests complete, *before* Istanbul reports are generated. [More...][23]|
+| onIstanbulComplete[<sup>*</sup>][14] | *Function* |  | Hook run *after* the Istanbul reports are generated, *before* the ganache server is shut down. Useful if you need to clean resources up. [More...][23]|
 
 [<sup>*</sup> Advanced use][14]
 

lib/api.js

@@ -46,6 +46,7 @@ class API {
     this.port = config.port || this.defaultPort;
     this.host = config.host || "127.0.0.1";
     this.providerOptions = config.providerOptions || {};
+    this.autoLaunchServer = config.autoLaunchServer === false ? false : true;
 
     this.skipFiles = config.skipFiles || [];
 
@@ -60,7 +61,6 @@ class API {
 
     this.setLoggingLevel(config.silent);
     this.ui = new AppUI(this.log);
-
   }
 
   /**
@@ -131,11 +131,15 @@ class API {
   }
 
   /**
-   * Launches an in-process ethereum client server, hooking the DataCollector to its VM.
+   * Enables coverage collection on in-process ethereum client server, hooking the DataCollector
+   * to its VM. By default, method will return a url after server has begun listening on the port
+   * specified in the config. When `autoLaunchServer` is false, method returns`ganache.server` so
+   * the consumer can control the 'server.listen' invocation themselves.
    * @param  {Object} client             ganache client
-   * @return {String}        address of server to connect to
+   * @param  {Boolean} autoLaunchServer  boolean
+   * @return {String | Server}  address of server to connect to, or initialized, unlaunched server.
    */
-  async ganache(client){
+  async ganache(client, autoLaunchServer){
     // Check for port-in-use
     if (await detect(this.port) !== this.port){
       throw new Error(this.ui.generate('server-fail', [this.port]))
@@ -146,7 +150,7 @@ class API {
     this.providerOptions.gasLimit = this.gasLimitString;
     this.providerOptions.allowUnlimitedContractSize = true;
 
-    // Launch server and attach to vm step of supplied client
+    // Attach to vm step of supplied client
     try {
       if (this.config.forceBackupServer) throw new Error()
       await this.attachToVM(client)
@@ -159,6 +163,11 @@ class API {
       await this.attachToVM(_ganache);
     }
 
+    if (autoLaunchServer === false || this.autoLaunchServer === false){
+      return this.server;
+    }
+
+    await pify(this.server.listen)(this.port);
     const address = `http://${this.host}:${this.port}`;
     this.ui.report('server', [address]);
     return address;
@@ -234,8 +243,6 @@ class API {
       vm.on('step', self.collector.step.bind(self.collector));
       return vm;
     }
-
-    await pify(this.server.listen)(this.port);
   }
 
   assertHasBlockchain(provider){

lib/validator.js

@@ -18,9 +18,12 @@ const configSchema = {
     port:                 {type: "number"},
     providerOptions:      {type: "object"},
     silent:               {type: "boolean"},
+    autoLaunchServer:     {type: "boolean"},
+    istanbulFolder:       {type: "string"},
 
     // Hooks:
     onServerReady:        {type: "function", format: "isFunction"},
+    onCompileComplete:    {type: "function", format: "isFunction"},
     onTestComplete:       {type: "function", format: "isFunction"},
     onIstanbulComplete:   {type: "function", format: "isFunction"},
 

test/units/api.js

@@ -1,9 +1,13 @@
 const assert = require('assert');
 const util = require('./../util/util.js');
 const API = require('./../../lib/api.js');
+const detect = require('detect-port');
+const Ganache = require('ganache-cli');
 
 describe('api', () => {
-  const opts = {silent: true};
+  let opts;
+
+  beforeEach(() => opts = {silent: true})
 
   it('getInstrumentationData', function(){
     const api = new API(opts);
@@ -51,4 +55,34 @@ describe('api', () => {
     const cloneC = api.getInstrumentationData();
     assert(cloneC[hash].hits === 5);
   });
+
+  it('ganache: autoLaunchServer === false', async function(){
+    const api = new API(opts);
+    const port = api.port;
+    const server = await api.ganache(Ganache, false);
+
+    assert(typeof port === 'number')
+    assert(typeof server === 'object');
+    assert(typeof server.listen === 'function');
+
+    const freePort = await detect(port);
+
+    assert(freePort === port);
+  });
+
+  it('config: autoLaunchServer: false', async function(){
+    opts.autoLaunchServer = false;
+
+    const api = new API(opts);
+    const port = api.port;
+    const server = await api.ganache(Ganache);
+
+    assert(typeof port === 'number')
+    assert(typeof server === 'object');
+    assert(typeof server.listen === 'function');
+
+    const freePort = await detect(port);
+
+    assert(freePort === port);
+  })
 })

test/units/validator.js

@@ -22,6 +22,7 @@ describe('config validation', () => {
     const options =  [
       "cwd",
       "host",
+      "istanbulFolder"
     ]
 
     options.forEach(name => {
@@ -41,6 +42,29 @@ describe('config validation', () => {
     });
   });
 
+  it('validates the "boolean" options', function(){
+    const options =  [
+      "silent",
+      "autoLaunchServer",
+    ]
+
+    options.forEach(name => {
+      // Pass
+      solcoverjs = {};
+      solcoverjs[name] = false;
+      assert(validator.validate(solcoverjs), `${name} boolean should be valid`)
+
+      // Fail
+      solcoverjs[name] = "false";
+      try {
+        validator.validate(solcoverjs);
+        assert.fail()
+      } catch (err){
+        assert(err.message.includes(`"${name}" is not of a type(s) boolean`), err.message);
+      }
+    });
+  });
+
   it('validates the "object" options', function(){
     const options =  [
       "client",
@@ -86,7 +110,7 @@ describe('config validation', () => {
     });
   });
 
-  it('validates string array options', function(){
+  it('validates the "string[]" options', function(){
     const options =  [
       "skipFiles",
       "istanbulReporter",
@@ -109,9 +133,10 @@ describe('config validation', () => {
     });
   });
 
-  it('validates function options', function(){
+  it('validates the "function" options', function(){
 
     const options =  [
+      "onCompileComplete",
       "onServerReady",
       "onTestComplete",
       "onIstanbulComplete",