Handle native solidity tests / add test logger fixtures (#398)

README.md

@@ -22,7 +22,7 @@ $ npm install --save-dev solidity-coverage
 ### Usage notes:
 + For pragma solidity >=0.4.22 <0.6.0 / Petersburg
 + Coverage runs tests a little more slowly.
-+ Coverage distorts gas consumption. 
++ Coverage distorts gas consumption.
 + Coverage launches its own in-process ganache instance. You can
   set [ganache options](https://github.com/trufflesuite/ganache-core#options) via
   the `providerOptions` key in your `.solcover.js` config file.
@@ -45,10 +45,10 @@ truffle run coverage [options]
 ### Command Options
 | Option       | Example                        | Description |
 |--------------|--------------------------------|-------------|
-| --file       | --file="test/registry/*.js"    | Filename or glob describing a subset of JS tests to run. (Globs must be enclosed by quotes.)|
-| --solcoverjs | --solcoverjs ./../.solcover.js | Relative path from working directory to config. Useful for monorepo packages that share settings. (Path must be "./" prefixed) |
-| --version    |                                | Version info |
-| --help       |                                | Usage notes  |
+| `--file`       | `--file="test/registry/*.js"`    | Filename or glob describing a subset of JS tests to run. (Globs must be enclosed by quotes.)|
+| `--solcoverjs` | `--solcoverjs ./../.solcover.js` | Relative path from working directory to config. Useful for monorepo packages that share settings. (Path must be "./" prefixed) |
+| `--version`    |                                | Version info |
+| `--help`       |                                | Usage notes  |
 |<img width=250/>|<img width=500/>              |<img width=100/>|
 
 ### Config Options
@@ -70,6 +70,7 @@ module.exports = {
 | providerOptions | *Object* | {} | ganache-core options (ex: `network_id: 55`). Complete list of options [here](https://github.com/trufflesuite/ganache-core#options).  |
 | skipFiles | *Array* | `['Migrations.sol']` | Array of contracts or folders (with paths expressed relative to the `contracts` directory) that should be skipped when doing instrumentation. `Migrations.sol` is skipped by default, and does not need to be added to this configuration option if it is used. |
 | istanbulReporter | *Array* | ['html', 'lcov', 'text'] | Coverage reporters for Istanbul. Optional reporter replaces the default reporters. |
+| silent | *Boolean* | false | suppress logging output |
 
 ### FAQ
 

dist/truffle.plugin.js

@@ -53,6 +53,7 @@ async function plugin(truffleConfig){
     coverageConfig = req.silent(solcoverjs) || {};
     coverageConfig.cwd = truffleConfig.working_directory;
     coverageConfig.originalContractsDir = truffleConfig.contracts_directory;
+    coverageConfig.log = coverageConfig.log || truffleConfig.logger.log;
 
     app = new App(coverageConfig);
 
@@ -94,7 +95,7 @@ async function plugin(truffleConfig){
 
     // Additional config
     truffleConfig.all = true;
-    truffleConfig.test_files = tests(truffleConfig);
+    truffleConfig.test_files = tests(app, truffleConfig);
     truffleConfig.compilers.solc.settings.optimizer.enabled = false;
 
     // Compile
@@ -141,15 +142,20 @@ async function plugin(truffleConfig){
 
 // -------------------------------------- Helpers --------------------------------------------------
 
-function tests(truffle){
+function tests(app, truffle){
   let target;
 
   (typeof truffle.file === 'string')
     ? target = globby.sync([truffle.file])
     : target = dir.files(truffle.test_directory, { sync: true }) || [];
 
-  const regex = /.*\.(js|ts|es|es6|jsx|sol)$/;
-  return target.filter(f => f.match(regex) != null);
+  const solregex = /.*\.(sol)$/;
+  const hasSols = target.filter(f => f.match(solregex) != null);
+
+  if (hasSols.length > 0) app.ui.report('sol-tests', [hasSols.length]);
+
+  const testregex = /.*\.(js|ts|es|es6|jsx)$/;
+  return target.filter(f => f.match(testregex) != null);
 }
 
 

lib/app.js

@@ -38,7 +38,7 @@ class App {
     this.skippedFolders = [];
     this.skipFiles = config.skipFiles || [];
 
-    this.log = config.logger ? config.logger.log : console.log;
+    this.log = config.log || console.log;
     this.setLoggingLevel(config.silent);
 
     this.gasLimit = 0xfffffffffff;

lib/ui.js

@@ -20,18 +20,22 @@ class UI {
   report(kind, args=[]){
     const ct = c.bold.green('>');
     const ds = c.bold.yellow('>');
+    const w = ":warning:";
 
     const kinds = {
-      'vm-fail': `:warning:  ${c.red('There was a problem attaching to the ganache-core VM.')} `+
-                            `${c.red('Check the provider option syntax in solidity-coverage docs.')}\n`+
-                 `:warning:  ${c.red('Using ganache-core-sc (eq. core v2.7.0) instead.')}\n`,
+      'vm-fail': `${w}  ${c.red('There was a problem attaching to the ganache-core VM.')} `+
+                       `${c.red('Check the provider option syntax in solidity-coverage docs.')}\n`+
+                 `${w}  ${c.red('Using ganache-core-sc (eq. core v2.7.0) instead.')}\n`,
+
+      'sol-tests': `\n${w}  ${c.red("This plugin cannot run Truffle's native solidity tests: ")}`+
+                           `${args[0]} test(s) will be skipped.\n`,
 
       'truffle-local':  `\n${ct} ${c.grey('Using Truffle library from local node_modules.')}\n`,
       'truffle-global': `\n${ct} ${c.grey('Using Truffle library from global node_modules.')}\n`,
 
-      'truffle-warn': `:warning:  ${c.red('Unable to require Truffle library locally or globally. ')} `+
-                                 `${c.red('Expected to find installed Truffle >= v5.0.31 ...')}\n` +
-                      `:warning:  ${c.red('Using fallback Truffle library instead (v5.0.31)')}\n`,
+      'truffle-warn': `${w}  ${c.red('Unable to require Truffle library locally or globally. ')} `+
+                            `${c.red('Expected to find installed Truffle >= v5.0.31 ...')}\n` +
+                      `${w}  ${c.red('Using fallback Truffle library instead (v5.0.31)')}\n`,
 
       'truffle-help': `Usage: truffle run coverage [options]\n\n` +
                       `Options:\n` +

scripts/run-metacoin.sh

@@ -2,6 +2,8 @@
 #
 # E2E CI: installs PR candidate on Truffle's MetaCoin and runs coverage
 #
+# Also verifies that everything works w/ truffle installed globally.
+#
 
 set -o errexit
 
@@ -22,18 +24,18 @@ echo "PR_PATH >>>>> $PR_PATH"
 
 # Install truffle and metacoin box
 npm install -g truffle
+npm install -g yarn
+
 mkdir metacoin
 cd metacoin
 truffle unbox metacoin --force
+
+# Install config with plugin
 rm truffle-config.js
 echo "module.exports={plugins:['solidity-coverage']}" > truffle-config.js
 cat truffle-config.js
 
-# TODO: Remove this rm.
-# Unknown bug running truffle native solidity tests
-rm test/TestMetaCoin.sol
-
 # Install and run solidity-coverage @ PR
 npm init --yes
-npm install --save-dev $PR_PATH
+yarn add $PR_PATH --dev
 npx truffle run coverage

test/sources/js/TestSimple.sol

@@ -0,0 +1,16 @@
+pragma solidity >=0.4.25 <0.6.0;
+
+import "truffle/Assert.sol";
+import "truffle/DeployedAddresses.sol";
+import "../contracts/Simple.sol";
+
+contract TestSimple {
+
+  function test_x_is_0() public {
+    Simple simple = Simple(DeployedAddresses.Simple());
+
+    uint expected = 0;
+
+    Assert.equal(simple.x, expected, "x should equal 0");
+  }
+}

test/units/app.js

@@ -59,11 +59,9 @@ describe('app', function() {
   beforeEach(() => {
     mock.clean();
 
+    mock.loggerOutput.val = '';
     solcoverConfig = {};
     truffleConfig = mock.getDefaultTruffleConfig();
-
-    if (process.env.SILENT)
-      solcoverConfig.silent = true;
   })
 
   afterEach(() => mock.clean());
@@ -149,6 +147,20 @@ describe('app', function() {
     await plugin(truffleConfig);
   });
 
+  it('project contains native solidity tests', async function(){
+    assertCleanInitialState();
+
+    mock.install('Simple', 'TestSimple.sol', solcoverConfig);
+
+    truffleConfig.logger = mock.testLogger;
+    await plugin(truffleConfig);
+
+    assert(
+      mock.loggerOutput.val.includes('native solidity tests'),
+      `Should warn it is skipping native solidity tests (output --> ${mock.loggerOutput.val}`
+    );
+  });
+
   it('truffle run coverage --config ../.solcover.js', async function() {
     assertCleanInitialState();
 
@@ -179,15 +191,40 @@ describe('app', function() {
   it('truffle run coverage --help', async function(){
     assertCleanInitialState();
     truffleConfig.help = "true";
+
+    truffleConfig.logger = mock.testLogger;
     mock.install('Simple', 'simple.js', solcoverConfig);
     await plugin(truffleConfig);
+
+    assert(
+      mock.loggerOutput.val.includes('Usage'),
+      `Should output help with Usage instruction  (output --> ${mock.loggerOutput.val}`
+    );
   })
 
   it('truffle run coverage --version', async function(){
     assertCleanInitialState();
     truffleConfig.version = "true";
+
+    truffleConfig.logger = mock.testLogger;
     mock.install('Simple', 'simple.js', solcoverConfig);
     await plugin(truffleConfig);
+
+    assert(
+      mock.loggerOutput.val.includes('truffle'),
+      `Should output truffle version (output --> ${mock.loggerOutput.val}`
+    );
+
+    assert(
+      mock.loggerOutput.val.includes('ganache-core'),
+      `Should output ganache-core version (output --> ${mock.loggerOutput.val}`
+    );
+
+    assert(
+      mock.loggerOutput.val.includes('solidity-coverage'),
+      `Should output solidity-coverage version (output --> ${mock.loggerOutput.val}`
+    );
+
   })
 
   it('truffle run coverage --file test/<fileName>', async function() {

test/util/integration.truffle.js

@@ -18,6 +18,29 @@ const migrationPath =     `${temp}/migrations/2_deploy.js`;
 const templatePath =      './test/integration/truffle/*';
 const projectPath =       './test/integration/projects/'
 
+
+// ==========================
+// Misc Utils
+// ==========================
+function decacheConfigs(){
+  decache(`${process.cwd()}/${temp}/.solcover.js`);
+  decache(`${process.cwd()}/${temp}/${truffleConfigName}`);
+}
+
+function clean() {
+  shell.config.silent = true;
+
+  shell.rm('-Rf', temp);
+  shell.rm('-Rf', 'coverage');
+  shell.rm('coverage.json');
+  shell.rm('.solcover.js');
+
+  shell.config.silent = false;
+};
+
+// ==========================
+// Configuration
+// ==========================
 function getDefaultTruffleConfig(){
   const logger = process.env.SILENT ? { log: () => {} } : console;
   const reporter = process.env.SILENT ? 'dot' : 'spec';
@@ -59,6 +82,9 @@ function getTruffleConfigJS(config){
   return `module.exports = ${JSON.stringify(getDefaultTruffleConfig(), null, ' ')}`
 }
 
+// ==========================
+// Migration Generators
+// ==========================
 function deploySingle(contractName){
   return `
     const A = artifacts.require("${contractName}");
@@ -78,11 +104,9 @@ function deployDouble(contractNames){
   `;
 }
 
-function decacheConfigs(){
-  decache(`${process.cwd()}/${temp}/.solcover.js`);
-  decache(`${process.cwd()}/${temp}/${truffleConfigName}`);
-}
-
+// ==========================
+// Project Installers
+// ==========================
 /**
  * Installs mock truffle project at ./temp with a single contract
  * and test specified by the params.
@@ -161,23 +185,25 @@ function installFullProject(name) {
   decacheConfigs();
 }
 
-
-/**
- * Removes mock truffle project and coverage reports generated by test
- */
-function clean() {
-  shell.config.silent = true;
-
-  shell.rm('-Rf', temp);
-  shell.rm('-Rf', 'coverage');
-  shell.rm('coverage.json');
-  shell.rm('.solcover.js');
-
-  shell.config.silent = false;
+// ==========================
+// Logging
+// ==========================
+const loggerOutput = {
+  val: ''
 };
 
+const testLogger = {
+  log: (val) => {
+    if (val !== undefined) loggerOutput.val += val;
+    if (!process.env.SILENT && val !== undefined)
+       console.log(val)
+  }
+}
+
 
 module.exports = {
+  testLogger: testLogger,
+  loggerOutput: loggerOutput,
   getDefaultTruffleConfig: getDefaultTruffleConfig,
   install: install,
   installDouble: installDouble,