Add `--leave-running` flag to E2E test script (#11321)

* Add `--leave-running` flag to E2E test script

The `--leave-running` flag has been added to the E2E test runner. This
ensures the browser, ganache, and everything else stays running upon
test failure. This is useful for local debugging, for investigating
what state the extension was in when it failed.

* Add `--leave-running` support to `metamask-ui.spec.js`

test/e2e/helpers.js

@@ -33,6 +33,7 @@ async function withFixtures(options, testSuite) {
   let segmentStub;
 
   let webDriver;
+  let failed = false;
   try {
     await ganacheServer.start(ganacheOptions);
     if (ganacheOptions?.concurrent) {
@@ -103,6 +104,7 @@ async function withFixtures(options, testSuite) {
       }
     }
   } catch (error) {
+    failed = true;
     if (webDriver) {
       try {
         await webDriver.verboseReportOnFailure(title);
@@ -112,6 +114,7 @@ async function withFixtures(options, testSuite) {
     }
     throw error;
   } finally {
+    if (!failed || process.env.E2E_LEAVE_RUNNING !== 'true') {
       await fixtureServer.stop();
       await ganacheServer.quit();
       if (ganacheOptions?.concurrent) {
@@ -134,6 +137,7 @@ async function withFixtures(options, testSuite) {
         await segmentServer.stop();
       }
     }
+  }
 }
 
 module.exports = {

test/e2e/metamask-ui.spec.js

@@ -20,6 +20,8 @@ describe('MetaMask', function () {
 
   this.bail(true);
 
+  let failed = false;
+
   before(async function () {
     await ganacheServer.start();
     const dappDirectory = path.resolve(
@@ -54,11 +56,15 @@ describe('MetaMask', function () {
       }
     }
     if (this.currentTest.state === 'failed') {
+      failed = true;
       await driver.verboseReportOnFailure(this.currentTest.title);
     }
   });
 
   after(async function () {
+    if (process.env.E2E_LEAVE_RUNNING === 'true' && failed) {
+      return;
+    }
     await ganacheServer.quit();
     await driver.quit();
     await new Promise((resolve, reject) => {

test/e2e/run-e2e-test.js

@@ -24,6 +24,12 @@ async function main() {
               'Set how many times the test should be retried upon failure.',
             type: 'number',
           })
+          .option('leave-running', {
+            default: false,
+            description:
+              'Leaves the browser running after a test fails, along with anything else that the test used (ganache, the test dapp, etc.)',
+            type: 'boolean',
+          })
           .positional('e2e-test-path', {
             describe: 'The path for the E2E test to run.',
             type: 'string',
@@ -33,7 +39,7 @@ async function main() {
     .strict()
     .help('help');
 
-  const { browser, e2eTestPath, retries } = argv;
+  const { browser, e2eTestPath, retries, leaveRunning } = argv;
 
   if (!browser) {
     exitWithError(
@@ -63,6 +69,10 @@ async function main() {
     throw error;
   }
 
+  if (leaveRunning) {
+    process.env.E2E_LEAVE_RUNNING = 'true';
+  }
+
   await retry(retries, async () => {
     await runInShell('yarn', ['mocha', '--no-timeouts', e2eTestPath]);
   });