Convert LavaMoat policy generation script to Yargs application (#15626)

This PR converts `generate-lavamoat-policies.sh` to `.js` using Yargs. This makes it easier to only generate policy files for a specific build type (using the `-t` flag), which is often useful during Flask development. In addition, the `lavamoat:background:auto` scripts are renamed, and the main readme is updated with some useful tips.

Note that `lavamoat:background:auto:dev` is removed and `lavamoat:background:auto` should be used during local development.

.circleci/scripts/validate-lavamoat-policy.sh

@@ -4,7 +4,7 @@ set -e
 set -u
 set -o pipefail
 
-yarn lavamoat:auto
+yarn lavamoat:auto:ci
 
 if git diff --exit-code
 then

README.md

@@ -91,15 +91,17 @@ Whenever you change dependencies (adding, removing, or updating, either in `pack
 * The `allow-scripts` configuration in `package.json`
   * Run `yarn allow-scripts auto` to update the `allow-scripts` configuration automatically. This config determines whether the package's install/postinstall scripts are allowed to run. Review each new package to determine whether the install script needs to run or not, testing if necessary.
   * Unfortunately, `yarn allow-scripts auto` will behave inconsistently on different platforms. macOS and Windows users may see extraneous changes relating to optional dependencies.
-* The LavaMoat policy files. The _tl;dr_ is to run `yarn lavamoat:auto` to update these files, but there can be devils in the details. Continue reading for more information.
+* The LavaMoat policy files. The _tl;dr_ is to run `yarn lavamoat:auto` to update these files, but there can be devils in the details:
   * There are two sets of LavaMoat policy files:
-    * The production LavaMoat policy files (`lavamoat/browserify/*/policy.json`), which are re-generated using `yarn lavamoat:background:auto`.
+    * The production LavaMoat policy files (`lavamoat/browserify/*/policy.json`), which are re-generated using `yarn lavamoat:background:auto`. Add `--help` for usage.
       * These should be regenerated whenever the production dependencies for the background change.
     * The build system LavaMoat policy file (`lavamoat/build-system/policy.json`), which is re-generated using `yarn lavamoat:build:auto`.
       * This should be regenerated whenever the dependencies used by the build system itself change.
   * Whenever you regenerate a policy file, review the changes to determine whether the access granted to each package seems appropriate.
   * Unfortunately, `yarn lavamoat:auto` will behave inconsistently on different platforms.
   macOS and Windows users may see extraneous changes relating to optional dependencies.
+  * If you keep getting policy failures even after regenerating the policy files, try regenerating the policies after a clean install by doing:
+    * `rm -rf node_modules/ && yarn setup && yarn lavamoat:auto`
   * Keep in mind that any kind of dynamic import or dynamic use of globals may elude LavaMoat's static analysis.
   Refer to the LavaMoat documentation or ask for help if you run into any issues.
 

development/generate-lavamoat-policies.js

@@ -0,0 +1,53 @@
+const concurrently = require('concurrently');
+const yargs = require('yargs/yargs');
+const { hideBin } = require('yargs/helpers');
+const { BuildType } = require('./lib/build-type');
+
+start().catch((error) => {
+  console.error('Policy generation failed.', error);
+  process.exitCode = 1;
+});
+
+async function start() {
+  const {
+    argv: { buildTypes, parallel },
+  } = yargs(hideBin(process.argv)).usage(
+    '$0 [options]',
+    'Generate the LavaMoat policy file for one more more build types.',
+    (yargsInstance) =>
+      yargsInstance
+        .option('build-types', {
+          alias: ['t'],
+          choices: Object.values(BuildType),
+          default: Object.values(BuildType),
+          demandOption: true,
+          description: 'The build type(s) to generate policy files for.',
+        })
+        .option('parallel', {
+          alias: ['p'],
+          default: true,
+          demandOption: true,
+          description: 'Whether to generate policies in parallel.',
+          type: 'boolean',
+        })
+        .strict(),
+  );
+
+  await concurrently(
+    (Array.isArray(buildTypes) ? buildTypes : [buildTypes]).map(
+      (buildType) => ({
+        command: `yarn build scripts:prod --policy-only --build-type=${buildType}`,
+        env: {
+          WRITE_AUTO_POLICY: 1,
+        },
+        name: buildType,
+      }),
+    ),
+    {
+      killOthers: true,
+      maxProcesses: parallel ? buildTypes.length : 1,
+    },
+  );
+
+  console.log('Policy file(s) successfully generated!');
+}

development/generate-lavamoat-policies.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -u
-set -o pipefail
-
-extraArgs=()
-if [[ $# -lt 1 ]]; then
-    extraArgs+=(-m 1)
-fi
-
-# Generate LavaMoat policies for the extension background script for each build
-# type.
-# ATTN: This may tax your device when running it locally.
-concurrently --kill-others-on-fail -n main,beta,flask \
-    "${extraArgs[@]}" \
-    "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only" \
-    "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only --build-type beta" \
-    "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only --build-type flask"

package.json

@@ -73,9 +73,10 @@
     "lavamoat:build": "lavamoat development/build/index.js --policy lavamoat/build-system/policy.json --policyOverride lavamoat/build-system/policy-override.json",
     "lavamoat:build:auto": "yarn lavamoat:build --writeAutoPolicy",
     "lavamoat:debug:build": "yarn lavamoat:build --writeAutoPolicyDebug --policydebug lavamoat/build-system/policy-debug.json",
-    "lavamoat:background:auto": "./development/generate-lavamoat-policies.sh",
-    "lavamoat:background:auto:dev": "./development/generate-lavamoat-policies.sh --dev",
+    "lavamoat:background:auto": "node ./development/generate-lavamoat-policies.js",
+    "lavamoat:background:auto:ci": "node ./development/generate-lavamoat-policies.js --parallel=false",
     "lavamoat:auto": "yarn lavamoat:build:auto && yarn lavamoat:background:auto",
+    "lavamoat:auto:ci": "yarn lavamoat:build:auto && yarn lavamoat:background:auto:ci",
     "ts-migration:enumerate": "ts-node development/ts-migration-dashboard/scripts/write-list-of-files-to-convert.ts",
     "ts-migration:dashboard:watch": "ts-node development/ts-migration-dashboard/scripts/build.ts --watch",
     "ts-migration:dashboard:build": "ts-node development/ts-migration-dashboard/scripts/build.ts",