Fix LavaMoat background policy generation (#12844)

The LavaMoat policy generation script would sporadically fail because
it ran the build concurrently three times, and the build includes
steps that delete the `dist` directory and write to it. So if one build
process tried to write to the directory after another deleted it, it
would fail.

This was solved by adding a new `--policy-only` flag to the build
script, and a new `scripts:prod` task. The `scripts:prod` task only
runs the script tasks for prod, rather than the entire build process.
The `--policy-only` flag stops the script tasks once the policy has
been written, and stops any other files from being written to disk.

This prevents the three concurrent build processes from getting in each
others way, and it dramatically speeds up the process.

development/build/README.md

@@ -52,6 +52,9 @@ Options:
                       bundle. Setting this to `false` is useful e.g. when
                       linking dependencies that are incompatible with lockdown.
                                                        [boolean] [default: true]
+  --policy-only       Stops the build after generating the LavaMoat policy,
+                      skipping any writes to disk.
+                                                       [boolean] [deafult: false]
   --skip-stats        Whether to refrain from logging build progress. Mostly
                       used internally.
                                                       [boolean] [default: false]

development/build/index.js

@@ -52,6 +52,7 @@ function defineAndRunBuildTasks() {
     buildType,
     entryTask,
     isLavaMoat,
+    policyOnly,
     shouldIncludeLockdown,
     shouldLintFenceFiles,
     skipStats,
@@ -84,6 +85,7 @@ function defineAndRunBuildTasks() {
     ignoredFiles,
     isLavaMoat,
     livereload,
+    policyOnly,
     shouldLintFenceFiles,
   });
 
@@ -134,6 +136,9 @@ function defineAndRunBuildTasks() {
     ),
   );
 
+  // build just production scripts, for LavaMoat policy generation purposes
+  createTask('scripts:prod', scriptTasks.prod);
+
   // build for CI testing
   createTask(
     'test',
@@ -157,6 +162,7 @@ function parseArgv() {
     BuildType: 'build-type',
     LintFenceFiles: 'lint-fence-files',
     Lockdown: 'lockdown',
+    PolicyOnly: 'policy-only',
     SkipStats: 'skip-stats',
   };
 
@@ -164,6 +170,7 @@ function parseArgv() {
     boolean: [
       NamedArgs.LintFenceFiles,
       NamedArgs.Lockdown,
+      NamedArgs.PolicyOnly,
       NamedArgs.SkipStats,
     ],
     string: [NamedArgs.BuildType],
@@ -171,6 +178,7 @@ function parseArgv() {
       [NamedArgs.BuildType]: BuildType.main,
       [NamedArgs.LintFenceFiles]: true,
       [NamedArgs.Lockdown]: true,
+      [NamedArgs.PolicyOnly]: false,
       [NamedArgs.SkipStats]: false,
     },
   });
@@ -198,10 +206,13 @@ function parseArgv() {
     ? argv[NamedArgs.LintFenceFiles]
     : !/dev/iu.test(entryTask);
 
+  const policyOnly = argv[NamedArgs.PolicyOnly];
+
   return {
     buildType,
     entryTask,
     isLavaMoat: process.argv[0].includes('lavamoat'),
+    policyOnly,
     shouldIncludeLockdown: argv[NamedArgs.Lockdown],
     shouldLintFenceFiles,
     skipStats: argv[NamedArgs.SkipStats],

development/build/scripts.js

@@ -134,6 +134,10 @@ function getSegmentWriteKey({ buildType, environment }) {
   throw new Error(`Invalid build type: '${buildType}'`);
 }
 
+const noopWriteStream = through.obj((_file, _fileEncoding, callback) =>
+  callback(),
+);
+
 module.exports = createScriptTasks;
 
 function createScriptTasks({
@@ -143,6 +147,7 @@ function createScriptTasks({
   isLavaMoat,
   livereload,
   shouldLintFenceFiles,
+  policyOnly,
 }) {
   // internal tasks
   const core = {
@@ -185,6 +190,7 @@ function createScriptTasks({
           return `./app/scripts/${label}.js`;
         }),
         ignoredFiles,
+        policyOnly,
         shouldLintFenceFiles,
         testing,
       }),
@@ -241,6 +247,7 @@ function createScriptTasks({
       runInChildProcess(subtask, {
         buildType,
         isLavaMoat,
+        policyOnly,
         shouldLintFenceFiles,
       }),
     );
@@ -258,6 +265,8 @@ function createScriptTasks({
       entryFilepath: `./app/scripts/${label}.js`,
       ignoredFiles,
       label,
+      testing,
+      policyOnly,
       shouldLintFenceFiles,
     });
   }
@@ -272,6 +281,8 @@ function createScriptTasks({
       entryFilepath: `./app/scripts/${label}.js`,
       ignoredFiles,
       label,
+      testing,
+      policyOnly,
       shouldLintFenceFiles,
     });
   }
@@ -286,6 +297,8 @@ function createScriptTasks({
       entryFilepath: `./app/scripts/${label}.js`,
       ignoredFiles,
       label,
+      testing,
+      policyOnly,
       shouldLintFenceFiles,
     });
   }
@@ -303,6 +316,7 @@ function createScriptTasks({
         entryFilepath: `./app/scripts/${inpage}.js`,
         label: inpage,
         ignoredFiles,
+        policyOnly,
         shouldLintFenceFiles,
         testing,
       }),
@@ -314,6 +328,7 @@ function createScriptTasks({
         entryFilepath: `./app/scripts/${contentscript}.js`,
         label: contentscript,
         ignoredFiles,
+        policyOnly,
         shouldLintFenceFiles,
         testing,
       }),
@@ -327,6 +342,7 @@ function createFactoredBuild({
   devMode,
   entryFiles,
   ignoredFiles,
+  policyOnly,
   shouldLintFenceFiles,
   testing,
 }) {
@@ -346,6 +362,7 @@ function createFactoredBuild({
       devMode,
       envVars,
       ignoredFiles,
+      policyOnly,
       minify,
       reloadOnChange,
       shouldLintFenceFiles,
@@ -417,12 +434,17 @@ function createFactoredBuild({
       // setup bundle destination
       browserPlatforms.forEach((platform) => {
         const dest = `./dist/${platform}/`;
-        pipeline.get('dest').push(gulp.dest(dest));
+        const destination = policyOnly ? noopWriteStream : gulp.dest(dest);
+        pipeline.get('dest').push(destination);
       });
     });
 
     // wait for bundle completion for postprocessing
     events.on('bundleDone', () => {
+      // Skip HTML generation if nothing is to be written to disk
+      if (policyOnly) {
+        return;
+      }
       const commonSet = sizeGroupMap.get('common');
       // create entry points for each file
       for (const [groupLabel, groupSet] of sizeGroupMap.entries()) {
@@ -496,6 +518,7 @@ function createNormalBundle({
   extraEntries = [],
   ignoredFiles,
   label,
+  policyOnly,
   modulesToExpose,
   shouldLintFenceFiles,
   testing,
@@ -516,6 +539,7 @@ function createNormalBundle({
       devMode,
       envVars,
       ignoredFiles,
+      policyOnly,
       minify,
       reloadOnChange,
       shouldLintFenceFiles,
@@ -540,7 +564,8 @@ function createNormalBundle({
       // setup bundle destination
       browserPlatforms.forEach((platform) => {
         const dest = `./dist/${platform}/`;
-        pipeline.get('dest').push(gulp.dest(dest));
+        const destination = policyOnly ? noopWriteStream : gulp.dest(dest);
+        pipeline.get('dest').push(destination);
       });
     });
 
@@ -570,6 +595,7 @@ function setupBundlerDefaults(
     devMode,
     envVars,
     ignoredFiles,
+    policyOnly,
     minify,
     reloadOnChange,
     shouldLintFenceFiles,
@@ -613,6 +639,7 @@ function setupBundlerDefaults(
     setupReloadOnChange(buildConfiguration);
   }
 
+  if (!policyOnly) {
     if (minify) {
       setupMinification(buildConfiguration);
     }
@@ -620,6 +647,7 @@ function setupBundlerDefaults(
     // Setup source maps
     setupSourcemaps(buildConfiguration, { devMode });
   }
+}
 
 function setupReloadOnChange({ bundlerOpts, events }) {
   // Add plugin to options

development/build/task.js

@@ -50,7 +50,7 @@ function createTask(taskName, taskFn) {
 
 function runInChildProcess(
   task,
-  { buildType, isLavaMoat, shouldLintFenceFiles },
+  { buildType, isLavaMoat, policyOnly, shouldLintFenceFiles },
 ) {
   const taskName = typeof task === 'string' ? task : task.taskName;
   if (!taskName) {
@@ -74,6 +74,7 @@ function runInChildProcess(
           '--lint-fence-files',
           shouldLintFenceFiles,
           '--skip-stats',
+          ...(policyOnly ? ['--policy-only'] : []),
         ],
         {
           env: process.env,
@@ -90,6 +91,7 @@ function runInChildProcess(
           '--lint-fence-files',
           shouldLintFenceFiles,
           '--skip-stats',
+          ...(policyOnly ? ['--policy-only'] : []),
         ],
         {
           env: process.env,

development/generate-lavamoat-policies.sh

@@ -8,6 +8,6 @@ set -o pipefail
 # type.
 # ATTN: This may tax your device when running it locally.
 concurrently --kill-others-on-fail -n main,beta,flask \
-  "WRITE_AUTO_POLICY=1 yarn dist" \
-  "WRITE_AUTO_POLICY=1 yarn dist --build-type beta" \
-  "WRITE_AUTO_POLICY=1 yarn dist --build-type flask"
+  "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only" \
+  "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only --build-type beta" \
+  "WRITE_AUTO_POLICY=1 yarn build scripts:prod --policy-only --build-type flask"