Migrate the build script to `yargs` (#14836)

The build script now uses `yargs` rather than `minimist`. The CLI is
now better documented, and we have additional validation for each
option.

A patch for `yargs` was required because it would blow up on the line
`Error.captureStackTrace`. For some reason when running under LavaMoat,
that property did not exist.

Closes #12766
feature/default_network_editable
Mark Stacey 2 years ago committed by GitHub
parent ac7245b50c
commit b68aee1bef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 36
      development/build/README.md
  2. 48
      development/build/constants.js
  3. 7
      development/build/etc.js
  4. 157
      development/build/index.js
  5. 12
      development/build/manifest.js
  6. 5
      development/build/static.js
  7. 7
      development/build/styles.js
  8. 30
      lavamoat/build-system/policy-override.json
  9. 74
      lavamoat/build-system/policy.json
  10. 1
      package.json
  11. 11
      patches/yargs+17.4.1.patch

@ -26,38 +26,4 @@ Source file bundling tasks are implemented in the [`./development/build/scripts.
## Usage ## Usage
```text See `node ./development/build/index.js --help`
Usage: yarn build <entry-task> [options]
Commands:
yarn build prod Create an optimized build for production environments.
yarn build dev Create an unoptimized, live-reloaded build for local
development.
yarn build test Create an optimized build for running e2e tests.
yarn build testDev Create an unoptimized, live-reloaded build for running
e2e tests.
Options:
--build-type The "type" of build to create. One of: "beta", "flask",
"main"
[string] [default: "main"]
--lint-fence-files Whether files with code fences should be linted after
fences have been removed by the code fencing transform.
The build will fail if linting fails.
Defaults to `false` if the entry task is `dev` or
`testDev`, and `true` otherwise.
[boolean] [default: <varies>]
--lockdown Whether to include SES lockdown files in the extension
bundle. Setting this to `false` is useful e.g. when
linking dependencies that are incompatible with lockdown.
[boolean] [default: true]
--policy-only Stops the build after generating the LavaMoat policy,
skipping any writes to disk.
[boolean] [deafult: false]
--skip-stats Whether to refrain from logging build progress. Mostly
used internally.
[boolean] [default: false]
```

@ -0,0 +1,48 @@
const TASKS = {
CLEAN: 'clean',
DEV: 'dev',
LINT_SCSS: 'lint-scss',
MANIFEST_DEV: 'manifest:dev',
MANIFEST_PROD: 'manifest:prod',
MANIFEST_TEST: 'manifest:test',
MANIFEST_TEST_DEV: 'manifest:testDev',
PROD: 'prod',
RELOAD: 'reload',
SCRIPTS_PROD: 'scripts:prod',
SCRIPTS_CORE_DEV_STANDARD_ENTRY_POINTS:
'scripts:core:dev:standardEntryPoints',
SCRIPTS_CORE_DEV_CONTENTSCRIPT: 'scripts:core:dev:contentscript',
SCRIPTS_CORE_DEV_DISABLE_CONSOLE: 'scripts:core:dev:disable-console',
SCRIPTS_CORE_DEV_SENTRY: 'scripts:core:dev:sentry',
SCRIPTS_CORE_DEV_PHISHING_DETECT: 'scripts:core:dev:phishing-detect',
SCRIPTS_CORE_PROD_STANDARD_ENTRY_POINTS:
'scripts:core:prod:standardEntryPoints',
SCRIPTS_CORE_PROD_CONTENTSCRIPT: 'scripts:core:prod:contentscript',
SCRIPTS_CORE_PROD_DISABLE_CONSOLE: 'scripts:core:prod:disable-console',
SCRIPTS_CORE_PROD_SENTRY: 'scripts:core:prod:sentry',
SCRIPTS_CORE_PROD_PHISHING_DETECT: 'scripts:core:prod:phishing-detect',
SCRIPTS_CORE_TEST_LIVE_STANDARD_ENTRY_POINTS:
'scripts:core:test-live:standardEntryPoints',
SCRIPTS_CORE_TEST_LIVE_CONTENTSCRIPT: 'scripts:core:test-live:contentscript',
SCRIPTS_CORE_TEST_LIVE_DISABLE_CONSOLE:
'scripts:core:test-live:disable-console',
SCRIPTS_CORE_TEST_LIVE_SENTRY: 'scripts:core:test-live:sentry',
SCRIPTS_CORE_TEST_LIVE_PHISHING_DETECT:
'scripts:core:test-live:phishing-detect',
SCRIPTS_CORE_TEST_STANDARD_ENTRY_POINTS:
'scripts:core:test:standardEntryPoints',
SCRIPTS_CORE_TEST_CONTENTSCRIPT: 'scripts:core:test:contentscript',
SCRIPTS_CORE_TEST_DISABLE_CONSOLE: 'scripts:core:test:disable-console',
SCRIPTS_CORE_TEST_SENTRY: 'scripts:core:test:sentry',
SCRIPTS_CORE_TEST_PHISHING_DETECT: 'scripts:core:test:phishing-detect',
STATIC_DEV: 'static:dev',
STATIC_PROD: 'static:prod',
STYLES: 'styles',
STYLES_DEV: 'styles:dev',
STYLES_PROD: 'styles:prod',
TEST: 'test',
TEST_DEV: 'testDev',
ZIP: 'zip',
};
module.exports = { TASKS };

@ -7,12 +7,13 @@ const pify = require('pify');
const pump = pify(require('pump')); const pump = pify(require('pump'));
const { BuildType } = require('../lib/build-type'); const { BuildType } = require('../lib/build-type');
const { TASKS } = require('./constants');
const { createTask, composeParallel } = require('./task'); const { createTask, composeParallel } = require('./task');
module.exports = createEtcTasks; module.exports = createEtcTasks;
function createEtcTasks({ browserPlatforms, buildType, livereload, version }) { function createEtcTasks({ browserPlatforms, buildType, livereload, version }) {
const clean = createTask('clean', async function clean() { const clean = createTask(TASKS.CLEAN, async function clean() {
await del(['./dist/*']); await del(['./dist/*']);
await Promise.all( await Promise.all(
browserPlatforms.map(async (platform) => { browserPlatforms.map(async (platform) => {
@ -21,13 +22,13 @@ function createEtcTasks({ browserPlatforms, buildType, livereload, version }) {
); );
}); });
const reload = createTask('reload', function devReload() { const reload = createTask(TASKS.RELOAD, function devReload() {
livereload.listen({ port: 35729 }); livereload.listen({ port: 35729 });
}); });
// zip tasks for distribution // zip tasks for distribution
const zip = createTask( const zip = createTask(
'zip', TASKS.ZIP,
composeParallel( composeParallel(
...browserPlatforms.map((platform) => ...browserPlatforms.map((platform) =>
createZipTask(platform, buildType, version), createZipTask(platform, buildType, version),

@ -5,10 +5,12 @@
// //
const path = require('path'); const path = require('path');
const livereload = require('gulp-livereload'); const livereload = require('gulp-livereload');
const minimist = require('minimist'); const yargs = require('yargs/yargs');
const { hideBin } = require('yargs/helpers');
const { sync: globby } = require('globby'); const { sync: globby } = require('globby');
const { getVersion } = require('../lib/get-version'); const { getVersion } = require('../lib/get-version');
const { BuildType } = require('../lib/build-type'); const { BuildType } = require('../lib/build-type');
const { TASKS } = require('./constants');
const { const {
createTask, createTask,
composeSeries, composeSeries,
@ -110,7 +112,7 @@ function defineAndRunBuildTasks() {
// build for development (livereload) // build for development (livereload)
createTask( createTask(
'dev', TASKS.DEV,
composeSeries( composeSeries(
clean, clean,
styleTasks.dev, styleTasks.dev,
@ -125,7 +127,7 @@ function defineAndRunBuildTasks() {
// build for test development (livereload) // build for test development (livereload)
createTask( createTask(
'testDev', TASKS.TEST_DEV,
composeSeries( composeSeries(
clean, clean,
styleTasks.dev, styleTasks.dev,
@ -140,7 +142,7 @@ function defineAndRunBuildTasks() {
// build for prod release // build for prod release
createTask( createTask(
'prod', TASKS.PROD,
composeSeries( composeSeries(
clean, clean,
styleTasks.prod, styleTasks.prod,
@ -150,11 +152,11 @@ function defineAndRunBuildTasks() {
); );
// build just production scripts, for LavaMoat policy generation purposes // build just production scripts, for LavaMoat policy generation purposes
createTask('scripts:prod', scriptTasks.prod); createTask(TASKS.SCRIPTS_PROD, scriptTasks.prod);
// build for CI testing // build for CI testing
createTask( createTask(
'test', TASKS.TEST,
composeSeries( composeSeries(
clean, clean,
styleTasks.prod, styleTasks.prod,
@ -164,89 +166,108 @@ function defineAndRunBuildTasks() {
); );
// special build for minimal CI testing // special build for minimal CI testing
createTask('styles', styleTasks.prod); createTask(TASKS.styles, styleTasks.prod);
// Finally, start the build process by running the entry task. // Finally, start the build process by running the entry task.
runTask(entryTask, { skipStats }); runTask(entryTask, { skipStats });
} }
function parseArgv() { function parseArgv() {
const NamedArgs = { const { argv } = yargs(hideBin(process.argv))
ApplyLavaMoat: 'apply-lavamoat', .usage('$0 <task> [options]', 'Build the MetaMask extension.', (_yargs) =>
BuildType: 'build-type', _yargs
BuildVersion: 'build-version', .positional('task', {
LintFenceFiles: 'lint-fence-files', description: `The task to run. There are a number of main tasks, each of which calls other tasks internally. The main tasks are:
Lockdown: 'lockdown',
PolicyOnly: 'policy-only',
SkipStats: 'skip-stats',
};
const argv = minimist(process.argv.slice(2), { prod: Create an optimized build for a production environment.
boolean: [
NamedArgs.ApplyLavaMoat,
NamedArgs.LintFenceFiles,
NamedArgs.Lockdown,
NamedArgs.PolicyOnly,
NamedArgs.SkipStats,
],
string: [NamedArgs.BuildType, NamedArgs.BuildVersion],
default: {
[NamedArgs.ApplyLavaMoat]: true,
[NamedArgs.BuildType]: BuildType.main,
[NamedArgs.BuildVersion]: '0',
[NamedArgs.LintFenceFiles]: true,
[NamedArgs.Lockdown]: true,
[NamedArgs.PolicyOnly]: false,
[NamedArgs.SkipStats]: false,
},
});
if (argv._.length !== 1) { dev: Create an unoptimized, live-reloading build for local development.
throw new Error(
`Metamask build: Expected a single positional argument, but received "${argv._.length}" arguments.`,
);
}
const entryTask = argv._[0]; test: Create an optimized build for running e2e tests.
if (!entryTask) {
throw new Error('MetaMask build: No entry task specified.');
}
const buildType = argv[NamedArgs.BuildType]; testDev: Create an unoptimized, live-reloading build for debugging e2e tests.`,
if (!(buildType in BuildType)) { type: 'string',
throw new Error(`MetaMask build: Invalid build type: "${buildType}"`); })
} .option('apply-lavamoat', {
default: true,
description:
'Whether to use LavaMoat. Setting this to `false` can be useful during development if you want to handle LavaMoat errors later.',
type: 'boolean',
})
.option('build-type', {
default: BuildType.main,
description: 'The type of build to create.',
choices: Object.keys(BuildType),
})
.option('build-version', {
default: 0,
description:
'The build version. This is set only for non-main build types. The build version is used in the "prerelease" segment of the extension version, e.g. `[major].[minor].[patch]-[build-type].[build-version]`',
type: 'number',
})
.option('lint-fence-files', {
description:
'Whether files with code fences should be linted after fences have been removed. The build will fail if linting fails. This defaults to `false` if the entry task is `dev` or `testDev`. Otherwise this defaults to `true`.',
type: 'boolean',
})
.option('lockdown', {
default: true,
description:
'Whether to include SES lockdown files in the extension bundle. Setting this to `false` can be useful during development if you want to handle lockdown errors later.',
type: 'boolean',
})
.option('policy-only', {
default: false,
description:
'Stop the build after generating the LavaMoat policy, skipping any writes to disk other than the LavaMoat policy itself.',
type: 'boolean',
})
.option('skip-stats', {
default: false,
description:
'Whether to skip logging the time to completion for each task to the console. This is meant primarily for internal use, to prevent duplicate logging.',
hidden: true,
type: 'boolean',
})
.check((args) => {
if (!Number.isInteger(args.buildVersion)) {
throw new Error(
`Expected integer for 'build-version', got '${args.buildVersion}'`,
);
} else if (!Object.values(TASKS).includes(args.task)) {
throw new Error(`Invalid task: '${args.task}'`);
}
return true;
}),
)
// TODO: Enable `.strict()` after this issue is resolved: https://github.com/LavaMoat/LavaMoat/issues/344
.help('help');
const rawBuildVersion = argv[NamedArgs.BuildVersion]; const {
const buildVersion = Number.parseInt(rawBuildVersion, 10); applyLavamoat: applyLavaMoat,
if (rawBuildVersion.match(/^\d+$/u) === null || Number.isNaN(buildVersion)) { buildType,
throw new Error( buildVersion,
`MetaMask build: Invalid build version: "${rawBuildVersion}"`, lintFenceFiles,
); lockdown,
} policyOnly,
skipStats,
task,
} = argv;
// Manually default this to `false` for dev builds only. // Manually default this to `false` for dev builds only.
const shouldLintFenceFiles = process.argv.includes( const shouldLintFenceFiles = lintFenceFiles ?? !/dev/iu.test(task);
`--${NamedArgs.LintFenceFiles}`,
)
? argv[NamedArgs.LintFenceFiles]
: !/dev/iu.test(entryTask);
const policyOnly = argv[NamedArgs.PolicyOnly];
const version = getVersion(buildType, buildVersion); const version = getVersion(buildType, buildVersion);
return { return {
// Should we apply LavaMoat to the build output? applyLavaMoat,
applyLavaMoat: argv[NamedArgs.ApplyLavaMoat],
buildType, buildType,
entryTask, entryTask: task,
// Is this process running in lavamoat-node?
isLavaMoat: process.argv[0].includes('lavamoat'), isLavaMoat: process.argv[0].includes('lavamoat'),
policyOnly, policyOnly,
shouldIncludeLockdown: argv[NamedArgs.Lockdown], shouldIncludeLockdown: lockdown,
shouldLintFenceFiles, shouldLintFenceFiles,
skipStats: argv[NamedArgs.SkipStats], skipStats,
version, version,
}; };
} }

@ -7,6 +7,7 @@ const baseManifest = process.env.ENABLE_MV3
: require('../../app/manifest/v2/_base.json'); : require('../../app/manifest/v2/_base.json');
const { BuildType } = require('../lib/build-type'); const { BuildType } = require('../lib/build-type');
const { TASKS } = require('./constants');
const { createTask, composeSeries } = require('./task'); const { createTask, composeSeries } = require('./task');
module.exports = createManifestTasks; module.exports = createManifestTasks;
@ -68,19 +69,22 @@ function createManifestTasks({
}); });
// high level manifest tasks // high level manifest tasks
const dev = createTask('manifest:dev', composeSeries(prepPlatforms, envDev)); const dev = createTask(
TASKS.MANIFEST_DEV,
composeSeries(prepPlatforms, envDev),
);
const testDev = createTask( const testDev = createTask(
'manifest:testDev', TASKS.MANIFEST_TEST_DEV,
composeSeries(prepPlatforms, envTestDev), composeSeries(prepPlatforms, envTestDev),
); );
const test = createTask( const test = createTask(
'manifest:test', TASKS.MANIFEST_TEST,
composeSeries(prepPlatforms, envTest), composeSeries(prepPlatforms, envTest),
); );
const prod = createTask('manifest:prod', prepPlatforms); const prod = createTask(TASKS.MANIFEST_PROD, prepPlatforms);
return { prod, dev, testDev, test }; return { prod, dev, testDev, test };

@ -6,6 +6,7 @@ const glob = require('fast-glob');
const locales = require('../../app/_locales/index.json'); const locales = require('../../app/_locales/index.json');
const { BuildType } = require('../lib/build-type'); const { BuildType } = require('../lib/build-type');
const { TASKS } = require('./constants');
const { createTask, composeSeries } = require('./task'); const { createTask, composeSeries } = require('./task');
const EMPTY_JS_FILE = './development/empty.js'; const EMPTY_JS_FILE = './development/empty.js';
@ -41,7 +42,7 @@ module.exports = function createStaticAssetTasks({
} }
const prod = createTask( const prod = createTask(
'static:prod', TASKS.STATIC_PROD,
composeSeries( composeSeries(
...copyTargetsProd.map((target) => { ...copyTargetsProd.map((target) => {
return async function copyStaticAssets() { return async function copyStaticAssets() {
@ -51,7 +52,7 @@ module.exports = function createStaticAssetTasks({
), ),
); );
const dev = createTask( const dev = createTask(
'static:dev', TASKS.STATIC_DEV,
composeSeries( composeSeries(
...copyTargetsDev.map((target) => { ...copyTargetsDev.map((target) => {
return async function copyStaticAssets() { return async function copyStaticAssets() {

@ -7,6 +7,7 @@ const sourcemaps = require('gulp-sourcemaps');
const rtlcss = require('gulp-rtlcss'); const rtlcss = require('gulp-rtlcss');
const rename = require('gulp-rename'); const rename = require('gulp-rename');
const pump = pify(require('pump')); const pump = pify(require('pump'));
const { TASKS } = require('./constants');
const { createTask } = require('./task'); const { createTask } = require('./task');
let sass; let sass;
@ -16,7 +17,7 @@ module.exports = createStyleTasks;
function createStyleTasks({ livereload }) { function createStyleTasks({ livereload }) {
const prod = createTask( const prod = createTask(
'styles:prod', TASKS.STYLES_PROD,
createScssBuildTask({ createScssBuildTask({
src: 'ui/css/index.scss', src: 'ui/css/index.scss',
dest: 'ui/css/output', dest: 'ui/css/output',
@ -25,7 +26,7 @@ function createStyleTasks({ livereload }) {
); );
const dev = createTask( const dev = createTask(
'styles:dev', TASKS.STYLES_DEV,
createScssBuildTask({ createScssBuildTask({
src: 'ui/css/index.scss', src: 'ui/css/index.scss',
dest: 'ui/css/output', dest: 'ui/css/output',
@ -34,7 +35,7 @@ function createStyleTasks({ livereload }) {
}), }),
); );
const lint = createTask('lint-scss', function () { const lint = createTask(TASKS.LINT_SCSS, function () {
return gulp.src('ui/css/itcss/**/*.scss').pipe( return gulp.src('ui/css/itcss/**/*.scss').pipe(
gulpStylelint({ gulpStylelint({
reporters: [{ formatter: 'string', console: true }], reporters: [{ formatter: 'string', console: true }],

@ -101,6 +101,36 @@
"globals": { "globals": {
"globalThis": true "globalThis": true
} }
},
"yargs": {
"builtin": {
"assert": true,
"fs": true,
"path": true,
"util": true
},
"globals": {
"console": true,
"Error": true,
"process": true
}
},
"yargs>y18n": {
"builtin": {
"fs": true,
"path": true,
"util": true
}
},
"yargs>yargs-parser": {
"builtin": {
"fs": true,
"path": true,
"util": true
},
"globals": {
"process": true
}
} }
} }
} }

@ -2364,7 +2364,7 @@
"packages": { "packages": {
"eslint-plugin-import>tsconfig-paths>json5": true, "eslint-plugin-import>tsconfig-paths>json5": true,
"eslint-plugin-import>tsconfig-paths>strip-bom": true, "eslint-plugin-import>tsconfig-paths>strip-bom": true,
"minimist": true "rc>minimist": true
} }
}, },
"eslint-plugin-import>tsconfig-paths>json5": { "eslint-plugin-import>tsconfig-paths>json5": {
@ -6624,9 +6624,9 @@
"process.platform": true "process.platform": true
}, },
"packages": { "packages": {
"minimist": true,
"rc>deep-extend": true, "rc>deep-extend": true,
"rc>ini": true, "rc>ini": true,
"rc>minimist": true,
"rc>strip-json-comments": true "rc>strip-json-comments": true
} }
}, },
@ -8393,12 +8393,82 @@
"enzyme>rst-selector-parser>nearley>randexp>ret": true "enzyme>rst-selector-parser>nearley>randexp>ret": true
} }
}, },
"yargs": {
"builtin": {
"assert": true,
"fs": true,
"path": true,
"util": true
},
"globals": {
"Error": true,
"console": true,
"process": true
},
"packages": {
"yargs>cliui": true,
"yargs>escalade": true,
"yargs>get-caller-file": true,
"yargs>require-directory": true,
"yargs>string-width": true,
"yargs>y18n": true,
"yargs>yargs-parser": true
}
},
"yargs>cliui": {
"packages": {
"eslint>strip-ansi": true,
"yargs>cliui>wrap-ansi": true,
"yargs>string-width": true
}
},
"yargs>cliui>wrap-ansi": {
"packages": {
"chalk>ansi-styles": true,
"eslint>strip-ansi": true,
"yargs>string-width": true
}
},
"yargs>escalade": {
"builtin": {
"fs.readdirSync": true,
"fs.statSync": true,
"path.dirname": true,
"path.resolve": true
}
},
"yargs>require-directory": {
"builtin": {
"fs.readdirSync": true,
"fs.statSync": true,
"path.dirname": true,
"path.join": true,
"path.resolve": true
}
},
"yargs>string-width": { "yargs>string-width": {
"packages": { "packages": {
"eslint>strip-ansi": true, "eslint>strip-ansi": true,
"yargs>string-width>emoji-regex": true, "yargs>string-width>emoji-regex": true,
"yargs>string-width>is-fullwidth-code-point": true "yargs>string-width>is-fullwidth-code-point": true
} }
},
"yargs>y18n": {
"builtin": {
"fs": true,
"path": true,
"util": true
}
},
"yargs>yargs-parser": {
"builtin": {
"fs": true,
"path": true,
"util": true
},
"globals": {
"process": true
}
} }
} }
} }

@ -335,7 +335,6 @@
"lavamoat-viz": "^6.0.9", "lavamoat-viz": "^6.0.9",
"lockfile-lint": "^4.0.0", "lockfile-lint": "^4.0.0",
"loose-envify": "^1.4.0", "loose-envify": "^1.4.0",
"minimist": "^1.2.6",
"mocha": "^7.2.0", "mocha": "^7.2.0",
"mockttp": "^2.6.0", "mockttp": "^2.6.0",
"nock": "^9.0.14", "nock": "^9.0.14",

File diff suppressed because one or more lines are too long
Loading…
Cancel
Save