From 479879694382cfde1ce811be9f37ddccbce55f24 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 24 Jan 2022 18:51:04 -0800
Subject: [PATCH 01/16] update

---
 lighthouse-cli/cli-flags.js                   | 16 +++++++++++----
 lighthouse-cli/test/cli/cli-flags-test.js     | 15 ++++++++++++--
 .../cli-flags-path-inline-budgets.json        | 11 ++++++++++
 lighthouse-core/runner.js                     |  9 +++++++++
 .../test/results/artifacts/artifacts.json     |  6 +-----
 lighthouse-core/test/results/sample_v2.json   |  6 +-----
 package.json                                  |  2 +-
 yarn.lock                                     | 20 ++++++++++++++++++-
 8 files changed, 67 insertions(+), 18 deletions(-)
 create mode 100644 lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json

diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index 57b29a3f57d2..23dfb96a57f2 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -365,7 +365,9 @@ function splitCommaSeparatedValues(strings) {
  * @return {boolean|string|undefined}
  */
 function coerceOptionalStringBoolean(value) {
-  if (typeof value !== 'undefined' && typeof value !== 'string' && typeof value !== 'boolean') {
+  if (value === undefined) return;
+
+  if (typeof value !== 'string' && typeof value !== 'boolean') {
     throw new Error('Invalid value: Argument must be a string or a boolean');
   }
   return value;
@@ -399,9 +401,11 @@ function coerceOutput(values) {
  * allowlist specific locales. Why? So we can support the user who requests 'es-MX' (unsupported)
  * and we'll fall back to 'es' (supported).
  * @param {unknown} value
- * @return {LH.Locale}
+ * @return {LH.Locale|undefined}
  */
 function coerceLocale(value) {
+  if (value === undefined) return;
+
   if (typeof value !== 'string') throw new Error(`Invalid value: Argument 'locale' must be a string`);
   return /** @type {LH.Locale} */ (value);
 }
@@ -431,9 +435,11 @@ function coerceExtraHeaders(value) {
 /**
  * Take yarg's unchecked object value and ensure it's proper throttling settings.
  * @param {unknown} value
- * @return {LH.ThrottlingSettings}
+ * @return {LH.ThrottlingSettings|undefined}
  */
 function coerceThrottling(value) {
+  if (value === undefined) return;
+
   if (!isObjectOfUnknownValues(value)) {
     throw new Error(`Invalid value: Argument 'throttling' must be an object, specified per-property ('throttling.rttMs', 'throttling.throughputKbps', etc)`);
   }
@@ -465,9 +471,11 @@ function coerceThrottling(value) {
 /**
  * Take yarg's unchecked object value and ensure it is a proper LH.screenEmulationSettings.
  * @param {unknown} value
- * @return {Partial<LH.ScreenEmulationSettings>}
+ * @return {Partial<LH.ScreenEmulationSettings>|undefined}
  */
 function coerceScreenEmulation(value) {
+  if (value === undefined) return;
+
   if (!isObjectOfUnknownValues(value)) {
     throw new Error(`Invalid value: Argument 'screenEmulation' must be an object, specified per-property ('screenEmulation.width', 'screenEmulation.deviceScaleFactor', etc)`);
   }
diff --git a/lighthouse-cli/test/cli/cli-flags-test.js b/lighthouse-cli/test/cli/cli-flags-test.js
index 4089d08e03b8..72ec287f5fee 100644
--- a/lighthouse-cli/test/cli/cli-flags-test.js
+++ b/lighthouse-cli/test/cli/cli-flags-test.js
@@ -26,8 +26,9 @@ describe('CLI flags', function() {
     Object.keys(optionGroups).forEach(key => {
       allOptions.push(...optionGroups[key]);
     });
-    // @ts-expect-error - getUsageInstance is private
-    const optionsWithDescriptions = Object.keys(yargs.getUsageInstance().getDescriptions());
+    const optionsWithDescriptions =
+      // @ts-expect-error - getUsageInstance is private
+      Object.keys(yargs.getInternalMethods().getUsageInstance().getDescriptions());
 
     allOptions.forEach(opt => {
       assert.ok(optionsWithDescriptions.includes(opt), `cli option '${opt}' has no description`);
@@ -54,6 +55,16 @@ describe('CLI flags', function() {
     });
   });
 
+  it('settings are accepted from a file path (inlined budgets)', () => {
+    const flags = getFlags([
+      'http://www.example.com',
+      // eslint-disable-next-line max-len
+      `--cli-flags-path="${LH_ROOT}/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json"`,
+    ].join(' '));
+
+    expect(flags.budgets).toMatchObject([{'anything': 'works'}]);
+  });
+
   it('array values support csv when appropriate', () => {
     const flags = getFlags([
       'http://www.example.com',
diff --git a/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json b/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json
new file mode 100644
index 000000000000..4ec1da214078
--- /dev/null
+++ b/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json
@@ -0,0 +1,11 @@
+{
+  "budgets": [{"anything": "works"}],
+  "onlyCategories": ["performance", "seo"],
+  "chromeFlags": "--window-size 800,600",
+  "extraHeaders": {"X-Men": "wolverine"},
+  "throttling-method": "devtools",
+  "throttling": {
+    "requestLatencyMs": 700,
+    "cpuSlowdownMultiplier": 6
+  }
+}
diff --git a/lighthouse-core/runner.js b/lighthouse-core/runner.js
index a7c1ab5b7455..155d9f299fe7 100644
--- a/lighthouse-core/runner.js
+++ b/lighthouse-core/runner.js
@@ -259,10 +259,19 @@ class Runner {
         output: undefined,
         channel: undefined,
         budgets: undefined,
+        skipAudits: undefined,
+        onlyAudits: undefined,
+        onlyCategories: undefined,
+        extraHeaders: undefined,
       };
       const normalizedGatherSettings = Object.assign({}, artifacts.settings, overrides);
       const normalizedAuditSettings = Object.assign({}, settings, overrides);
 
+      for (const k of Object.keys(normalizedGatherSettings)) {
+        if (!isDeepEqual(normalizedGatherSettings[k], normalizedAuditSettings[k])) {
+          throw new Error('Cannot change settings between gathering and auditing: ' + k);
+        }
+      }
       if (!isDeepEqual(normalizedGatherSettings, normalizedAuditSettings)) {
         throw new Error('Cannot change settings between gathering and auditing');
       }
diff --git a/lighthouse-core/test/results/artifacts/artifacts.json b/lighthouse-core/test/results/artifacts/artifacts.json
index f499c4d09c70..ead27d43b1b4 100644
--- a/lighthouse-core/test/results/artifacts/artifacts.json
+++ b/lighthouse-core/test/results/artifacts/artifacts.json
@@ -160,11 +160,7 @@
     "locale": "en-US",
     "blockedUrlPatterns": null,
     "additionalTraceCategories": null,
-    "extraHeaders": null,
-    "precomputedLanternData": null,
-    "onlyAudits": null,
-    "onlyCategories": null,
-    "skipAudits": null
+    "precomputedLanternData": null
   },
   "URL": {
     "requestedUrl": "http://localhost:10200/dobetterweb/dbw_tester.html",
diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
index c5d1c3b36bce..8a1f6a8795b2 100644
--- a/lighthouse-core/test/results/sample_v2.json
+++ b/lighthouse-core/test/results/sample_v2.json
@@ -5892,11 +5892,7 @@
     "locale": "en-US",
     "blockedUrlPatterns": null,
     "additionalTraceCategories": null,
-    "extraHeaders": null,
-    "precomputedLanternData": null,
-    "onlyAudits": null,
-    "onlyCategories": null,
-    "skipAudits": null
+    "precomputedLanternData": null
   },
   "categories": {
     "performance": {
diff --git a/package.json b/package.json
index ac70851e24e6..451f4f024b8c 100644
--- a/package.json
+++ b/package.json
@@ -212,7 +212,7 @@
     "third-party-web": "^0.12.7",
     "update-notifier": "^4.1.0",
     "ws": "^7.0.0",
-    "yargs": "^16.1.1",
+    "yargs": "^17.3.1",
     "yargs-parser": "^20.2.4"
   },
   "resolutions": {
diff --git a/yarn.lock b/yarn.lock
index 14828b8eaa53..4848bd1f5420 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7591,7 +7591,7 @@ string-length@^4.0.1:
     char-regex "^1.0.2"
     strip-ansi "^6.0.0"
 
-string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0:
+string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -8486,6 +8486,11 @@ yargs-parser@^20.0.0, yargs-parser@^20.2.2, yargs-parser@^20.2.4:
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54"
   integrity sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==
 
+yargs-parser@^21.0.0:
+  version "21.0.0"
+  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.0.0.tgz#a485d3966be4317426dd56bdb6a30131b281dc55"
+  integrity sha512-z9kApYUOCwoeZ78rfRYYWdiU/iNL6mwwYlkkZfJoyMR1xps+NEBX5X7XmRpxkZHhXJ6+Ey00IwKxBBSW9FIjyA==
+
 yargs@^15.0.0:
   version "15.4.1"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-15.4.1.tgz#0d87a16de01aee9d8bec2bfbf74f67851730f4f8"
@@ -8516,6 +8521,19 @@ yargs@^16.0.0, yargs@^16.0.3, yargs@^16.1.1, yargs@^16.2.0:
     y18n "^5.0.5"
     yargs-parser "^20.2.2"
 
+yargs@^17.3.1:
+  version "17.3.1"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.3.1.tgz#da56b28f32e2fd45aefb402ed9c26f42be4c07b9"
+  integrity sha512-WUANQeVgjLbNsEmGk20f+nlHgOqzRFpiGWVaBrYGYIGANIIu3lWjoyi0fNlFmJkvfhCZ6BXINe7/W2O2bV4iaA==
+  dependencies:
+    cliui "^7.0.2"
+    escalade "^3.1.1"
+    get-caller-file "^2.0.5"
+    require-directory "^2.1.1"
+    string-width "^4.2.3"
+    y18n "^5.0.5"
+    yargs-parser "^21.0.0"
+
 yauzl@^2.10.0:
   version "2.10.0"
   resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"

From 5d1f2ca8299a54bbe99a3b42589934fa115dffbd Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 24 Jan 2022 18:56:28 -0800
Subject: [PATCH 02/16] go back to being nully

---
 lighthouse-cli/cli-flags.js                           | 4 ++--
 lighthouse-core/test/results/artifacts/artifacts.json | 5 ++++-
 lighthouse-core/test/results/sample_v2.json           | 5 ++++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index 23dfb96a57f2..b34cb7ab057c 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -352,10 +352,10 @@ function getFlags(manualArgv, options = {}) {
 /**
  * Support comma-separated values for some array flags by splitting on any ',' found.
  * @param {Array<string>=} strings
- * @return {Array<string>|undefined}
+ * @return {Array<string>|null}
  */
 function splitCommaSeparatedValues(strings) {
-  if (!strings) return strings;
+  if (!strings) return null;
 
   return strings.flatMap(value => value.split(','));
 }
diff --git a/lighthouse-core/test/results/artifacts/artifacts.json b/lighthouse-core/test/results/artifacts/artifacts.json
index ead27d43b1b4..29a8b2d7983d 100644
--- a/lighthouse-core/test/results/artifacts/artifacts.json
+++ b/lighthouse-core/test/results/artifacts/artifacts.json
@@ -160,7 +160,10 @@
     "locale": "en-US",
     "blockedUrlPatterns": null,
     "additionalTraceCategories": null,
-    "precomputedLanternData": null
+    "precomputedLanternData": null,
+    "onlyAudits": null,
+    "onlyCategories": null,
+    "skipAudits": null
   },
   "URL": {
     "requestedUrl": "http://localhost:10200/dobetterweb/dbw_tester.html",
diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
index 8a1f6a8795b2..dd6b20d3cb6c 100644
--- a/lighthouse-core/test/results/sample_v2.json
+++ b/lighthouse-core/test/results/sample_v2.json
@@ -5892,7 +5892,10 @@
     "locale": "en-US",
     "blockedUrlPatterns": null,
     "additionalTraceCategories": null,
-    "precomputedLanternData": null
+    "precomputedLanternData": null,
+    "onlyAudits": null,
+    "onlyCategories": null,
+    "skipAudits": null
   },
   "categories": {
     "performance": {

From 544e0cbfd24e43ce6fbc79be7808f120f426c89b Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 24 Jan 2022 19:13:44 -0800
Subject: [PATCH 03/16] update types

---
 lighthouse-cli/cli-flags.js                                | 2 +-
 lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js | 3 ++-
 lighthouse-core/scripts/compare-runs.js                    | 3 ++-
 lighthouse-core/scripts/pptr-run-devtools.js               | 3 ++-
 package.json                                               | 2 +-
 yarn.lock                                                  | 7 +++++++
 6 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index b34cb7ab057c..1caa3f36b9b1 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -343,7 +343,7 @@ function getFlags(manualArgv, options = {}) {
 
   // Augmenting yargs type with auto-camelCasing breaks in tsc@4.1.2 and @types/yargs@15.0.11,
   // so for now cast to add yarg's camelCase properties to type.
-  const argv = parser.argv;
+  const argv = /** @type {Awaited<typeof parser.argv>} */ (parser.argv);
   const cliFlags = /** @type {typeof argv & CamelCasify<typeof argv>} */ (argv);
 
   return cliFlags;
diff --git a/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js b/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js
index 1130a946b80d..19b5dcc25446 100644
--- a/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js
+++ b/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js
@@ -175,7 +175,8 @@ async function begin() {
 
   // Augmenting yargs type with auto-camelCasing breaks in tsc@4.1.2 and @types/yargs@15.0.11,
   // so for now cast to add yarg's camelCase properties to type.
-  const argv = /** @type {typeof rawArgv & CamelCasify<typeof rawArgv>} */ (rawArgv);
+  const argv =
+    /** @type {Awaited<typeof rawArgv> & CamelCasify<Awaited<typeof rawArgv>>} */ (rawArgv);
 
   const jobs = Number.isFinite(argv.jobs) ? argv.jobs : undefined;
   const retries = Number.isFinite(argv.retries) ? argv.retries : undefined;
diff --git a/lighthouse-core/scripts/compare-runs.js b/lighthouse-core/scripts/compare-runs.js
index 710f2b6608f3..300913d83575 100644
--- a/lighthouse-core/scripts/compare-runs.js
+++ b/lighthouse-core/scripts/compare-runs.js
@@ -74,7 +74,8 @@ const rawArgv = y
 
 // Augmenting yargs type with auto-camelCasing breaks in tsc@4.1.2 and @types/yargs@15.0.11,
 // so for now cast to add yarg's camelCase properties to type.
-const argv = /** @type {typeof rawArgv & CamelCasify<typeof rawArgv>} */ (rawArgv);
+const argv =
+  /** @type {Awaited<typeof rawArgv> & CamelCasify<Awaited<typeof rawArgv>>} */ (rawArgv);
 
 const reportExcludeRegex =
   argv.reportExclude !== 'none' ? new RegExp(argv.reportExclude, 'i') : null;
diff --git a/lighthouse-core/scripts/pptr-run-devtools.js b/lighthouse-core/scripts/pptr-run-devtools.js
index 50f23c17ed83..471f4b707cc7 100644
--- a/lighthouse-core/scripts/pptr-run-devtools.js
+++ b/lighthouse-core/scripts/pptr-run-devtools.js
@@ -31,7 +31,7 @@ import yargs from 'yargs';
 import * as yargsHelpers from 'yargs/helpers';
 
 const y = yargs(yargsHelpers.hideBin(process.argv));
-const argv = y
+const argv_ = y
   .usage('$0 [url]')
   .help('help').alias('help', 'h')
   .option('_', {type: 'string'})
@@ -50,6 +50,7 @@ const argv = y
   })
   .argv;
 
+const argv = /** @type {Awaited<typeof argv_>} */ (argv_);
 /** @type {LH.Config.Json=} */
 const config = argv.config ? JSON.parse(argv.config) : undefined;
 
diff --git a/package.json b/package.json
index 451f4f024b8c..0779cb4ce5f3 100644
--- a/package.json
+++ b/package.json
@@ -127,7 +127,7 @@
     "@types/tabulator-tables": "^4.9.1",
     "@types/update-notifier": "^4.1.0",
     "@types/ws": "^7.0.0",
-    "@types/yargs": "^16.0.0",
+    "@types/yargs": "^17.0.8",
     "@types/yargs-parser": "^15.0.0",
     "@typescript-eslint/eslint-plugin": "^5.6.0",
     "@typescript-eslint/parser": "^5.6.0",
diff --git a/yarn.lock b/yarn.lock
index 4848bd1f5420..6aeef6f6fe69 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1741,6 +1741,13 @@
   dependencies:
     "@types/yargs-parser" "*"
 
+"@types/yargs@^17.0.8":
+  version "17.0.8"
+  resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.8.tgz#d23a3476fd3da8a0ea44b5494ca7fa677b9dad4c"
+  integrity sha512-wDeUwiUmem9FzsyysEwRukaEdDNcwbROvQ9QGRKaLI6t+IltNzbn4/i4asmB10auvZGQCzSQ6t0GSczEThlUXw==
+  dependencies:
+    "@types/yargs-parser" "*"
+
 "@types/yauzl@^2.9.1":
   version "2.9.1"
   resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.9.1.tgz#d10f69f9f522eef3cf98e30afb684a1e1ec923af"

From c4e8df47dc78f2335e5dfb843e536cf1ea80b5b3 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 10:27:54 -0800
Subject: [PATCH 04/16] tweak

---
 .../test/fixtures/cli-flags-path-inline-budgets.json   | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json b/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json
index 4ec1da214078..a5bf83dae60c 100644
--- a/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json
+++ b/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json
@@ -1,11 +1,3 @@
 {
-  "budgets": [{"anything": "works"}],
-  "onlyCategories": ["performance", "seo"],
-  "chromeFlags": "--window-size 800,600",
-  "extraHeaders": {"X-Men": "wolverine"},
-  "throttling-method": "devtools",
-  "throttling": {
-    "requestLatencyMs": 700,
-    "cpuSlowdownMultiplier": 6
-  }
+  "budgets": [{"anything": "works"}]
 }

From 3cd1441145ca766fa4f59e0cef9161d30e50ba04 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 11:00:12 -0800
Subject: [PATCH 05/16] tweak extraHeaders coerce

---
 lighthouse-cli/cli-flags.js                 | 2 +-
 lighthouse-core/test/results/sample_v2.json | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index 1caa3f36b9b1..368f79fca55f 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -418,7 +418,7 @@ function coerceLocale(value) {
  */
 function coerceExtraHeaders(value) {
   // TODO: this function does not actually verify the object type.
-  if (value === undefined) return value;
+  if (value === undefined) return null;
   if (typeof value === 'object') return /** @type {LH.SharedFlagsSettings['extraHeaders']} */ (value);
   if (typeof value !== 'string') {
     throw new Error(`Invalid value: Argument 'extra-headers' must be a string`);
diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
index dd6b20d3cb6c..c5d1c3b36bce 100644
--- a/lighthouse-core/test/results/sample_v2.json
+++ b/lighthouse-core/test/results/sample_v2.json
@@ -5892,6 +5892,7 @@
     "locale": "en-US",
     "blockedUrlPatterns": null,
     "additionalTraceCategories": null,
+    "extraHeaders": null,
     "precomputedLanternData": null,
     "onlyAudits": null,
     "onlyCategories": null,

From c23ea16794dd8faa92b26cf2953d47d8b639ecbf Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 13:20:39 -0800
Subject: [PATCH 06/16] comments

---
 lighthouse-core/runner.js | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/lighthouse-core/runner.js b/lighthouse-core/runner.js
index 155d9f299fe7..cc28b52fef23 100644
--- a/lighthouse-core/runner.js
+++ b/lighthouse-core/runner.js
@@ -267,11 +267,19 @@ class Runner {
       const normalizedGatherSettings = Object.assign({}, artifacts.settings, overrides);
       const normalizedAuditSettings = Object.assign({}, settings, overrides);
 
-      for (const k of Object.keys(normalizedGatherSettings)) {
+      // First, try each key individually so we can print which key differed.
+      const keys = new Set([
+        ...Object.keys(normalizedGatherSettings),
+        ...Object.keys(normalizedAuditSettings),
+      ]);
+      for (const k of keys) {
         if (!isDeepEqual(normalizedGatherSettings[k], normalizedAuditSettings[k])) {
-          throw new Error('Cannot change settings between gathering and auditing: ' + k);
+          throw new Error(
+            `Cannot change settings between gathering and auditing. Difference found at: ${k}`);
         }
       }
+
+      // Call `isDeepEqual` on the entire thing, just in case something was missed.
       if (!isDeepEqual(normalizedGatherSettings, normalizedAuditSettings)) {
         throw new Error('Cannot change settings between gathering and auditing');
       }

From 2324848debfe99d8c7fd0657dc041875fdb69e66 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 13:56:54 -0800
Subject: [PATCH 07/16] tests: add snapshot tests for cli-flags.js

---
 .../cli/__snapshots__/cli-flags-test.js.snap  | 458 ++++++++++++++++++
 lighthouse-cli/test/cli/cli-flags-test.js     |   9 +
 2 files changed, 467 insertions(+)
 create mode 100644 lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap

diff --git a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
new file mode 100644
index 000000000000..8b9202285fc5
--- /dev/null
+++ b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
@@ -0,0 +1,458 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`CLI flags array values do not support csv when appropriate 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "blocked-url-patterns": Array [
+    ".*x,y\\\\.png",
+  ],
+  "blockedUrlPatterns": Array [
+    ".*x,y\\\\.png",
+  ],
+  "channel": "cli",
+  "chrome-flags": Array [
+    "--window-size 800,600",
+    "--enabled-features=NetworkService,VirtualTime",
+  ],
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": Array [
+    "--window-size 800,600",
+    "--enabled-features=NetworkService,VirtualTime",
+  ],
+  "chromeIgnoreDefaultFlags": false,
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "verbose": false,
+  "view": false,
+}
+`;
+
+exports[`CLI flags array values support csv when appropriate 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "channel": "cli",
+  "chrome-flags": "",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "",
+  "chromeIgnoreDefaultFlags": false,
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "only-categories": Array [
+    "performance",
+    "seo",
+  ],
+  "onlyCategories": Array [
+    "performance",
+    "seo",
+  ],
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "skip-audits": Array [
+    "unused-javascript",
+    "redirects",
+    "bootup-time",
+  ],
+  "skipAudits": Array [
+    "unused-javascript",
+    "redirects",
+    "bootup-time",
+  ],
+  "verbose": false,
+  "view": false,
+}
+`;
+
+exports[`CLI flags extraHeaders should convert extra headers to object 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "channel": "cli",
+  "chrome-flags": "",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "",
+  "chromeIgnoreDefaultFlags": false,
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "extra-headers": Object {
+    "foo": "bar",
+  },
+  "extraHeaders": Object {
+    "foo": "bar",
+  },
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "verbose": false,
+  "view": false,
+}
+`;
+
+exports[`CLI flags extraHeaders should read extra headers from file 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "channel": "cli",
+  "chrome-flags": "",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "",
+  "chromeIgnoreDefaultFlags": false,
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "extra-headers": Object {
+    "Cookie": "monster=blue",
+    "x-men": "wolverine",
+  },
+  "extraHeaders": Object {
+    "Cookie": "monster=blue",
+    "x-men": "wolverine",
+  },
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "verbose": false,
+  "view": false,
+}
+`;
+
+exports[`CLI flags screenEmulation deviceScaleFactor parses a non-integer numeric value 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "channel": "cli",
+  "chrome-flags": "",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "",
+  "chromeIgnoreDefaultFlags": false,
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "screen-emulation": Object {
+    "deviceScaleFactor": 1.325,
+    "disabled": undefined,
+    "height": undefined,
+    "mobile": undefined,
+    "width": undefined,
+  },
+  "screenEmulation": Object {
+    "deviceScaleFactor": 1.325,
+    "disabled": undefined,
+    "height": undefined,
+    "mobile": undefined,
+    "width": undefined,
+  },
+  "verbose": false,
+  "view": false,
+}
+`;
+
+exports[`CLI flags screenEmulation disabled parses the flag with no value as true 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "channel": "cli",
+  "chrome-flags": "",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "",
+  "chromeIgnoreDefaultFlags": false,
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "screen-emulation": Object {
+    "deviceScaleFactor": undefined,
+    "disabled": true,
+    "height": undefined,
+    "mobile": undefined,
+    "width": undefined,
+  },
+  "screenEmulation": Object {
+    "deviceScaleFactor": undefined,
+    "disabled": true,
+    "height": undefined,
+    "mobile": undefined,
+    "width": undefined,
+  },
+  "verbose": false,
+  "view": false,
+}
+`;
+
+exports[`CLI flags screenEmulation mobile parses the flag with no value as true 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "channel": "cli",
+  "chrome-flags": "",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "",
+  "chromeIgnoreDefaultFlags": false,
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "screen-emulation": Object {
+    "deviceScaleFactor": undefined,
+    "disabled": undefined,
+    "height": undefined,
+    "mobile": true,
+    "width": undefined,
+  },
+  "screenEmulation": Object {
+    "deviceScaleFactor": undefined,
+    "disabled": undefined,
+    "height": undefined,
+    "mobile": true,
+    "width": undefined,
+  },
+  "verbose": false,
+  "view": false,
+}
+`;
+
+exports[`CLI flags screenEmulation width parses a number value 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "channel": "cli",
+  "chrome-flags": "",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "",
+  "chromeIgnoreDefaultFlags": false,
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "screen-emulation": Object {
+    "deviceScaleFactor": undefined,
+    "disabled": undefined,
+    "height": undefined,
+    "mobile": undefined,
+    "width": 500,
+  },
+  "screenEmulation": Object {
+    "deviceScaleFactor": undefined,
+    "disabled": undefined,
+    "height": undefined,
+    "mobile": undefined,
+    "width": 500,
+  },
+  "verbose": false,
+  "view": false,
+}
+`;
+
+exports[`CLI flags settings are accepted from a file path 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "budget-path": "path/to/my/budget-from-config.json",
+  "budgetPath": "path/to/my/budget-from-config.json",
+  "budgets-path": "path/to/my/budget-from-command-line.json",
+  "budgetsPath": "path/to/my/budget-from-command-line.json",
+  "channel": "cli",
+  "chrome-flags": "--window-size 800,600",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "--window-size 800,600",
+  "chromeIgnoreDefaultFlags": false,
+  "cli-flags-path": "/Users/cjamcl/src/lighthouse/lighthouse-cli/test/fixtures/cli-flags-path.json",
+  "cliFlagsPath": "/Users/cjamcl/src/lighthouse/lighthouse-cli/test/fixtures/cli-flags-path.json",
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "extra-headers": Object {
+    "X-Men": "wolverine",
+  },
+  "extraHeaders": Object {
+    "X-Men": "wolverine",
+  },
+  "extraheaders": Object {
+    "xMen": "wolverine",
+  },
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "only-categories": Array [
+    "performance",
+    "seo",
+  ],
+  "onlyCategories": Array [
+    "performance",
+    "seo",
+  ],
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "throttling": Object {
+    "cpuSlowdownMultiplier": 6,
+    "downloadThroughputKbps": undefined,
+    "requestLatencyMs": 700,
+    "rttMs": undefined,
+    "throughputKbps": undefined,
+    "uploadThroughputKbps": undefined,
+  },
+  "throttling-method": "devtools",
+  "throttlingMethod": "devtools",
+  "verbose": false,
+  "view": false,
+}
+`;
diff --git a/lighthouse-cli/test/cli/cli-flags-test.js b/lighthouse-cli/test/cli/cli-flags-test.js
index 4089d08e03b8..174cf9b1c100 100644
--- a/lighthouse-cli/test/cli/cli-flags-test.js
+++ b/lighthouse-cli/test/cli/cli-flags-test.js
@@ -52,6 +52,7 @@ describe('CLI flags', function() {
         cpuSlowdownMultiplier: 6,
       },
     });
+    expect(flags).toMatchSnapshot();
   });
 
   it('array values support csv when appropriate', () => {
@@ -63,6 +64,7 @@ describe('CLI flags', function() {
     ].join(' '));
     expect(flags.onlyCategories).toEqual(['performance', 'seo']);
     expect(flags.skipAudits).toEqual(['unused-javascript', 'redirects', 'bootup-time']);
+    expect(flags).toMatchSnapshot();
   });
 
   it('array values do not support csv when appropriate', () => {
@@ -77,6 +79,7 @@ describe('CLI flags', function() {
       '--enabled-features=NetworkService,VirtualTime',
     ]);
     expect(flags.blockedUrlPatterns).toEqual(['.*x,y\\.png']);
+    expect(flags).toMatchSnapshot();
   });
 
   describe('extraHeaders', () => {
@@ -87,6 +90,7 @@ describe('CLI flags', function() {
       ].join(' '));
 
       expect(flags).toHaveProperty('extraHeaders', {foo: 'bar'});
+      expect(flags).toMatchSnapshot();
     });
 
     it('should read extra headers from file', () => {
@@ -98,6 +102,7 @@ describe('CLI flags', function() {
       ].join(' '));
 
       expect(flags).toHaveProperty('extraHeaders', headers);
+      expect(flags).toMatchSnapshot();
     });
   });
 
@@ -108,6 +113,7 @@ describe('CLI flags', function() {
       it('parses a number value', () => {
         const flags = getFlags(`${url} --screenEmulation.width=500`, {noExitOnFailure: true});
         expect(flags.screenEmulation).toEqual({width: 500});
+        expect(flags).toMatchSnapshot();
       });
 
       it('throws on a non-number', () => {
@@ -143,6 +149,7 @@ describe('CLI flags', function() {
         const flags = getFlags(`${url} --screenEmulation.deviceScaleFactor=1.325`,
             {noExitOnFailure: true});
         expect(flags.screenEmulation).toEqual({deviceScaleFactor: 1.325});
+        expect(flags).toMatchSnapshot();
       });
 
       it('throws on a non-number', () => {
@@ -162,6 +169,7 @@ describe('CLI flags', function() {
       it('parses the flag with no value as true', () => {
         const flags = getFlags(`${url} --screenEmulation.mobile`, {noExitOnFailure: true});
         expect(flags.screenEmulation).toEqual({mobile: true});
+        expect(flags).toMatchSnapshot();
       });
 
       it('parses the --no-mobile flag as false', () => {
@@ -187,6 +195,7 @@ describe('CLI flags', function() {
       it('parses the flag with no value as true', () => {
         const flags = getFlags(`${url} --screenEmulation.disabled`, {noExitOnFailure: true});
         expect(flags.screenEmulation).toEqual({disabled: true});
+        expect(flags).toMatchSnapshot();
       });
 
       it('parses the --no-disabled flag as false', () => {

From e076404116f6f7a5e1a51ea57f4cddaaca5b1dc1 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 14:22:13 -0800
Subject: [PATCH 08/16] normalize

---
 .../cli/__snapshots__/cli-flags-test.js.snap  |  4 +--
 lighthouse-cli/test/cli/cli-flags-test.js     | 36 ++++++++++++++-----
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
index 8b9202285fc5..d5a947238b75 100644
--- a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
+++ b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
@@ -403,8 +403,8 @@ Object {
   "chrome-ignore-default-flags": false,
   "chromeFlags": "--window-size 800,600",
   "chromeIgnoreDefaultFlags": false,
-  "cli-flags-path": "/Users/cjamcl/src/lighthouse/lighthouse-cli/test/fixtures/cli-flags-path.json",
-  "cliFlagsPath": "/Users/cjamcl/src/lighthouse/lighthouse-cli/test/fixtures/cli-flags-path.json",
+  "cli-flags-path": "__REPLACED__/lighthouse-cli/test/fixtures/cli-flags-path.json",
+  "cliFlagsPath": "__REPLACED__/lighthouse-cli/test/fixtures/cli-flags-path.json",
   "enable-error-reporting": undefined,
   "enableErrorReporting": undefined,
   "extra-headers": Object {
diff --git a/lighthouse-cli/test/cli/cli-flags-test.js b/lighthouse-cli/test/cli/cli-flags-test.js
index 174cf9b1c100..a15942e5b910 100644
--- a/lighthouse-cli/test/cli/cli-flags-test.js
+++ b/lighthouse-cli/test/cli/cli-flags-test.js
@@ -15,6 +15,24 @@ import yargs from 'yargs';
 import {getFlags} from '../../cli-flags.js';
 import {LH_ROOT} from '../../../root.js';
 
+/**
+ * @param {LH.CliFlags} flags
+ */
+function snapshot(flags) {
+  flags = {...flags};
+
+  // `path` properties will have different values based on the filesystem,
+  // so normalize.
+  for (const [k, v] of Object.entries(flags)) {
+    if (typeof v === 'string') {
+      // @ts-expect-error
+      flags[k] = v.replace(process.cwd(), '__REPLACED__');
+    }
+  }
+
+  expect(flags).toMatchSnapshot();
+}
+
 describe('CLI flags', function() {
   it('all options should have descriptions', () => {
     getFlags('chrome://version');
@@ -52,7 +70,7 @@ describe('CLI flags', function() {
         cpuSlowdownMultiplier: 6,
       },
     });
-    expect(flags).toMatchSnapshot();
+    snapshot(flags);
   });
 
   it('array values support csv when appropriate', () => {
@@ -64,7 +82,7 @@ describe('CLI flags', function() {
     ].join(' '));
     expect(flags.onlyCategories).toEqual(['performance', 'seo']);
     expect(flags.skipAudits).toEqual(['unused-javascript', 'redirects', 'bootup-time']);
-    expect(flags).toMatchSnapshot();
+    snapshot(flags);
   });
 
   it('array values do not support csv when appropriate', () => {
@@ -79,7 +97,7 @@ describe('CLI flags', function() {
       '--enabled-features=NetworkService,VirtualTime',
     ]);
     expect(flags.blockedUrlPatterns).toEqual(['.*x,y\\.png']);
-    expect(flags).toMatchSnapshot();
+    snapshot(flags);
   });
 
   describe('extraHeaders', () => {
@@ -90,7 +108,7 @@ describe('CLI flags', function() {
       ].join(' '));
 
       expect(flags).toHaveProperty('extraHeaders', {foo: 'bar'});
-      expect(flags).toMatchSnapshot();
+      snapshot(flags);
     });
 
     it('should read extra headers from file', () => {
@@ -102,7 +120,7 @@ describe('CLI flags', function() {
       ].join(' '));
 
       expect(flags).toHaveProperty('extraHeaders', headers);
-      expect(flags).toMatchSnapshot();
+      snapshot(flags);
     });
   });
 
@@ -113,7 +131,7 @@ describe('CLI flags', function() {
       it('parses a number value', () => {
         const flags = getFlags(`${url} --screenEmulation.width=500`, {noExitOnFailure: true});
         expect(flags.screenEmulation).toEqual({width: 500});
-        expect(flags).toMatchSnapshot();
+        snapshot(flags);
       });
 
       it('throws on a non-number', () => {
@@ -149,7 +167,7 @@ describe('CLI flags', function() {
         const flags = getFlags(`${url} --screenEmulation.deviceScaleFactor=1.325`,
             {noExitOnFailure: true});
         expect(flags.screenEmulation).toEqual({deviceScaleFactor: 1.325});
-        expect(flags).toMatchSnapshot();
+        snapshot(flags);
       });
 
       it('throws on a non-number', () => {
@@ -169,7 +187,7 @@ describe('CLI flags', function() {
       it('parses the flag with no value as true', () => {
         const flags = getFlags(`${url} --screenEmulation.mobile`, {noExitOnFailure: true});
         expect(flags.screenEmulation).toEqual({mobile: true});
-        expect(flags).toMatchSnapshot();
+        snapshot(flags);
       });
 
       it('parses the --no-mobile flag as false', () => {
@@ -195,7 +213,7 @@ describe('CLI flags', function() {
       it('parses the flag with no value as true', () => {
         const flags = getFlags(`${url} --screenEmulation.disabled`, {noExitOnFailure: true});
         expect(flags.screenEmulation).toEqual({disabled: true});
-        expect(flags).toMatchSnapshot();
+        snapshot(flags);
       });
 
       it('parses the --no-disabled flag as false', () => {

From 994c974c6d6563265066b864dc6353abae746a5b Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 14:55:10 -0800
Subject: [PATCH 09/16] slashes

---
 lighthouse-cli/test/cli/cli-flags-test.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lighthouse-cli/test/cli/cli-flags-test.js b/lighthouse-cli/test/cli/cli-flags-test.js
index a15942e5b910..bd54c09785e8 100644
--- a/lighthouse-cli/test/cli/cli-flags-test.js
+++ b/lighthouse-cli/test/cli/cli-flags-test.js
@@ -26,7 +26,9 @@ function snapshot(flags) {
   for (const [k, v] of Object.entries(flags)) {
     if (typeof v === 'string') {
       // @ts-expect-error
-      flags[k] = v.replace(process.cwd(), '__REPLACED__');
+      flags[k] = v
+        .replace(process.cwd(), '__REPLACED__')
+        .replace('\\', '/');
     }
   }
 

From 90a07d11cf1795c150ef301cad6b704bdaa44347 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 15:05:15 -0800
Subject: [PATCH 10/16] update

---
 lighthouse-cli/cli-flags.js                       | 15 ++++++++++++---
 .../test/cli/__snapshots__/cli-flags-test.js.snap |  3 ---
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index 368f79fca55f..49739835a380 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -346,16 +346,25 @@ function getFlags(manualArgv, options = {}) {
   const argv = /** @type {Awaited<typeof parser.argv>} */ (parser.argv);
   const cliFlags = /** @type {typeof argv & CamelCasify<typeof argv>} */ (argv);
 
+  // yargs will return `undefined` for options that have a `coerce` function but
+  // are not actually present in the user input. Instead of passing properties
+  // explicitly set to undefined, delete them from the flags object.
+  for (const [k, v] of Object.entries(cliFlags)) {
+    // This property is meant to possibly be explicitly undefined.
+    if (k === 'enable-error-reporting' || k === 'enableErrorReporting') continue;
+    if (v === undefined) delete cliFlags[k];
+  }
+
   return cliFlags;
 }
 
 /**
  * Support comma-separated values for some array flags by splitting on any ',' found.
  * @param {Array<string>=} strings
- * @return {Array<string>|null}
+ * @return {Array<string>=}
  */
 function splitCommaSeparatedValues(strings) {
-  if (!strings) return null;
+  if (!strings) return undefined;
 
   return strings.flatMap(value => value.split(','));
 }
@@ -418,7 +427,7 @@ function coerceLocale(value) {
  */
 function coerceExtraHeaders(value) {
   // TODO: this function does not actually verify the object type.
-  if (value === undefined) return null;
+  if (value === undefined) return value;
   if (typeof value === 'object') return /** @type {LH.SharedFlagsSettings['extraHeaders']} */ (value);
   if (typeof value !== 'string') {
     throw new Error(`Invalid value: Argument 'extra-headers' must be a string`);
diff --git a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
index d5a947238b75..b49bfaa7f341 100644
--- a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
+++ b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
@@ -413,9 +413,6 @@ Object {
   "extraHeaders": Object {
     "X-Men": "wolverine",
   },
-  "extraheaders": Object {
-    "xMen": "wolverine",
-  },
   "fraggle-rock": false,
   "fraggleRock": false,
   "hostname": "127.0.0.1",

From d288da16d6a155a43445f48081a3b86e86ab68d2 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 15:07:07 -0800
Subject: [PATCH 11/16] update

---
 lighthouse-core/test/results/artifacts/artifacts.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lighthouse-core/test/results/artifacts/artifacts.json b/lighthouse-core/test/results/artifacts/artifacts.json
index 6e587b6f0d09..7dbf2543022e 100644
--- a/lighthouse-core/test/results/artifacts/artifacts.json
+++ b/lighthouse-core/test/results/artifacts/artifacts.json
@@ -160,6 +160,7 @@
     "locale": "en-US",
     "blockedUrlPatterns": null,
     "additionalTraceCategories": null,
+    "extraHeaders": null,
     "precomputedLanternData": null,
     "onlyAudits": null,
     "onlyCategories": null,

From a5ce3903cd3a78eedcc479227b7546d258693ecf Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 15:35:03 -0800
Subject: [PATCH 12/16] g

---
 lighthouse-cli/test/cli/cli-flags-test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lighthouse-cli/test/cli/cli-flags-test.js b/lighthouse-cli/test/cli/cli-flags-test.js
index bd54c09785e8..832019f29e95 100644
--- a/lighthouse-cli/test/cli/cli-flags-test.js
+++ b/lighthouse-cli/test/cli/cli-flags-test.js
@@ -28,7 +28,7 @@ function snapshot(flags) {
       // @ts-expect-error
       flags[k] = v
         .replace(process.cwd(), '__REPLACED__')
-        .replace('\\', '/');
+        .replace(/\\/g, '/');
     }
   }
 

From 80ed3e74db2ac4e797a88f4d355d69027e0823c0 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 16:32:44 -0800
Subject: [PATCH 13/16] snap

---
 .../cli/__snapshots__/cli-flags-test.js.snap  | 43 +++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
index b49bfaa7f341..7b8dd8e6bcf9 100644
--- a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
+++ b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
@@ -388,6 +388,49 @@ Object {
 }
 `;
 
+exports[`CLI flags settings are accepted from a file path (inlined budgets) 1`] = `
+Object {
+  "$0": "node_modules/jest/bin/jest.js",
+  "_": Array [
+    "http://www.example.com",
+  ],
+  "budgets": Array [
+    Object {
+      "anything": "works",
+    },
+  ],
+  "channel": "cli",
+  "chrome-flags": "",
+  "chrome-ignore-default-flags": false,
+  "chromeFlags": "",
+  "chromeIgnoreDefaultFlags": false,
+  "cli-flags-path": "__REPLACED__/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json",
+  "cliFlagsPath": "__REPLACED__/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json",
+  "enable-error-reporting": undefined,
+  "enableErrorReporting": undefined,
+  "fraggle-rock": false,
+  "fraggleRock": false,
+  "hostname": "127.0.0.1",
+  "list-all-audits": false,
+  "list-locales": false,
+  "list-trace-categories": false,
+  "listAllAudits": false,
+  "listLocales": false,
+  "listTraceCategories": false,
+  "output": Array [
+    "html",
+  ],
+  "port": 0,
+  "print-config": false,
+  "printConfig": false,
+  "quiet": false,
+  "save-assets": false,
+  "saveAssets": false,
+  "verbose": false,
+  "view": false,
+}
+`;
+
 exports[`CLI flags settings are accepted from a file path 1`] = `
 Object {
   "$0": "node_modules/jest/bin/jest.js",

From 64fce54bba64a542c8088690699ef0563d8c822a Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Tue, 25 Jan 2022 16:34:58 -0800
Subject: [PATCH 14/16] dont need those

---
 lighthouse-core/runner.js | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lighthouse-core/runner.js b/lighthouse-core/runner.js
index cc28b52fef23..9c597ad78024 100644
--- a/lighthouse-core/runner.js
+++ b/lighthouse-core/runner.js
@@ -259,10 +259,6 @@ class Runner {
         output: undefined,
         channel: undefined,
         budgets: undefined,
-        skipAudits: undefined,
-        onlyAudits: undefined,
-        onlyCategories: undefined,
-        extraHeaders: undefined,
       };
       const normalizedGatherSettings = Object.assign({}, artifacts.settings, overrides);
       const normalizedAuditSettings = Object.assign({}, settings, overrides);

From 08878c07bc8d2b4441c57307cb4ebb195ced3044 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Wed, 26 Jan 2022 12:29:46 -0800
Subject: [PATCH 15/16] tweak

---
 lighthouse-cli/cli-flags.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index 49739835a380..e477ca70d443 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -364,7 +364,7 @@ function getFlags(manualArgv, options = {}) {
  * @return {Array<string>=}
  */
 function splitCommaSeparatedValues(strings) {
-  if (!strings) return undefined;
+  if (!strings) return;
 
   return strings.flatMap(value => value.split(','));
 }

From 5c35f443b52c89f3cb08678a4772fc3c70237110 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Wed, 26 Jan 2022 13:03:17 -0800
Subject: [PATCH 16/16] update

---
 lighthouse-cli/cli-flags.js                   |  3 ---
 .../cli/__snapshots__/cli-flags-test.js.snap  | 20 -------------------
 2 files changed, 23 deletions(-)

diff --git a/lighthouse-cli/cli-flags.js b/lighthouse-cli/cli-flags.js
index e477ca70d443..90087541fcc4 100644
--- a/lighthouse-cli/cli-flags.js
+++ b/lighthouse-cli/cli-flags.js
@@ -178,7 +178,6 @@ function getFlags(manualArgv, options = {}) {
         },
         'enable-error-reporting': {
           type: 'boolean',
-          default: undefined, // Explicitly `undefined` so prompted by default.
           describe: 'Enables error reporting, overriding any saved preference. --no-enable-error-reporting will do the opposite. More: https://git.io/vFFTO',
         },
         'gather-mode': {
@@ -350,8 +349,6 @@ function getFlags(manualArgv, options = {}) {
   // are not actually present in the user input. Instead of passing properties
   // explicitly set to undefined, delete them from the flags object.
   for (const [k, v] of Object.entries(cliFlags)) {
-    // This property is meant to possibly be explicitly undefined.
-    if (k === 'enable-error-reporting' || k === 'enableErrorReporting') continue;
     if (v === undefined) delete cliFlags[k];
   }
 
diff --git a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
index 7b8dd8e6bcf9..e81cf228a544 100644
--- a/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
+++ b/lighthouse-cli/test/cli/__snapshots__/cli-flags-test.js.snap
@@ -23,8 +23,6 @@ Object {
     "--enabled-features=NetworkService,VirtualTime",
   ],
   "chromeIgnoreDefaultFlags": false,
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "fraggle-rock": false,
   "fraggleRock": false,
   "hostname": "127.0.0.1",
@@ -59,8 +57,6 @@ Object {
   "chrome-ignore-default-flags": false,
   "chromeFlags": "",
   "chromeIgnoreDefaultFlags": false,
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "fraggle-rock": false,
   "fraggleRock": false,
   "hostname": "127.0.0.1",
@@ -113,8 +109,6 @@ Object {
   "chrome-ignore-default-flags": false,
   "chromeFlags": "",
   "chromeIgnoreDefaultFlags": false,
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "extra-headers": Object {
     "foo": "bar",
   },
@@ -155,8 +149,6 @@ Object {
   "chrome-ignore-default-flags": false,
   "chromeFlags": "",
   "chromeIgnoreDefaultFlags": false,
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "extra-headers": Object {
     "Cookie": "monster=blue",
     "x-men": "wolverine",
@@ -199,8 +191,6 @@ Object {
   "chrome-ignore-default-flags": false,
   "chromeFlags": "",
   "chromeIgnoreDefaultFlags": false,
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "fraggle-rock": false,
   "fraggleRock": false,
   "hostname": "127.0.0.1",
@@ -249,8 +239,6 @@ Object {
   "chrome-ignore-default-flags": false,
   "chromeFlags": "",
   "chromeIgnoreDefaultFlags": false,
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "fraggle-rock": false,
   "fraggleRock": false,
   "hostname": "127.0.0.1",
@@ -299,8 +287,6 @@ Object {
   "chrome-ignore-default-flags": false,
   "chromeFlags": "",
   "chromeIgnoreDefaultFlags": false,
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "fraggle-rock": false,
   "fraggleRock": false,
   "hostname": "127.0.0.1",
@@ -349,8 +335,6 @@ Object {
   "chrome-ignore-default-flags": false,
   "chromeFlags": "",
   "chromeIgnoreDefaultFlags": false,
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "fraggle-rock": false,
   "fraggleRock": false,
   "hostname": "127.0.0.1",
@@ -406,8 +390,6 @@ Object {
   "chromeIgnoreDefaultFlags": false,
   "cli-flags-path": "__REPLACED__/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json",
   "cliFlagsPath": "__REPLACED__/lighthouse-cli/test/fixtures/cli-flags-path-inline-budgets.json",
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "fraggle-rock": false,
   "fraggleRock": false,
   "hostname": "127.0.0.1",
@@ -448,8 +430,6 @@ Object {
   "chromeIgnoreDefaultFlags": false,
   "cli-flags-path": "__REPLACED__/lighthouse-cli/test/fixtures/cli-flags-path.json",
   "cliFlagsPath": "__REPLACED__/lighthouse-cli/test/fixtures/cli-flags-path.json",
-  "enable-error-reporting": undefined,
-  "enableErrorReporting": undefined,
   "extra-headers": Object {
     "X-Men": "wolverine",
   },