-
Notifications
You must be signed in to change notification settings - Fork 9.3k
/
smokehouse-bin.js
261 lines (229 loc) · 9.36 KB
/
smokehouse-bin.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
#!/usr/bin/env node
/**
* @license Copyright 2019 The Lighthouse Authors. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
*/
'use strict';
/**
* @fileoverview A smokehouse frontend for running from the command line. Parse
* flags, start fixture webservers, then run smokehouse.
*/
/* eslint-disable no-console */
import path from 'path';
import fs from 'fs';
import url from 'url';
import cloneDeep from 'lodash.clonedeep';
import yargs from 'yargs';
import * as yargsHelpers from 'yargs/helpers';
import log from 'lighthouse-logger';
import {runSmokehouse, getShardedDefinitions} from '../smokehouse.js';
import {updateTestDefnFormat} from './back-compat-util.js';
import {LH_ROOT} from '../../../../root.js';
const coreTestDefnsPath =
path.join(LH_ROOT, 'lighthouse-cli/test/smokehouse/core-tests.js');
/**
* Possible Lighthouse runners. Loaded dynamically so e.g. a CLI run isn't
* contingent on having built all the bundles.
*/
const runnerPaths = {
cli: '../lighthouse-runners/cli.js',
bundle: '../lighthouse-runners/bundle.js',
devtools: '../lighthouse-runners/devtools.js',
};
/**
* Determine batches of smoketests to run, based on the `requestedIds`.
* @param {Array<Smokehouse.TestDfn>} allTestDefns
* @param {Array<string>} requestedIds
* @param {{invertMatch: boolean}} options
* @return {Array<Smokehouse.TestDfn>}
*/
function getDefinitionsToRun(allTestDefns, requestedIds, {invertMatch}) {
let smokes = [];
const usage = ` ${log.dim}yarn smoke ${allTestDefns.map(t => t.id).join(' ')}${log.reset}\n`;
if (requestedIds.length === 0 && !invertMatch) {
smokes = [...allTestDefns];
console.log('Running ALL smoketests. Equivalent to:');
console.log(usage);
} else {
smokes = allTestDefns.filter(test => {
// Include all tests that *include* requested id.
// e.g. a requested 'pwa' will match 'pwa-airhorner', 'pwa-caltrain', etc
let isRequested = requestedIds.some(requestedId => test.id.includes(requestedId));
if (invertMatch) isRequested = !isRequested;
return isRequested;
});
console.log(`Running ONLY smoketests for: ${smokes.map(t => t.id).join(' ')}\n`);
}
const unmatchedIds = requestedIds.filter(requestedId => {
return !allTestDefns.map(t => t.id).some(id => id.includes(requestedId));
});
if (unmatchedIds.length) {
console.log(log.redify(`Smoketests not found for: ${unmatchedIds.join(' ')}`));
console.log(usage);
}
if (!smokes.length) {
throw new Error('no smoketest found to run');
}
return smokes;
}
/**
* Prune the `networkRequests` from the test expectations when `takeNetworkRequestUrls`
* is not defined. Custom servers may not have this method available in-process.
* Also asserts that any expectation with `networkRequests` is run serially. For core
* tests, we don't currently have a good way to map requests to test definitions if
* the tests are run in parallel.
* @param {Array<Smokehouse.TestDfn>} testDefns
* @param {Function|undefined} takeNetworkRequestUrls
* @return {Array<Smokehouse.TestDfn>}
*/
function pruneExpectedNetworkRequests(testDefns, takeNetworkRequestUrls) {
const pruneNetworkRequests = !takeNetworkRequestUrls;
const clonedDefns = cloneDeep(testDefns);
for (const {id, expectations, runSerially} of clonedDefns) {
if (!runSerially && expectations.networkRequests) {
throw new Error(`'${id}' must be set to 'runSerially: true' to assert 'networkRequests'`);
}
if (pruneNetworkRequests && expectations.networkRequests) {
// eslint-disable-next-line max-len
const msg = `'networkRequests' cannot be asserted in test '${id}'. They should only be asserted on tests from an in-process server`;
if (process.env.CI) {
// If we're in CI, we require any networkRequests expectations to be asserted.
throw new Error(msg);
}
console.warn(log.redify('Warning:'),
`${msg}. Pruning expectation: ${JSON.stringify(expectations.networkRequests)}`);
expectations.networkRequests = undefined;
}
}
return clonedDefns;
}
/**
* CLI entry point.
*/
async function begin() {
const y = yargs(yargsHelpers.hideBin(process.argv));
const rawArgv = y
.help('help')
.usage('node $0 [<options>] <test-ids>')
.example('node $0 -j=1 pwa seo', 'run pwa and seo tests serially')
.example('node $0 --invert-match byte', 'run all smoke tests but `byte`')
.option('_', {
array: true,
type: 'string',
})
.options({
'debug': {
type: 'boolean',
default: false,
describe: 'Save test artifacts and output verbose logs',
},
'fraggle-rock': {
type: 'boolean',
default: false,
describe: 'Use the new Fraggle Rock runner',
},
'jobs': {
type: 'number',
alias: 'j',
describe: 'Manually set the number of jobs to run at once. `1` runs all tests serially',
},
'retries': {
type: 'number',
describe: 'The number of times to retry failing tests before accepting. Defaults to 0',
},
'runner': {
default: 'cli',
choices: ['cli', 'bundle', 'devtools'],
describe: 'The method of running Lighthouse',
},
'tests-path': {
type: 'string',
describe: 'The path to a set of test definitions to run. Defaults to core smoke tests.',
},
'invert-match': {
type: 'boolean',
default: false,
describe: 'Run all available tests except the ones provided',
},
'shard': {
type: 'string',
// eslint-disable-next-line max-len
describe: 'A argument of the form "n/d", which divides the selected tests into d groups and runs the nth group. n and d must be positive integers with 1 ≤ n ≤ d.',
},
})
.wrap(y.terminalWidth())
.argv;
// Augmenting yargs type with auto-camelCasing breaks in tsc@4.1.2 and @types/yargs@15.0.11,
// so for now cast to add yarg's camelCase properties to type.
const argv =
/** @type {Awaited<typeof rawArgv> & CamelCasify<Awaited<typeof rawArgv>>} */ (rawArgv);
const jobs = Number.isFinite(argv.jobs) ? argv.jobs : undefined;
const retries = Number.isFinite(argv.retries) ? argv.retries : undefined;
const runnerPath = runnerPaths[/** @type {keyof typeof runnerPaths} */ (argv.runner)];
if (argv.runner === 'bundle') {
console.log('\n✨ Be sure to have recently run this: yarn build-all');
}
const {runLighthouse} = await import(runnerPath);
// Find test definition file and filter by requestedTestIds.
let testDefnPath = argv.testsPath || coreTestDefnsPath;
testDefnPath = path.resolve(process.cwd(), testDefnPath);
const requestedTestIds = argv._;
const {default: rawTestDefns} = await import(url.pathToFileURL(testDefnPath).href);
const allTestDefns = updateTestDefnFormat(rawTestDefns);
const invertMatch = argv.invertMatch;
const requestedTestDefns = getDefinitionsToRun(allTestDefns, requestedTestIds, {invertMatch});
const testDefns = getShardedDefinitions(requestedTestDefns, argv.shard);
let smokehouseResult;
let server;
let serverForOffline;
let takeNetworkRequestUrls = undefined;
try {
// If running the core tests, spin up the test server.
if (testDefnPath === coreTestDefnsPath) {
({server, serverForOffline} = await import('../../fixtures/static-server.js'));
server.listen(10200, 'localhost');
serverForOffline.listen(10503, 'localhost');
takeNetworkRequestUrls = server.takeRequestUrls.bind(server);
}
const prunedTestDefns = pruneExpectedNetworkRequests(testDefns, takeNetworkRequestUrls);
const options = {
jobs,
retries,
isDebug: argv.debug,
useFraggleRock: argv.fraggleRock,
lighthouseRunner: runLighthouse,
takeNetworkRequestUrls,
};
smokehouseResult = (await runSmokehouse(prunedTestDefns, options));
} finally {
if (server) await server.close();
if (serverForOffline) await serverForOffline.close();
}
if (!smokehouseResult.success) {
const failedTestResults = smokehouseResult.testResults.filter(r => r.failed);
// For CI, save failed runs to directory to be uploaded.
if (process.env.CI) {
const failuresDir = `${LH_ROOT}/.tmp/smokehouse-ci-failures`;
fs.mkdirSync(failuresDir, {recursive: true});
for (const testResult of failedTestResults) {
for (let i = 0; i < testResult.runs.length; i++) {
const run = testResult.runs[i];
fs.writeFileSync(`${failuresDir}/${testResult.id}-${i}.json`, JSON.stringify({
...run,
lighthouseLog: run.lighthouseLog.split('\n'),
assertionLog: run.assertionLog.split('\n'),
}, null, 2));
}
}
}
const cmd = `yarn smoke ${failedTestResults.map(r => r.id).join(' ')}`;
console.log(`rerun failures: ${cmd}`);
}
const exitCode = smokehouseResult.success ? 0 : 1;
process.exit(exitCode);
}
begin().catch(e => {
console.error(e);
process.exit(1);
});