-
-
Notifications
You must be signed in to change notification settings - Fork 71
/
argParser.ts
651 lines (555 loc) · 18 KB
/
argParser.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
import path from "path";
import fs from "fs";
import os from "os";
import yaml from "js-yaml";
import { KnownDevices as devices } from "puppeteer-core";
import yargs, { Options } from "yargs";
import { hideBin } from "yargs/helpers";
import {
BEHAVIOR_LOG_FUNC,
WAIT_UNTIL_OPTS,
EXTRACT_TEXT_TYPES,
} from "./constants.js";
import { ScopedSeed } from "./seeds.js";
import { interpolateFilename } from "./storage.js";
import { screenshotTypes } from "./screenshots.js";
import { LOG_CONTEXT_TYPES, logger } from "./logger.js";
// ============================================================================
class ArgParser {
get cliOpts(): { [key: string]: Options } {
const coerce = (array: string[]) => {
return array.flatMap((v) => v.split(",")).filter((x) => !!x);
};
return {
seeds: {
alias: "url",
describe: "The URL to start crawling from",
type: "array",
default: [],
},
seedFile: {
alias: ["urlFile"],
describe:
"If set, read a list of seed urls, one per line, from the specified",
type: "string",
},
workers: {
alias: "w",
describe: "The number of workers to run in parallel",
default: 1,
type: "number",
},
crawlId: {
alias: "id",
describe:
"A user provided ID for this crawl or crawl configuration (can also be set via CRAWL_ID env var, defaults to hostname)",
type: "string",
},
waitUntil: {
describe:
"Puppeteer page.goto() condition to wait for before continuing, can be multiple separated by ','",
type: "array",
default: ["load", "networkidle2"],
choices: WAIT_UNTIL_OPTS,
coerce,
},
depth: {
describe: "The depth of the crawl for all seeds",
default: -1,
type: "number",
},
extraHops: {
describe: "Number of extra 'hops' to follow, beyond the current scope",
default: 0,
type: "number",
},
pageLimit: {
alias: "limit",
describe: "Limit crawl to this number of pages",
default: 0,
type: "number",
},
maxPageLimit: {
describe:
"Maximum pages to crawl, overriding pageLimit if both are set",
default: 0,
type: "number",
},
pageLoadTimeout: {
alias: "timeout",
describe: "Timeout for each page to load (in seconds)",
default: 90,
type: "number",
},
scopeType: {
describe:
"A predefined scope of the crawl. For more customization, use 'custom' and set scopeIncludeRx regexes",
type: "string",
choices: [
"page",
"page-spa",
"prefix",
"host",
"domain",
"any",
"custom",
],
},
scopeIncludeRx: {
alias: "include",
describe:
"Regex of page URLs that should be included in the crawl (defaults to the immediate directory of URL)",
},
scopeExcludeRx: {
alias: "exclude",
describe: "Regex of page URLs that should be excluded from the crawl.",
},
allowHashUrls: {
describe:
"Allow Hashtag URLs, useful for single-page-application crawling or when different hashtags load dynamic content",
},
blockRules: {
describe:
"Additional rules for blocking certain URLs from being loaded, by URL regex and optionally via text match in an iframe",
type: "array",
default: [],
},
blockMessage: {
describe:
"If specified, when a URL is blocked, a record with this error message is added instead",
type: "string",
},
blockAds: {
alias: "blockads",
describe:
"If set, block advertisements from being loaded (based on Stephen Black's blocklist)",
type: "boolean",
default: false,
},
adBlockMessage: {
describe:
"If specified, when an ad is blocked, a record with this error message is added instead",
type: "string",
},
collection: {
alias: "c",
describe:
"Collection name to crawl to (replay will be accessible under this name in pywb preview)",
type: "string",
default: "crawl-@ts",
},
headless: {
describe: "Run in headless mode, otherwise start xvfb",
type: "boolean",
default: false,
},
driver: {
describe: "JS driver for the crawler",
type: "string",
default: "./defaultDriver.js",
},
generateCDX: {
alias: ["generatecdx", "generateCdx"],
describe:
"If set, generate index (CDXJ) for use with pywb after crawl is done",
type: "boolean",
default: false,
},
combineWARC: {
alias: ["combinewarc", "combineWarc"],
describe: "If set, combine the warcs",
type: "boolean",
default: false,
},
rolloverSize: {
describe: "If set, declare the rollover size",
default: 1000000000,
type: "number",
},
generateWACZ: {
alias: ["generatewacz", "generateWacz"],
describe: "If set, generate wacz",
type: "boolean",
default: false,
},
logging: {
describe:
"Logging options for crawler, can include: stats (enabled by default), jserrors, debug",
type: "array",
default: ["stats"],
coerce,
},
logLevel: {
describe: "Comma-separated list of log levels to include in logs",
type: "array",
default: [],
coerce,
},
context: {
alias: "logContext",
describe: "Comma-separated list of contexts to include in logs",
type: "array",
default: [],
choices: LOG_CONTEXT_TYPES,
coerce,
},
text: {
describe:
"Extract initial (default) or final text to pages.jsonl or WARC resource record(s)",
type: "array",
choices: EXTRACT_TEXT_TYPES,
coerce: (array) => {
// backwards compatibility: default --text true / --text -> --text to-pages
if (!array.length || (array.length === 1 && array[0] === "true")) {
return ["to-pages"];
}
if (array.length === 1 && array[0] === "false") {
return [];
}
return coerce(array);
},
},
cwd: {
describe:
"Crawl working directory for captures (pywb root). If not set, defaults to process.cwd()",
type: "string",
default: process.cwd(),
},
mobileDevice: {
describe:
"Emulate mobile device by name from: https://github.com/puppeteer/puppeteer/blob/main/src/common/DeviceDescriptors.ts",
type: "string",
},
userAgent: {
describe: "Override user-agent with specified string",
type: "string",
},
userAgentSuffix: {
describe:
"Append suffix to existing browser user-agent (ex: +MyCrawler, info@example.com)",
type: "string",
},
useSitemap: {
alias: "sitemap",
describe:
"If enabled, check for sitemaps at /sitemap.xml, or custom URL if URL is specified",
},
sitemapFromDate: {
alias: "sitemapFrom",
describe:
"If set, filter URLs from sitemaps to those greater than or equal to provided ISO Date string (YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS or partial date)",
},
statsFilename: {
describe:
"If set, output stats as JSON to this file. (Relative filename resolves to crawl working directory)",
},
behaviors: {
describe: "Which background behaviors to enable on each page",
type: "array",
default: ["autoplay", "autofetch", "autoscroll", "siteSpecific"],
choices: ["autoplay", "autofetch", "autoscroll", "siteSpecific"],
coerce,
},
behaviorTimeout: {
describe:
"If >0, timeout (in seconds) for in-page behavior will run on each page. If 0, a behavior can run until finish.",
default: 90,
type: "number",
},
pageExtraDelay: {
alias: "delay",
describe:
"If >0, amount of time to sleep (in seconds) after behaviors before moving on to next page",
default: 0,
type: "number",
},
dedupPolicy: {
describe: "Deduplication policy",
default: "skip",
type: "string",
choices: ["skip", "revisit", "keep"],
},
profile: {
describe:
"Path to tar.gz file which will be extracted and used as the browser profile",
type: "string",
},
screenshot: {
describe:
"Screenshot options for crawler, can include: view, thumbnail, fullPage",
type: "array",
default: [],
choices: Array.from(Object.keys(screenshotTypes)),
coerce,
},
screencastPort: {
describe:
"If set to a non-zero value, starts an HTTP server with screencast accessible on this port",
type: "number",
default: 0,
},
screencastRedis: {
describe:
"If set, will use the state store redis pubsub for screencasting. Requires --redisStoreUrl to be set",
type: "boolean",
default: false,
},
warcInfo: {
alias: ["warcinfo"],
describe:
"Optional fields added to the warcinfo record in combined WARCs",
//type: "object"
},
redisStoreUrl: {
describe:
"If set, url for remote redis server to store state. Otherwise, using in-memory store",
type: "string",
default: "redis://localhost:6379/0",
},
saveState: {
describe:
"If the crawl state should be serialized to the crawls/ directory. Defaults to 'partial', only saved when crawl is interrupted",
type: "string",
default: "partial",
choices: ["never", "partial", "always"],
},
saveStateInterval: {
describe:
"If save state is set to 'always', also save state during the crawl at this interval (in seconds)",
type: "number",
default: 300,
},
saveStateHistory: {
describe:
"Number of save states to keep during the duration of a crawl",
type: "number",
default: 5,
},
sizeLimit: {
describe:
"If set, save state and exit if size limit exceeds this value",
type: "number",
default: 0,
},
diskUtilization: {
describe:
"If set, save state and exit if disk utilization exceeds this percentage value",
type: "number",
default: 90,
},
timeLimit: {
describe: "If set, save state and exit after time limit, in seconds",
type: "number",
default: 0,
},
healthCheckPort: {
describe: "port to run healthcheck on",
type: "number",
default: 0,
},
overwrite: {
describe:
"overwrite current crawl data: if set, existing collection directory will be deleted before crawl is started",
type: "boolean",
default: false,
},
waitOnDone: {
describe:
"if set, wait for interrupt signal when finished instead of exiting",
type: "boolean",
default: false,
},
restartsOnError: {
describe:
"if set, assume will be restarted if interrupted, don't run post-crawl processes on interrupt",
type: "boolean",
default: false,
},
netIdleWait: {
describe:
"if set, wait for network idle after page load and after behaviors are done (in seconds). if -1 (default), determine based on scope",
type: "number",
default: -1,
},
lang: {
describe:
"if set, sets the language used by the browser, should be ISO 639 language[-country] code",
type: "string",
},
title: {
describe:
"If set, write supplied title into WACZ datapackage.json metadata",
type: "string",
},
description: {
alias: ["desc"],
describe:
"If set, write supplied description into WACZ datapackage.json metadata",
type: "string",
},
originOverride: {
describe:
"if set, will redirect requests from each origin in key to origin in the value, eg. --originOverride https://host:port=http://alt-host:alt-port",
type: "array",
default: [],
},
logErrorsToRedis: {
describe: "If set, write error messages to redis",
type: "boolean",
default: false,
},
writePagesToRedis: {
describe: "If set, write page objects to redis",
type: "boolean",
default: false,
},
failOnFailedSeed: {
describe:
"If set, crawler will fail with exit code 1 if any seed fails",
type: "boolean",
default: false,
},
failOnFailedLimit: {
describe:
"If set, save state and exit if number of failed pages exceeds this value",
type: "number",
default: 0,
},
failOnInvalidStatus: {
describe:
"If set, will treat pages with non-200 response as failures. When combined with --failOnFailedLimit or --failOnFailedSeed" +
"may result in crawl failing due to non-200 responses",
type: "boolean",
default: false,
},
customBehaviors: {
describe:
"injects a custom behavior file or set of behavior files in a directory",
type: "string",
},
debugAccessRedis: {
describe:
"if set, runs internal redis without protected mode to allow external access (for debugging)",
type: "boolean",
},
warcPrefix: {
describe:
"prefix for WARC files generated, including WARCs added to WACZ",
type: "string",
},
};
}
parseArgs(argvParams?: string[]) {
let argv = argvParams || process.argv;
if (process.env.CRAWL_ARGS) {
argv = argv.concat(this.splitCrawlArgsQuoteSafe(process.env.CRAWL_ARGS));
}
let origConfig = {};
const parsed = yargs(hideBin(argv))
.usage("crawler [options]")
.option(this.cliOpts)
.config(
"config",
"Path to YAML config file",
(configPath: string | number) => {
if (configPath === "/crawls/stdin") {
configPath = process.stdin.fd;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
origConfig = yaml.load(fs.readFileSync(configPath, "utf8")) as any;
return origConfig;
},
)
.check((argv) => this.validateArgs(argv)).argv;
return { parsed, origConfig };
}
splitCrawlArgsQuoteSafe(crawlArgs: string): string[] {
// Split process.env.CRAWL_ARGS on spaces but retaining spaces within double quotes
const regex = /"[^"]+"|[^\s]+/g;
const res = crawlArgs.match(regex);
return res ? res.map((e) => e.replace(/"(.+)"/, "$1")) : [];
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
validateArgs(argv: Record<string, any>) {
argv.crawlId = argv.crawlId || process.env.CRAWL_ID || os.hostname;
argv.collection = interpolateFilename(argv.collection, argv.crawlId);
// Check that the collection name is valid.
if (argv.collection.search(/^[\w][\w-]*$/) === -1) {
logger.fatal(
`\n${argv.collection} is an invalid collection name. Please supply a collection name only using alphanumeric characters and the following characters [_ - ]\n`,
);
}
// background behaviors to apply
const behaviorOpts: { [key: string]: string | boolean } = {};
argv.behaviors.forEach((x: string) => (behaviorOpts[x] = true));
behaviorOpts.log = BEHAVIOR_LOG_FUNC;
argv.behaviorOpts = JSON.stringify(behaviorOpts);
argv.text = argv.text || [];
if (argv.mobileDevice) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
argv.emulateDevice = (devices as Record<string, any>)[
argv.mobileDevice.replace("-", " ")
];
if (!argv.emulateDevice) {
logger.fatal("Unknown device: " + argv.mobileDevice);
}
} else {
argv.emulateDevice = { viewport: null };
}
if (argv.seedFile) {
const urlSeedFile = fs.readFileSync(argv.seedFile, "utf8");
const urlSeedFileList = urlSeedFile.split("\n");
if (typeof argv.seeds === "string") {
argv.seeds = [argv.seeds];
}
for (const seed of urlSeedFileList) {
if (seed) {
argv.seeds.push(seed);
}
}
}
if (argv.netIdleWait === -1) {
if (argv.scopeType === "page" || argv.scopeType === "page-spa") {
argv.netIdleWait = 15;
} else {
argv.netIdleWait = 2;
}
//logger.debug(`Set netIdleWait to ${argv.netIdleWait} seconds`);
}
const scopeOpts = {
scopeType: argv.scopeType,
sitemap: argv.sitemap,
include: argv.include,
exclude: argv.exclude,
depth: argv.depth,
extraHops: argv.extraHops,
};
argv.scopedSeeds = [];
for (let seed of argv.seeds) {
if (typeof seed === "string") {
seed = { url: seed };
}
try {
argv.scopedSeeds.push(new ScopedSeed({ ...scopeOpts, ...seed }));
} catch (e) {
if (argv.failOnFailedSeed) {
logger.fatal(`Invalid Seed "${seed.url}" specified, aborting crawl.`);
}
}
}
if (!argv.scopedSeeds.length) {
logger.fatal("No valid seeds specified, aborting crawl.");
}
// Resolve statsFilename
if (argv.statsFilename) {
argv.statsFilename = path.resolve(argv.cwd, argv.statsFilename);
}
if (argv.diskUtilization < 0 || argv.diskUtilization > 99) {
argv.diskUtilization = 90;
}
return true;
}
}
export function parseArgs(argv?: string[]) {
return new ArgParser().parseArgs(argv);
}