Make sure --failOnFailedSeed results in exit code of 1

When pages are unreachable due to DNS not resolving, the crawler was failing with exit code 17, which conflicts with expected and documented behavior.
webrecorder · May 8, 2024 · 46a66e0 · 46a66e0
1 parent ddc3e10
commit 46a66e0
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 13 deletions.
diff --git a/src/crawler.ts b/src/crawler.ts
@@ -1734,10 +1734,15 @@ self.__bx_behaviors.selectMainBehavior();
 
       if (failed) {
         if (failCrawlOnError) {
-          logger.fatal("Seed Page Load Error, failing crawl", {
-            status,
-            ...logDetails,
-          });
+          logger.fatal(
+            "Seed Page Load Error, failing crawl",
+            {
+              status,
+              ...logDetails,
+            },
+            "general",
+            1,
+          );
         } else {
           logger.error(
             isChromeError ? "Page Crashed on Load" : "Page Invalid Status",
@@ -1775,10 +1780,15 @@ self.__bx_behaviors.selectMainBehavior();
           data.skipBehaviors = true;
         } else if (failCrawlOnError) {
           // if fail on error, immediately fail here
-          logger.fatal("Page Load Timeout, failing crawl", {
-            msg,
-            ...logDetails,
-          });
+          logger.fatal(
+            "Page Load Timeout, failing crawl",
+            {
+              msg,
+              ...logDetails,
+            },
+            "general",
+            1,
+          );
         } else {
           // log if not already log and rethrow
           if (msg !== "logged") {

diff --git a/src/util/argParser.ts b/src/util/argParser.ts
@@ -498,7 +498,8 @@ class ArgParser {
 
       failOnFailedSeed: {
         describe:
-          "If set, crawler will fail with exit code 1 if any seed fails",
+          "If set, crawler will fail with exit code 1 if any seed fails. Crawl will fail if any seeds have a" +
+          "4xx or 5xx response regardless of whether failOnInvalidStatus is set",
         type: "boolean",
         default: false,
       },
@@ -512,7 +513,7 @@ class ArgParser {
 
       failOnInvalidStatus: {
         describe:
-          "If set, will treat pages with non-200 response as failures. When combined with --failOnFailedLimit or --failOnFailedSeed" +
+          "If set, will treat pages with non-200 response as failures. When combined with --failOnFailedLimit" +
           "may result in crawl failing due to non-200 responses",
         type: "boolean",
         default: false,

diff --git a/tests/seeds.test.js b/tests/seeds.test.js
@@ -7,7 +7,7 @@ test("ensure one invalid seed doesn't end crawl if failOnFailedSeed is not set",
   let passed = true;
   try {
     await exec(
-      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --collection invalidseed",
+      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://example.invalid --generateWACZ --limit 1 --collection invalidseed",
     );
   } catch (error) {
     console.log(error);
@@ -20,7 +20,7 @@ test("ensure one invalid seed fails crawl if failOnFailedSeed is set", async ()
   let passed = true;
   try {
     await exec(
-      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --failOnFailedSeed --collection failseed",
+      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://example.invalid --generateWACZ --limit 1 --failOnFailedSeed --collection failseed",
     );
   } catch (error) {
     passed = false;
@@ -32,7 +32,7 @@ test("ensure crawl fails if no valid seeds are passed", async () => {
   let passed = true;
   try {
     await exec(
-      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --collection allinvalidseeds",
+      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url iana.org/ --url example.invalid --generateWACZ --limit 1 --collection allinvalidseeds",
     );
   } catch (error) {
     passed = false;