Always fail crawl if failOnFailedSeed set and seed is 4xx/5xx

webrecorder · May 6, 2024 · 29903e3 · 29903e3
1 parent ddc3e10
commit 29903e3
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 4 deletions.
diff --git a/src/crawler.ts b/src/crawler.ts
@@ -1727,8 +1727,12 @@ self.__bx_behaviors.selectMainBehavior();
 
       let failed = isChromeError;
 
-      if (this.params.failOnInvalidStatus && status >= 400) {
+      if (
+        (this.params.failOnInvalidStatus || failCrawlOnError) &&
+        status >= 400
+      ) {
         // Handle 4xx or 5xx response as a page load error
+        // Always fail seeds on 4xx or 5xx response if failOnFailedSeed is set
         failed = true;
       }
 

diff --git a/tests/seeds.test.js b/tests/seeds.test.js
@@ -7,7 +7,7 @@ test("ensure one invalid seed doesn't end crawl if failOnFailedSeed is not set",
   let passed = true;
   try {
     await exec(
-      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --collection invalidseed",
+      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://example.invalid --generateWACZ --limit 1 --collection invalidseed",
     );
   } catch (error) {
     console.log(error);
@@ -20,7 +20,7 @@ test("ensure one invalid seed fails crawl if failOnFailedSeed is set", async ()
   let passed = true;
   try {
     await exec(
-      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --failOnFailedSeed --collection failseed",
+      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://example.invalid --generateWACZ --limit 1 --failOnFailedSeed --collection failseed",
     );
   } catch (error) {
     passed = false;
@@ -32,7 +32,7 @@ test("ensure crawl fails if no valid seeds are passed", async () => {
   let passed = true;
   try {
     await exec(
-      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url iana.org/ --url example.com/invalid-seed --generateWACZ --limit 1 --collection allinvalidseeds",
+      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url iana.org/ --url https://example.invalid --generateWACZ --limit 1 --collection allinvalidseeds",
     );
   } catch (error) {
     passed = false;