feat: configurable status code behaviours (#340)

src/logger.ts

@@ -26,12 +26,12 @@ export const Logger = winston.createLogger({
 });
 
 export const Print = {
-	backoff(link: Link, store: Store, delay: number, color?: boolean): string {
+	backoff(link: Link, store: Store, parameters: {delay: number; statusCode: number}, color?: boolean): string {
 		if (color) {
-			return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`);
+			return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`BACKOFF DELAY status=${parameters.statusCode} delay=${parameters.delay}`);
 		}
 
-		return `✖ ${buildProductString(link, store)} :: REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`;
+		return `✖ ${buildProductString(link, store)} :: BACKOFF DELAY status=${parameters.statusCode} delay=${parameters.delay}`;
 	},
 	badStatusCode(link: Link, store: Store, statusCode: number, color?: boolean): string {
 		if (color) {

src/store/lookup.ts

@@ -2,22 +2,16 @@ import {Browser, Page, Response} from 'puppeteer';
 import {Link, Store} from './model';
 import {Logger, Print} from '../logger';
 import {Selector, pageIncludesLabels} from './includes-labels';
-import {closePage, delay, getSleepTime} from '../util';
+import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util';
 import {Config} from '../config';
 import {disableBlockerInPage} from '../adblocker';
 import {filterStoreLink} from './filter';
 import open from 'open';
+import {processBackoffDelay} from './model/helpers/backoff';
 import {sendNotification} from '../notification';
 
-type Backoff = {
-	count: number;
-	time: number;
-};
-
 const inStock: Record<string, boolean> = {};
 
-const storeBackoff: Record<string, Backoff> = {};
-
 /**
  * Responsible for looking up information about a each product within
  * a `Store`. It's important that we ignore `no-await-in-loop` here
@@ -50,53 +44,42 @@ async function lookup(browser: Browser, store: Store) {
 			}
 		}
 
+		let statusCode = 0;
+
 		try {
-			await lookupCard(browser, store, page, link);
+			statusCode = await lookupCard(browser, store, page, link);
 		} catch (error) {
 			Logger.error(`✖ [${store.name}] ${link.brand} ${link.model} - ${error.message as string}`);
 		}
 
+		// Must apply backoff before closing the page, e.g. if CloudFlare is
+		// used to detect bot traffic, it introduces a 5 second page delay
+		// before redirecting to the next page
+		await processBackoffDelay(store, link, statusCode);
+
 		await closePage(page);
 	}
 	/* eslint-enable no-await-in-loop */
 }
 
-async function lookupCard(browser: Browser, store: Store, page: Page, link: Link) {
+async function lookupCard(browser: Browser, store: Store, page: Page, link: Link): Promise<number> {
 	const givenWaitFor = store.waitUntil ? store.waitUntil : 'networkidle0';
 	const response: Response | null = await page.goto(link.url, {waitUntil: givenWaitFor});
 
 	if (!response) {
 		Logger.debug(Print.noResponse(link, store, true));
 	}
 
-	let backoff = storeBackoff[store.name];
-
-	if (!backoff) {
-		backoff = {count: 0, time: Config.browser.minBackoff};
-		storeBackoff[store.name] = backoff;
-	}
-
-	if (response?.status() === 403) {
-		Logger.warn(Print.backoff(link, store, backoff.time, true));
-		await delay(backoff.time);
-		backoff.count++;
-		backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff);
-		return;
-	}
-
-	if (response?.status() === 429) {
-		Logger.warn(Print.rateLimit(link, store, true));
-		return;
-	}
-
-	if ((response?.status() || 200) >= 400) {
-		Logger.warn(Print.badStatusCode(link, store, response!.status(), true));
-		return;
-	}
+	const successStatusCodes = store.successStatusCodes ?? [[0, 399]];
+	const statusCode = response?.status() ?? 0;
+	if (!isStatusCodeInRange(statusCode, successStatusCodes)) {
+		if (statusCode === 429) {
+			Logger.warn(Print.rateLimit(link, store, true));
+		} else {
+			Logger.warn(Print.badStatusCode(link, store, statusCode, true));
+		}
 
-	if (backoff.count > 0) {
-		backoff.count--;
-		backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff);
+		return statusCode;
 	}
 
 	if (await lookupCardInStock(store, page, link)) {
@@ -128,6 +111,8 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
 			await page.screenshot({path: link.screenshot});
 		}
 	}
+
+	return statusCode;
 }
 
 async function lookupCardInStock(store: Store, page: Page, link: Link) {

src/store/model/asus.ts

@@ -33,6 +33,7 @@ export const Asus: Store = {
 			url: 'https://store.asus.com/us/item/202009AM150000003/'
 		}
 	],
-	name: 'asus'
+	name: 'asus',
+	successStatusCodes: [[0, 399], 404]
 };
 

src/store/model/helpers/backoff.ts

@@ -0,0 +1,52 @@
+import {Link, Store} from '..';
+import {Logger, Print} from '../../../logger';
+import {delay, isStatusCodeInRange} from '../../../util';
+import {Config} from '../../../config';
+
+type Backoff = {
+	count: number;
+	time: number;
+};
+
+const stores: Record<string, Backoff> = {};
+
+export async function processBackoffDelay(store: Store, link: Link, statusCode: number): Promise<number> {
+	/**
+	 * We treat statusCode 0 as successful as some of the puppeteer plugins
+	 * cause side-effects resulting in an empty response object even though
+	 * the page renders fine and its content is accessible.
+	 */
+
+	let backoffStatusCodes = store.backoffStatusCodes;
+
+	if (!backoffStatusCodes) {
+		backoffStatusCodes = [403];
+	}
+
+	const isBackoff = isStatusCodeInRange(statusCode, backoffStatusCodes);
+	let backoff = stores[store.name];
+
+	if (!backoff) {
+		backoff = {count: 0, time: Config.browser.minBackoff};
+		stores[store.name] = backoff;
+	}
+
+	if (!isBackoff) {
+		if (backoff.count > 0) {
+			backoff.count--;
+			backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff);
+		}
+
+		return -1;
+	}
+
+	const backoffTime = backoff.time;
+	Logger.debug(Print.backoff(link, store, {delay: backoffTime, statusCode}, true));
+
+	await delay(backoff.time);
+
+	backoff.count++;
+	backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff);
+
+	return backoffTime;
+}

src/store/model/store.ts

@@ -27,7 +27,15 @@ export type Labels = {
 	outOfStock?: LabelQuery;
 };
 
+export type StatusCodeRangeArray = Array<(number | [number, number])>;
+
 export type Store = {
+	/**
+	 * The range of status codes which will trigger backoff, i.e. an increasing
+	 * delay between requests. Setting an empty array will disable the feature.
+	 * If not defined, the default range will be used: 403.
+	 */
+	backoffStatusCodes?: StatusCodeRangeArray;
 	disableAdBlocker?: boolean;
 	links: Link[];
 	linksBuilder?: {
@@ -37,5 +45,12 @@ export type Store = {
 	labels: Labels;
 	name: string;
 	setupAction?: (browser: Browser) => void;
+	/**
+	 * The range of status codes which considered successful, i.e. without error
+	 * allowing request parsing to continue. Setting an empty array will cause
+	 * all requests to fail. If not defined, the default range will be used:
+	 * 0 -> 399 inclusive.
+	 */
+	successStatusCodes?: StatusCodeRangeArray;
 	waitUntil?: LoadEvent;
 };

src/store/model/zotac.ts

@@ -1,6 +1,7 @@
 import {Store} from './store';
 
 export const Zotac: Store = {
+	backoffStatusCodes: [403, 503],
 	labels: {
 		inStock: {
 			container: '.add-to-cart-wrapper',

src/util.ts

@@ -1,6 +1,7 @@
 import {Browser, Page, Response} from 'puppeteer';
 import {Config} from './config';
 import {Logger} from './logger';
+import {StatusCodeRangeArray} from './store/model';
 import {disableBlockerInPage} from './adblocker';
 
 export function getSleepTime() {
@@ -13,6 +14,25 @@ export async function delay(ms: number) {
 	});
 }
 
+export function isStatusCodeInRange(statusCode: number, range: StatusCodeRangeArray) {
+	for (const value of range) {
+		let min: number;
+		let max: number;
+		if (typeof value === 'number') {
+			min = value;
+			max = value;
+		} else {
+			[min, max] = value;
+		}
+
+		if (min <= statusCode && statusCode <= max) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
 export async function usingResponse<T>(
 	browser: Browser,
 	url: string,