diff --git a/src/logger.ts b/src/logger.ts index 9b8208b5ae..2183b3f43a 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -26,12 +26,12 @@ export const Logger = winston.createLogger({ }); export const Print = { - backoff(link: Link, store: Store, delay: number, color?: boolean): string { + backoff(link: Link, store: Store, parameters: {delay: number; statusCode: number}, color?: boolean): string { if (color) { - return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`); + return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`BACKOFF DELAY status=${parameters.statusCode} delay=${parameters.delay}`); } - return `✖ ${buildProductString(link, store)} :: REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`; + return `✖ ${buildProductString(link, store)} :: BACKOFF DELAY status=${parameters.statusCode} delay=${parameters.delay}`; }, badStatusCode(link: Link, store: Store, statusCode: number, color?: boolean): string { if (color) { diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 4f367b7cfc..fe0ab61b6d 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -2,22 +2,16 @@ import {Browser, Page, Response} from 'puppeteer'; import {Link, Store} from './model'; import {Logger, Print} from '../logger'; import {Selector, pageIncludesLabels} from './includes-labels'; -import {closePage, delay, getSleepTime} from '../util'; +import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util'; import {Config} from '../config'; import {disableBlockerInPage} from '../adblocker'; import {filterStoreLink} from './filter'; import open from 'open'; +import {processBackoffDelay} from './model/helpers/backoff'; import {sendNotification} from '../notification'; -type Backoff = { - count: number; - time: number; -}; - const inStock: Record = {}; -const storeBackoff: Record = {}; - /** * Responsible for looking up information about a each product within * a `Store`. It's important that we ignore `no-await-in-loop` here @@ -50,18 +44,25 @@ async function lookup(browser: Browser, store: Store) { } } + let statusCode = 0; + try { - await lookupCard(browser, store, page, link); + statusCode = await lookupCard(browser, store, page, link); } catch (error) { Logger.error(`✖ [${store.name}] ${link.brand} ${link.model} - ${error.message as string}`); } + // Must apply backoff before closing the page, e.g. if CloudFlare is + // used to detect bot traffic, it introduces a 5 second page delay + // before redirecting to the next page + await processBackoffDelay(store, link, statusCode); + await closePage(page); } /* eslint-enable no-await-in-loop */ } -async function lookupCard(browser: Browser, store: Store, page: Page, link: Link) { +async function lookupCard(browser: Browser, store: Store, page: Page, link: Link): Promise { const givenWaitFor = store.waitUntil ? store.waitUntil : 'networkidle0'; const response: Response | null = await page.goto(link.url, {waitUntil: givenWaitFor}); @@ -69,34 +70,16 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link Logger.debug(Print.noResponse(link, store, true)); } - let backoff = storeBackoff[store.name]; - - if (!backoff) { - backoff = {count: 0, time: Config.browser.minBackoff}; - storeBackoff[store.name] = backoff; - } - - if (response?.status() === 403) { - Logger.warn(Print.backoff(link, store, backoff.time, true)); - await delay(backoff.time); - backoff.count++; - backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff); - return; - } - - if (response?.status() === 429) { - Logger.warn(Print.rateLimit(link, store, true)); - return; - } - - if ((response?.status() || 200) >= 400) { - Logger.warn(Print.badStatusCode(link, store, response!.status(), true)); - return; - } + const successStatusCodes = store.successStatusCodes ?? [[0, 399]]; + const statusCode = response?.status() ?? 0; + if (!isStatusCodeInRange(statusCode, successStatusCodes)) { + if (statusCode === 429) { + Logger.warn(Print.rateLimit(link, store, true)); + } else { + Logger.warn(Print.badStatusCode(link, store, statusCode, true)); + } - if (backoff.count > 0) { - backoff.count--; - backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff); + return statusCode; } if (await lookupCardInStock(store, page, link)) { @@ -128,6 +111,8 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link await page.screenshot({path: link.screenshot}); } } + + return statusCode; } async function lookupCardInStock(store: Store, page: Page, link: Link) { diff --git a/src/store/model/asus.ts b/src/store/model/asus.ts index 04930d6f2d..71aff0dcbf 100644 --- a/src/store/model/asus.ts +++ b/src/store/model/asus.ts @@ -33,6 +33,7 @@ export const Asus: Store = { url: 'https://store.asus.com/us/item/202009AM150000003/' } ], - name: 'asus' + name: 'asus', + successStatusCodes: [[0, 399], 404] }; diff --git a/src/store/model/helpers/backoff.ts b/src/store/model/helpers/backoff.ts new file mode 100644 index 0000000000..af90109c7c --- /dev/null +++ b/src/store/model/helpers/backoff.ts @@ -0,0 +1,52 @@ +import {Link, Store} from '..'; +import {Logger, Print} from '../../../logger'; +import {delay, isStatusCodeInRange} from '../../../util'; +import {Config} from '../../../config'; + +type Backoff = { + count: number; + time: number; +}; + +const stores: Record = {}; + +export async function processBackoffDelay(store: Store, link: Link, statusCode: number): Promise { + /** + * We treat statusCode 0 as successful as some of the puppeteer plugins + * cause side-effects resulting in an empty response object even though + * the page renders fine and its content is accessible. + */ + + let backoffStatusCodes = store.backoffStatusCodes; + + if (!backoffStatusCodes) { + backoffStatusCodes = [403]; + } + + const isBackoff = isStatusCodeInRange(statusCode, backoffStatusCodes); + let backoff = stores[store.name]; + + if (!backoff) { + backoff = {count: 0, time: Config.browser.minBackoff}; + stores[store.name] = backoff; + } + + if (!isBackoff) { + if (backoff.count > 0) { + backoff.count--; + backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff); + } + + return -1; + } + + const backoffTime = backoff.time; + Logger.debug(Print.backoff(link, store, {delay: backoffTime, statusCode}, true)); + + await delay(backoff.time); + + backoff.count++; + backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff); + + return backoffTime; +} diff --git a/src/store/model/store.ts b/src/store/model/store.ts index 1b4bd82d27..bddc9db17e 100644 --- a/src/store/model/store.ts +++ b/src/store/model/store.ts @@ -27,7 +27,15 @@ export type Labels = { outOfStock?: LabelQuery; }; +export type StatusCodeRangeArray = Array<(number | [number, number])>; + export type Store = { + /** + * The range of status codes which will trigger backoff, i.e. an increasing + * delay between requests. Setting an empty array will disable the feature. + * If not defined, the default range will be used: 403. + */ + backoffStatusCodes?: StatusCodeRangeArray; disableAdBlocker?: boolean; links: Link[]; linksBuilder?: { @@ -37,5 +45,12 @@ export type Store = { labels: Labels; name: string; setupAction?: (browser: Browser) => void; + /** + * The range of status codes which considered successful, i.e. without error + * allowing request parsing to continue. Setting an empty array will cause + * all requests to fail. If not defined, the default range will be used: + * 0 -> 399 inclusive. + */ + successStatusCodes?: StatusCodeRangeArray; waitUntil?: LoadEvent; }; diff --git a/src/store/model/zotac.ts b/src/store/model/zotac.ts index 0158811b5f..18815c291f 100644 --- a/src/store/model/zotac.ts +++ b/src/store/model/zotac.ts @@ -1,6 +1,7 @@ import {Store} from './store'; export const Zotac: Store = { + backoffStatusCodes: [403, 503], labels: { inStock: { container: '.add-to-cart-wrapper', diff --git a/src/util.ts b/src/util.ts index ceb22383c8..0a983d35ac 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,6 +1,7 @@ import {Browser, Page, Response} from 'puppeteer'; import {Config} from './config'; import {Logger} from './logger'; +import {StatusCodeRangeArray} from './store/model'; import {disableBlockerInPage} from './adblocker'; export function getSleepTime() { @@ -13,6 +14,25 @@ export async function delay(ms: number) { }); } +export function isStatusCodeInRange(statusCode: number, range: StatusCodeRangeArray) { + for (const value of range) { + let min: number; + let max: number; + if (typeof value === 'number') { + min = value; + max = value; + } else { + [min, max] = value; + } + + if (min <= statusCode && statusCode <= max) { + return true; + } + } + + return false; +} + export async function usingResponse( browser: Browser, url: string,