From 503d76f3c7d01368fc978af4ab02924d3d036ac0 Mon Sep 17 00:00:00 2001 From: SesioN Date: Fri, 23 Oct 2020 05:03:58 +0200 Subject: [PATCH] feat: optional per store min and max page sleep time (#576) Co-authored-by: Jef LeCompte --- README.md | 2 +- src/config.ts | 9 ++++++++- src/index.ts | 2 +- src/store/lookup.ts | 4 ++-- src/store/model/index.ts | 8 ++++---- src/store/model/store.ts | 2 ++ src/util.ts | 7 ++++--- 7 files changed, 22 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 0b4e2ed76c..75edd87a4b 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ Here is a list of variables that you can use to customize your newly copied `.en | `SLACK_TOKEN` | Slack API token | | | `SMTP_ADDRESS` | IP Address or fqdn of smtp server | | `SMTP_PORT` | TCP Port number on which the smtp server is listening for connections | Default: `25` | -| `STORES` | [Supported stores](#supported-stores) you want to be scraped | Comma separated, default: `nvidia` | +| `STORES` | [Supported stores](#supported-stores) you want to be scraped | Both supported formats are comma separated

1. Standard E.g.: `"nvidia"`

2. Advanced E.g: `STORE:PAGE_SLEEP_MIN:PAGE_SLEEP_MAX`, E.g: `nvidia:10000:30000`

Default: `nvidia` | | `SCREENSHOT` | Capture screenshot of page if a card is found | Default: `true` | | `TELEGRAM_ACCESS_TOKEN` | Telegram access token | | | `TELEGRAM_CHAT_ID` | Telegram chat ID | Comma seperated, e.g.: `123456789`, `123456789,987654321` | diff --git a/src/config.ts b/src/config.ts index 8d9297d535..0d77966e30 100644 --- a/src/config.ts +++ b/src/config.ts @@ -245,7 +245,14 @@ const store = { showOnlyBrands: envOrArray(process.env.SHOW_ONLY_BRANDS), showOnlyModels: envOrArray(process.env.SHOW_ONLY_MODELS), showOnlySeries: envOrArray(process.env.SHOW_ONLY_SERIES, ['3070', '3080', '3090']), - stores: envOrArray(process.env.STORES, ['nvidia']) + stores: envOrArray(process.env.STORES, ['nvidia']).map(entry => { + const [name, minPageSleep, maxPageSleep] = entry.match(/[^:]+/g) ?? []; + return { + maxPageSleep: envOrNumberMax(minPageSleep, maxPageSleep, browser.maxSleep), + minPageSleep: envOrNumberMin(minPageSleep, maxPageSleep, browser.minSleep), + name: envOrString(name) + }; + }) }; export const config = { diff --git a/src/index.ts b/src/index.ts index 53725e69fc..dcd1ba191d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -60,7 +60,7 @@ async function main() { store.setupAction(browser); } - setTimeout(tryLookupAndLoop, getSleepTime(), browser, store); + setTimeout(tryLookupAndLoop, getSleepTime(store), browser, store); } } diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 4701703e13..81dbb779a5 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -183,7 +183,7 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { if (store.labels.captcha) { if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) { logger.warn(Print.captcha(link, store, true)); - await delay(getSleepTime()); + await delay(getSleepTime(store)); return false; } } @@ -212,7 +212,7 @@ export async function tryLookupAndLoop(browser: Browser, store: Store) { logger.error(error); } - const sleepTime = getSleepTime(); + const sleepTime = getSleepTime(store); logger.debug(`[${store.name}] Lookup done, next one in ${sleepTime} ms`); setTimeout(tryLookupAndLoop, sleepTime, browser, store); } diff --git a/src/store/model/index.ts b/src/store/model/index.ts index 70b66f38c1..a9bbafd56c 100644 --- a/src/store/model/index.ts +++ b/src/store/model/index.ts @@ -99,11 +99,11 @@ const masterList = new Map([ const list = new Map(); -for (const name of config.store.stores) { - if (masterList.has(name)) { - list.set(name, masterList.get(name)); +for (const storeData of config.store.stores) { + if (masterList.has(storeData.name)) { + list.set(storeData.name, {...masterList.get(storeData.name), storeData}); } else { - const logString = `No store named ${name}, skipping.`; + const logString = `No store named ${storeData.name}, skipping.`; logger.warn(logString); } } diff --git a/src/store/model/store.ts b/src/store/model/store.ts index 31e72f032f..a3eae50167 100644 --- a/src/store/model/store.ts +++ b/src/store/model/store.ts @@ -60,4 +60,6 @@ export type Store = { */ successStatusCodes?: StatusCodeRangeArray; waitUntil?: LoadEvent; + minPageSleep?: number; + maxPageSleep?: number; }; diff --git a/src/util.ts b/src/util.ts index 3111d26068..1c45ddfe8a 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,11 +1,12 @@ import {Browser, Page, Response} from 'puppeteer'; -import {StatusCodeRangeArray} from './store/model'; +import {StatusCodeRangeArray, Store} from './store/model'; import {config} from './config'; import {disableBlockerInPage} from './adblocker'; import {logger} from './logger'; -export function getSleepTime() { - return config.browser.minSleep + (Math.random() * (config.browser.maxSleep - config.browser.minSleep)); +export function getSleepTime(store: Store) { + const minSleep = store.minPageSleep as number; + return minSleep + (Math.random() * ((store.maxPageSleep as number) - minSleep)); } export async function delay(ms: number) {