diff --git a/.env-example b/.env-example index 79b8cc46cc..7e1ebbcf10 100644 --- a/.env-example +++ b/.env-example @@ -11,7 +11,8 @@ PHONE_CARRIER="tmobile" PLAY_SOUND="notification.mp3" PUSHOVER_TOKEN="123pushover-token456" PUSHOVER_USER="123pushover-user-key" -RATE_LIMIT_TIMEOUT="5000" +PAGE_SLEEP_MIN="5000" +PAGE_SLEEP_MAX="10000" SHOW_ONLY_BRANDS="evga" SLACK_CHANNEL="SlackChannelName" SLACK_TOKEN="slack-token" diff --git a/README.md b/README.md index 1190a24439..7b2a9730fa 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ Here is a list of variables that you can use to customize your newly copied `.en | `PLAY_SOUND` | Play this sound notification if a card is found | E.g.: `path/to/notification.wav`, relative path accepted, valid formats: wav, mp3, flac, [free sounds available](https://notificationsounds.com/) | | `PUSHOVER_TOKEN` | Pushover access token | Generate at https://pushover.net/apps/build | | `PUSHOVER_USERNAME` | Pushover username | -| `RATE_LIMIT_TIMEOUT` | Rate limit timeout for each full store cycle | Default: `5000` | +| `PAGE_SLEEP_MIN` | Minimum sleep time between queries of the same store | Default: `5000` | +| `PAGE_SLEEP_MAX` | Maximum sleep time between queries of the same store | Default: `10000` | | `SHOW_ONLY_BRANDS` | Filter to show specified brands | Comma separated, E.g.: `evga,zotac` | | `SLACK_CHANNEL` | Slack channel for posting | E.g., `update`, no need for `#` | | `SLACK_TOKEN` | Slack API token | diff --git a/package.json b/package.json index 281439cc43..4b05d229ac 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ }, "homepage": "https://github.com/jef/nvidia-snatcher#readme", "dependencies": { - "async": "^3.2.0", "dotenv": "^8.2.0", "messaging-api-telegram": "^1.0.0", "nodemailer": "^6.4.11", diff --git a/src/config.ts b/src/config.ts index cbac645eab..d181235ca5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,7 +6,8 @@ config({path: resolve(__dirname, '../.env')}); const browser = { isHeadless: process.env.HEADLESS ? process.env.HEADLESS === 'true' : true, open: process.env.OPEN_BROWSER === 'true', - rateLimitTimeout: process.env.RATE_LIMIT_TIMEOUT ? Number(process.env.RATE_LIMIT_TIMEOUT) : 5000 + minSleep: Number(process.env.PAGE_SLEEP_MIN ?? 5000), + maxSleep: Number(process.env.PAGE_SLEEP_MAX ?? 10000) }; const logLevel = process.env.LOG_LEVEL ?? 'info'; @@ -53,7 +54,7 @@ const page = { capture: process.env.SCREENSHOT === 'true', width: 1920, height: 1080, - navigationTimeout: Number(process.env.PAGE_TIMEOUT) ?? 30000, + navigationTimeout: Number(process.env.PAGE_TIMEOUT ?? 30000), userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36' }; diff --git a/src/index.ts b/src/index.ts index 5431347467..19af0edda1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,11 +5,28 @@ import {Config} from './config'; import {Store, Stores} from './store/model'; import {Logger} from './logger'; import {lookup} from './store'; -import async from 'async'; +import {Browser} from 'puppeteer'; puppeteer.use(stealthPlugin()); puppeteer.use(adblockerPlugin({blockTrackers: true})); +function getSleepTime() { + return Config.browser.minSleep + (Math.random() * (Config.browser.maxSleep - Config.browser.minSleep)); +} + +async function tryLookupAndLoop(browser: Browser, store: Store) { + Logger.debug(`[${store.name}] Starting lookup...`); + try { + await lookup(browser, store); + } catch (error) { + Logger.error(error); + } + + const sleepTime = getSleepTime(); + Logger.debug(`[${store.name}] Lookup done, next one in ${sleepTime} ms`); + setTimeout(tryLookupAndLoop, sleepTime, browser, store); +} + /** * Starts the bot. */ @@ -22,32 +39,10 @@ async function main() { } }); - const q = async.queue(async (store: Store, cb) => { - setTimeout(async () => { - try { - Logger.debug(`↗ scraping initialized - ${store.name}`); - await lookup(browser, store); - } catch (error) { - // Ignoring errors; more than likely due to rate limits - Logger.error(error); - } finally { - cb(); - q.push(store); - } - }, Config.browser.rateLimitTimeout); - }, Stores.length); - for (const store of Stores) { Logger.debug(store.links); - q.push(store); - if (Stores.length === 1) { - q.push(store); - } // Keep from completely draining + setTimeout(tryLookupAndLoop, getSleepTime(), browser, store); } - - await q.drain(); - - await browser.close(); } /**