From 0aa7ab596c907ce72c188eb4b1acdee088307437 Mon Sep 17 00:00:00 2001 From: Joshua Higgins Date: Sat, 26 Sep 2020 08:05:02 -0400 Subject: [PATCH] feat: low bandwidth mode (#294) Co-authored-by: Jef LeCompte --- .env-example | 1 + README.md | 1 + package-lock.json | 24 +++++++++++++++++++ package.json | 1 + src/config.ts | 1 + src/index.ts | 9 ++++++- ...uppeteer-extra-plugin-block-resources.d.ts | 1 + src/util.ts | 5 +++- 8 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 src/types/puppeteer-extra-plugin-block-resources.d.ts diff --git a/.env-example b/.env-example index 90143f5960..02a0f8e733 100644 --- a/.env-example +++ b/.env-example @@ -10,6 +10,7 @@ EMAIL_PASSWORD="" HEADLESS="" IN_STOCK_WAIT_TIME="" LOG_LEVEL="" +LOW_BANDWIDTH="" MICROCENTER_LOCATION="" OPEN_BROWSER="" PAGE_TIMEOUT="" diff --git a/README.md b/README.md index 4343cc56b0..3f61c78e8f 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ Here is a list of variables that you can use to customize your newly copied `.en | `HEADLESS` | Puppeteer to run headless or not | Debugging related, default: `true` | | `IN_STOCK_WAIT_TIME` | Time to wait between requests to the same link if it has that card in stock | In seconds, default: `0` | | `LOG_LEVEL` | [Logging levels](https://github.com/winstonjs/winston#logging-levels) | Debugging related, default: `info` | +| `LOW_BANDWIDTH` | Blocks images/fonts to reduce traffic (Note: disabled adblocker) | Default: `false` | | `MICROCENTER_LOCATION` | Specific MicroCenter location to search | Default : `web` | | `OPEN_BROWSER` | Toggle for whether or not the browser should open when item is found | Default: `true` | | `PAGE_TIMEOUT` | Navigation Timeout in milliseconds | `0` for infinite, default: `30000` | diff --git a/package-lock.json b/package-lock.json index f83c92e333..c420e11f2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5963,6 +5963,30 @@ } } }, + "puppeteer-extra-plugin-block-resources": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-block-resources/-/puppeteer-extra-plugin-block-resources-2.2.7.tgz", + "integrity": "sha512-u2vcynnB9pINEVs7ZQs7DS8cJ2eedD0paL1M8ZdpNOQMCZ4zAVRwUpXF6T5v865nMu5srDnA3fTfdv6Ak0DG0g==", + "requires": { + "debug": "^4.1.1", + "puppeteer-extra-plugin": "^3.1.7" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "puppeteer-extra-plugin-stealth": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.6.1.tgz", diff --git a/package.json b/package.json index 401530f07b..6dbec0bbdd 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "puppeteer": "^5.3.1", "puppeteer-extra": "^3.1.15", "puppeteer-extra-plugin-adblocker": "^2.11.6", + "puppeteer-extra-plugin-block-resources": "^2.2.7", "puppeteer-extra-plugin-stealth": "^2.6.1", "pushbullet": "^2.4.0", "pushover-notifications": "^1.2.2", diff --git a/src/config.ts b/src/config.ts index 305a25b3f9..49bd5d7169 100644 --- a/src/config.ts +++ b/src/config.ts @@ -49,6 +49,7 @@ function envOrNumber(environment: string | undefined, number?: number): number { const browser = { isHeadless: envOrBoolean(process.env.HEADLESS), isTrusted: envOrBoolean(process.env.BROWSER_TRUSTED, false), + lowBandwidth: envOrBoolean(process.env.LOW_BANDWIDTH, false), maxBackoff: envOrNumber(process.env.PAGE_BACKOFF_MAX, 3600000), maxSleep: envOrNumber(process.env.PAGE_SLEEP_MAX, 10000), minBackoff: envOrNumber(process.env.PAGE_BACKOFF_MIN, 10000), diff --git a/src/index.ts b/src/index.ts index f04bd7ce45..0fe2d8da44 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,11 +5,18 @@ import {adBlocker} from './adblocker'; import {fetchLinks} from './store/fetch-links'; import {getSleepTime} from './util'; import puppeteer from 'puppeteer-extra'; +import resourceBlock from 'puppeteer-extra-plugin-block-resources'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import {tryLookupAndLoop} from './store'; puppeteer.use(stealthPlugin()); -puppeteer.use(adBlocker); +if (Config.browser.lowBandwidth) { + puppeteer.use(resourceBlock({ + blockedTypes: new Set(['image', 'font']) + })); +} else { + puppeteer.use(adBlocker); +} /** * Starts the bot. diff --git a/src/types/puppeteer-extra-plugin-block-resources.d.ts b/src/types/puppeteer-extra-plugin-block-resources.d.ts new file mode 100644 index 0000000000..a8be065b4e --- /dev/null +++ b/src/types/puppeteer-extra-plugin-block-resources.d.ts @@ -0,0 +1 @@ +declare module 'puppeteer-extra-plugin-block-resources'; diff --git a/src/util.ts b/src/util.ts index b164701f71..ceb22383c8 100644 --- a/src/util.ts +++ b/src/util.ts @@ -42,6 +42,9 @@ export async function usingPage(browser: Browser, cb: (page: Page, browser: B } export async function closePage(page: Page) { - await disableBlockerInPage(page); + if (!Config.browser.lowBandwidth) { + await disableBlockerInPage(page); + } + await page.close(); }