diff --git a/.env-example b/.env-example index 9073ced1dc..ad6e4252df 100644 --- a/.env-example +++ b/.env-example @@ -111,5 +111,4 @@ TWITTER_ACCESS_TOKEN_SECRET="" TWITTER_CONSUMER_KEY="" TWITTER_CONSUMER_SECRET="" TWITTER_TWEET_TAGS="" -USER_AGENT="" WEB_PORT="" diff --git a/docs/help/troubleshoot.md b/docs/help/troubleshoot.md index 7bfb91f8cf..dfa806cd7c 100644 --- a/docs/help/troubleshoot.md +++ b/docs/help/troubleshoot.md @@ -4,19 +4,11 @@ ### Option 1 -If you're running into _a lot_ of captcha problems, be sure to update your user agent by searching ["what's my user agent" on Google](https://www.google.com/search?q=whats+my+user+agent). - -![image](https://user-images.githubusercontent.com/12074633/101272427-07a88100-375a-11eb-9cb3-4e8783db6ae5.png) - -You can update your user agent by using `USER_AGENT="your-result"`. - -### Option 2 - -If you're _still_ running into problems, try running in headful mode: `HEADLESS="false"`. +If you're running into problems, try running in headful mode: `HEADLESS="false"`. This will open a browser and run streetmerchant. Note that this isn't a great solution for those running in a headless environment, i.e.: VPS, cloud, docker. Instead, it would be a good solution for those running on separate computer that won't be blocked by running in the background. -### Option 3 +### Option 2 As a last case scenario, use `PUPPETEER_EXECUTABLE_PATH`. This will use your computer's Chrome browser. You can run this is headless or headful mode. diff --git a/docs/reference/application.md b/docs/reference/application.md index 898246db57..6ad6a93933 100644 --- a/docs/reference/application.md +++ b/docs/reference/application.md @@ -19,7 +19,6 @@ | `PROXY_ADDRESS` | IP Address or fqdn of proxy server | | `PROXY_PORT` | TCP Port number on which the proxy is listening for connections. Default: `80` | | `SCREENSHOT` | Capture screenshot of page if a card is found. Default: `true` | -| `USER_AGENT` | Custom User-Agents headers for HTTP requests. Newline separated, e.g.: `USER_AGENT_STRING1 \n USER_AGENT_STRING2`. Default: `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36` | | `WEB_PORT` | Starts a webserver to be able to control the bot while it is running. Setting this value starts this service. | ???+ info diff --git a/package-lock.json b/package-lock.json index 2f70655b87..d741969922 100644 --- a/package-lock.json +++ b/package-lock.json @@ -821,6 +821,11 @@ "@types/node": "*" } }, + "@types/random-useragent": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@types/random-useragent/-/random-useragent-0.3.0.tgz", + "integrity": "sha512-K6yHeQCCJJBvHeW1akRxXC8GnADsXmoRA2UdVEYITzD88U8Vk3lX+0c0A27j3CrAiic2I961glcnwScc9vUtWg==" + }, "@types/request": { "version": "2.48.5", "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.5.tgz", @@ -4906,9 +4911,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", + "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==", "dev": true }, "interpret": { @@ -6521,14 +6526,14 @@ "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==" }, "npm": { - "version": "7.0.12", - "resolved": "https://registry.npmjs.org/npm/-/npm-7.0.12.tgz", - "integrity": "sha512-0MDC+wxNRTxp9yNZ1t8zEUjVvKTN8K2xGAQ9Cuert9NwrTk0GDw+8hl4OOVcMVDFSFaHopW3F+w+KnxLGZMwYw==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/npm/-/npm-7.1.1.tgz", + "integrity": "sha512-66kBbZSTTLGrAyNPou2dUoVjAxDBZZiJkMdukGac3tUJf33tq3c86zW0rVKN+j5pUdWyJKyK3DnzEmJIcq06mw==", "requires": { - "@npmcli/arborist": "^1.0.11", + "@npmcli/arborist": "^2.0.0", "@npmcli/ci-detect": "^1.2.0", - "@npmcli/config": "^1.2.1", - "@npmcli/run-script": "^1.8.0", + "@npmcli/config": "^1.2.4", + "@npmcli/run-script": "^1.8.1", "abbrev": "~1.1.1", "ansicolors": "~0.3.2", "ansistyles": "~0.1.3", @@ -6547,20 +6552,20 @@ "graceful-fs": "^4.2.3", "hosted-git-info": "^3.0.6", "inherits": "^2.0.4", - "ini": "^1.3.5", + "ini": "^1.3.6", "init-package-json": "^2.0.1", "is-cidr": "^4.0.2", "leven": "^3.1.0", "libnpmaccess": "^4.0.1", - "libnpmfund": "^1.0.1", + "libnpmfund": "^1.0.2", "libnpmhook": "^6.0.1", "libnpmorg": "^2.0.1", "libnpmpack": "^2.0.0", "libnpmpublish": "^4.0.0", - "libnpmsearch": "^3.0.1", + "libnpmsearch": "^3.1.0", "libnpmteam": "^2.0.2", "libnpmversion": "^1.0.7", - "make-fetch-happen": "^8.0.9", + "make-fetch-happen": "^8.0.12", "mkdirp": "^1.0.4", "mkdirp-infer-owner": "^2.0.0", "ms": "^2.1.2", @@ -6582,7 +6587,7 @@ "read-package-json": "^3.0.0", "read-package-json-fast": "^1.2.1", "rimraf": "^3.0.2", - "semver": "^7.3.2", + "semver": "^7.3.4", "sorted-object": "~2.0.1", "ssri": "^8.0.0", "tar": "^6.0.5", @@ -6595,7 +6600,7 @@ }, "dependencies": { "@npmcli/arborist": { - "version": "1.0.11", + "version": "2.0.0", "bundled": true, "requires": { "@npmcli/installed-package-contents": "^1.0.5", @@ -6603,8 +6608,8 @@ "@npmcli/metavuln-calculator": "^1.0.0", "@npmcli/move-file": "^1.0.1", "@npmcli/name-from-folder": "^1.0.1", - "@npmcli/node-gyp": "^1.0.0", - "@npmcli/run-script": "^1.7.2", + "@npmcli/node-gyp": "^1.0.1", + "@npmcli/run-script": "^1.8.1", "bin-links": "^2.2.1", "cacache": "^15.0.3", "common-ancestor-path": "^1.0.1", @@ -6612,15 +6617,15 @@ "json-stringify-nice": "^1.1.1", "mkdirp-infer-owner": "^2.0.0", "npm-install-checks": "^4.0.0", - "npm-package-arg": "^8.0.0", + "npm-package-arg": "^8.1.0", "npm-pick-manifest": "^6.1.0", - "pacote": "^11.1.10", + "pacote": "^11.1.13", "parse-conflict-json": "^1.1.1", "promise-all-reject-late": "^1.0.0", "promise-call-limit": "^1.0.1", "read-package-json-fast": "^1.2.1", "readdir-scoped-modules": "^1.1.0", - "semver": "^7.1.2", + "semver": "^7.3.4", "treeverse": "^1.0.4", "walk-up-path": "^1.0.0" } @@ -6630,12 +6635,13 @@ "bundled": true }, "@npmcli/config": { - "version": "1.2.1", + "version": "1.2.4", "bundled": true, "requires": { "ini": "^1.3.5", "mkdirp-infer-owner": "^2.0.0", "nopt": "^5.0.0", + "semver": "^7.3.4", "walk-up-path": "^1.0.0" } }, @@ -6706,7 +6712,7 @@ } }, "@npmcli/run-script": { - "version": "1.8.0", + "version": "1.8.1", "bundled": true, "requires": { "@npmcli/node-gyp": "^1.0.0", @@ -6818,7 +6824,7 @@ "bundled": true }, "aws4": { - "version": "1.10.1", + "version": "1.11.0", "bundled": true }, "balanced-match": { @@ -7024,10 +7030,16 @@ } }, "debug": { - "version": "4.2.0", + "version": "4.3.1", "bundled": true, "requires": { "ms": "2.1.2" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "bundled": true + } } }, "debuglog": { @@ -7133,6 +7145,10 @@ "version": "1.0.0", "bundled": true }, + "function-bind": { + "version": "1.1.1", + "bundled": true + }, "gauge": { "version": "2.7.4", "bundled": true, @@ -7204,6 +7220,13 @@ "har-schema": "^2.0.0" } }, + "has": { + "version": "1.0.3", + "bundled": true, + "requires": { + "function-bind": "^1.1.1" + } + }, "has-flag": { "version": "4.0.0", "bundled": true @@ -7296,7 +7319,7 @@ "bundled": true }, "ini": { - "version": "1.3.5", + "version": "1.3.6", "bundled": true }, "init-package-json": { @@ -7328,6 +7351,13 @@ "cidr-regex": "^3.1.1" } }, + "is-core-module": { + "version": "2.2.0", + "bundled": true, + "requires": { + "has": "^1.0.3" + } + }, "is-fullwidth-code-point": { "version": "2.0.0", "bundled": true @@ -7413,10 +7443,10 @@ } }, "libnpmfund": { - "version": "1.0.1", + "version": "1.0.2", "bundled": true, "requires": { - "@npmcli/arborist": "^0.0.33 || ^1.x" + "@npmcli/arborist": "^2.0.0" } }, "libnpmhook": { @@ -7456,7 +7486,7 @@ } }, "libnpmsearch": { - "version": "3.0.1", + "version": "3.1.0", "bundled": true, "requires": { "npm-registry-fetch": "^9.0.0" @@ -7489,21 +7519,21 @@ } }, "make-fetch-happen": { - "version": "8.0.10", + "version": "8.0.12", "bundled": true, "requires": { - "agentkeepalive": "^4.1.0", - "cacache": "^15.0.0", - "http-cache-semantics": "^4.0.4", + "agentkeepalive": "^4.1.3", + "cacache": "^15.0.5", + "http-cache-semantics": "^4.1.0", "http-proxy-agent": "^4.0.1", "https-proxy-agent": "^5.0.0", "is-lambda": "^1.0.1", "lru-cache": "^6.0.0", "minipass": "^3.1.3", "minipass-collect": "^1.0.2", - "minipass-fetch": "^1.3.0", + "minipass-fetch": "^1.3.2", "minipass-flush": "^1.0.5", - "minipass-pipeline": "^1.2.2", + "minipass-pipeline": "^1.2.4", "promise-retry": "^1.1.1", "socks-proxy-agent": "^5.0.0", "ssri": "^8.0.0" @@ -7602,7 +7632,7 @@ } }, "ms": { - "version": "2.1.2", + "version": "2.1.3", "bundled": true }, "mute-stream": { @@ -7935,6 +7965,14 @@ "uuid": "^3.3.2" }, "dependencies": { + "tough-cookie": { + "version": "2.5.0", + "bundled": true, + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + }, "uuid": { "version": "3.4.0", "bundled": true @@ -7942,9 +7980,10 @@ } }, "resolve": { - "version": "1.17.0", + "version": "1.19.0", "bundled": true, "requires": { + "is-core-module": "^2.1.0", "path-parse": "^1.0.6" } }, @@ -7968,8 +8007,11 @@ "bundled": true }, "semver": { - "version": "7.3.2", - "bundled": true + "version": "7.3.4", + "bundled": true, + "requires": { + "lru-cache": "^6.0.0" + } }, "set-blocking": { "version": "2.0.0", @@ -7984,7 +8026,7 @@ "bundled": true }, "socks": { - "version": "2.4.4", + "version": "2.5.1", "bundled": true, "requires": { "ip": "^1.1.5", @@ -8025,7 +8067,7 @@ } }, "spdx-license-ids": { - "version": "3.0.6", + "version": "3.0.7", "bundled": true }, "sshpk": { @@ -8116,14 +8158,6 @@ "version": "1.3.0", "bundled": true }, - "tough-cookie": { - "version": "2.5.0", - "bundled": true, - "requires": { - "psl": "^1.1.28", - "punycode": "^2.1.1" - } - }, "treeverse": { "version": "1.0.4", "bundled": true @@ -8172,7 +8206,7 @@ "bundled": true }, "uuid": { - "version": "8.3.1", + "version": "8.3.2", "bundled": true }, "validate-npm-package-license": { @@ -9251,6 +9285,22 @@ "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==", "dev": true }, + "random-seed": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/random-seed/-/random-seed-0.3.0.tgz", + "integrity": "sha1-2UXy4fOPSejViRNDG4v2u5N1Vs0=", + "requires": { + "json-stringify-safe": "^5.0.1" + } + }, + "random-useragent": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/random-useragent/-/random-useragent-0.5.0.tgz", + "integrity": "sha512-FUMkqVdZeoSff5tErNL3FFGYXElDWZ1bEuedhm5u9MdCFwANriJWbHvDRYrLTOzp/fBsBGu5J1cWtDgifa97aQ==", + "requires": { + "random-seed": "^0.3.0" + } + }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", diff --git a/package.json b/package.json index 479251af8d..a5af2c0f2b 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "dependencies": { "@jef/pushbullet": "^2.4.3", "@slack/web-api": "^5.14.0", + "@types/random-useragent": "^0.3.0", "chalk": "^4.1.0", "cheerio": "^1.0.0-rc.3", "discord.js": "^12.5.1", @@ -47,6 +48,7 @@ "puppeteer-extra-plugin-stealth": "^2.6.5", "puppeteer-page-proxy": "^1.2.8", "pushover-notifications": "^1.2.2", + "random-useragent": "^0.5.0", "twilio": "^3.53.0", "twitch": "^4.3.5", "twitch-auth": "^4.3.5", diff --git a/src/config.ts b/src/config.ts index cb8cd3758d..a91661bf69 100644 --- a/src/config.ts +++ b/src/config.ts @@ -307,9 +307,6 @@ const page = { inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME), screenshot: envOrBoolean(process.env.SCREENSHOT), timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000), - userAgents: envOrArray(process.env.USER_AGENT, [ - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36' - ]), width: 1920 }; diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 4b0b051e54..6ab6a6159d 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -162,7 +162,7 @@ async function lookup(browser: Browser, store: Store) { const page = await context.newPage(); page.setDefaultNavigationTimeout(config.page.timeout); - await page.setUserAgent(getRandomUserAgent()); + await page.setUserAgent(await getRandomUserAgent(browser)); let adBlockRequestHandler: any; let pageProxy; diff --git a/src/store/model/amazon.ts b/src/store/model/amazon.ts index 2bece1441f..41f9134bb0 100644 --- a/src/store/model/amazon.ts +++ b/src/store/model/amazon.ts @@ -50,7 +50,7 @@ export const Amazon: Store = { brand: 'gigabyte', cartUrl: 'https://www.amazon.com/gp/aws/cart/add.html?ASIN.1=B08NYPLXPJ&Quantity.1=1', - model: 'gaming oc', + model: 'gaming oc pro', series: '3060ti', url: 'https://www.amazon.com/dp/B08NYPLXPJ' }, diff --git a/src/util.ts b/src/util.ts index 78ec9f92c9..0758bd21f7 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2,6 +2,7 @@ import {Browser, Page, Response} from 'puppeteer'; import {StatusCodeRangeArray, Store} from './store/model'; import {config} from './config'; import {disableBlockerInPage} from './adblocker'; +import {getRandom} from 'random-useragent'; import {logger} from './logger'; export function getSleepTime(store: Store) { @@ -57,7 +58,7 @@ export async function usingPage( ): Promise { const page = await browser.newPage(); page.setDefaultNavigationTimeout(config.page.timeout); - await page.setUserAgent(getRandomUserAgent()); + await page.setUserAgent(await getRandomUserAgent(browser)); try { return await cb(page, browser); @@ -78,8 +79,13 @@ export async function closePage(page: Page) { await page.close(); } -export function getRandomUserAgent(): string { - return config.page.userAgents[ - Math.floor(Math.random() * config.page.userAgents.length) - ]; +export async function getRandomUserAgent(browser: Browser): Promise { + const userAgent = + getRandom((ua) => { + return ua.browserName === 'Chrome' && ua.browserVersion > '20'; + }) ?? (await browser.userAgent()); + + logger.debug('user agent', userAgent); + + return userAgent; }