feat: add support for multiple browser user agents (#547)

Co-authored-by: Pawel Grzesik <p.grzesik@francotyp.com>

README.md

@@ -141,7 +141,7 @@ Here is a list of variables that you can use to customize your newly copied `.en
 | `TWITTER_CONSUMER_KEY` | Twitter Consumer Key | Generate all Twitter keys at: https://developer.twitter.com/ |
 | `TWITTER_CONSUMER_SECRET` | Twitter Consumer Secret | |
 | `TWITTER_TWEET_TAGS` | Optional list of hashtags to append to the tweet message | E.g.: `#nvidia #nvidiastock` |
-| `USER_AGENT` | Custom User-Agent header for HTTP requests | Default: `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36` |
+| `USER_AGENT` | Custom User-Agents headers for HTTP requests | Newline separated, E.g.: `USER_AGENT_STRING1 \n USER_AGENT_STRING2` | | Default: `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36` |
 
 > :point_right: If you have multi-factor authentication (MFA), you will need to create an [app password](https://myaccount.google.com/apppasswords) and use this instead of your Gmail password.
 

src/config.ts

@@ -14,7 +14,9 @@ config_({path: path.resolve(__dirname, '../.env')});
  * @param array Default array. If not set, is `[]`.
  */
 function envOrArray(environment: string | undefined, array?: string[]): string[] {
-	return (environment ? environment.split(',') : (array ?? [])).map(s => s.trim());
+	return (environment ? (
+		environment.includes('\n') ? environment.split('\n') : environment.split(',')
+	) : (array ?? [])).map(s => s.trim());
 }
 
 /**
@@ -205,7 +207,7 @@ const page = {
 	inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME),
 	screenshot: envOrBoolean(process.env.SCREENSHOT),
 	timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000),
-	userAgent: envOrString(process.env.USER_AGENT, 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'),
+	userAgents: envOrArray(process.env.USER_AGENT, ['Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36']),
 	width: 1920
 };
 

src/store/lookup.ts

@@ -2,7 +2,7 @@ import {Browser, Page, Response} from 'puppeteer';
 import {Link, Store} from './model';
 import {Print, logger} from '../logger';
 import {Selector, cardPrice, pageIncludesLabels} from './includes-labels';
-import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util';
+import {closePage, delay, getRandomUserAgent, getSleepTime, isStatusCodeInRange} from '../util';
 import {config} from '../config';
 import {disableBlockerInPage} from '../adblocker';
 import {fetchLinks} from './fetch-links';
@@ -38,7 +38,7 @@ async function lookup(browser: Browser, store: Store) {
 		const context = (config.browser.isIncognito ? await browser.createIncognitoBrowserContext() : browser.defaultBrowserContext());
 		const page = (config.browser.isIncognito ? await context.newPage() : await browser.newPage());
 		page.setDefaultNavigationTimeout(config.page.timeout);
-		await page.setUserAgent(config.page.userAgent);
+		await page.setUserAgent(getRandomUserAgent());
 
 		if (store.disableAdBlocker) {
 			try {

src/util.ts

@@ -48,7 +48,7 @@ export async function usingResponse<T>(
 export async function usingPage<T>(browser: Browser, cb: (page: Page, browser: Browser) => Promise<T>): Promise<T> {
 	const page = await browser.newPage();
 	page.setDefaultNavigationTimeout(config.page.timeout);
-	await page.setUserAgent(config.page.userAgent);
+	await page.setUserAgent(getRandomUserAgent());
 
 	try {
 		return await cb(page, browser);
@@ -68,3 +68,7 @@ export async function closePage(page: Page) {
 
 	await page.close();
 }
+
+export function getRandomUserAgent(): string {
+	return config.page.userAgents[Math.floor(Math.random() * config.page.userAgents.length)];
+}