diff --git a/.gitignore b/.gitignore index 608b34d07e..3d499735a2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .vscode/ build/ node_modules/ +src/config/*.yaml .env dotenv diff --git a/src/config.ts b/src/config.ts index da5adf8712..240cd15b6b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -173,7 +173,8 @@ const browser = { process.env.PAGE_SLEEP_MAX, 5000 ), - open: envOrBoolean(process.env.OPEN_BROWSER) + open: envOrBoolean(process.env.OPEN_BROWSER), + userAgent: '' }; const docker = envOrBoolean(process.env.DOCKER); diff --git a/src/index.ts b/src/index.ts index 5023e3ed72..0293392b83 100644 --- a/src/index.ts +++ b/src/index.ts @@ -48,6 +48,8 @@ async function main() { headless: config.browser.isHeadless }); + config.browser.userAgent = await browser.userAgent(); + for (const store of storeList.values()) { logger.debug('store links', {meta: {links: store.links}}); if (store.setupAction !== undefined) { diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 0b2c062286..356ae3ea7c 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -163,7 +163,7 @@ async function lookup(browser: Browser, store: Store) { await page.setRequestInterception(true); page.setDefaultNavigationTimeout(config.page.timeout); - await page.setUserAgent(await getRandomUserAgent(browser)); + await page.setUserAgent(await getRandomUserAgent()); let adBlockRequestHandler: any; let pageProxy; diff --git a/src/util.ts b/src/util.ts index 0758bd21f7..a38cdfefd8 100644 --- a/src/util.ts +++ b/src/util.ts @@ -58,7 +58,7 @@ export async function usingPage( ): Promise { const page = await browser.newPage(); page.setDefaultNavigationTimeout(config.page.timeout); - await page.setUserAgent(await getRandomUserAgent(browser)); + await page.setUserAgent(await getRandomUserAgent()); try { return await cb(page, browser); @@ -79,11 +79,24 @@ export async function closePage(page: Page) { await page.close(); } -export async function getRandomUserAgent(browser: Browser): Promise { +export async function getRandomUserAgent(): Promise { + const deprecatedUserAgent = (process.env.USER_AGENT + ? process.env.USER_AGENT.includes('\n') + ? process.env.USER_AGENT.split('\n') + : process.env.USER_AGENT.split(',') + : [] + ).map((s) => s.trim()); + + if (deprecatedUserAgent.length > 0) { + return deprecatedUserAgent[ + Math.floor(Math.random() * deprecatedUserAgent.length) + ]; + } + const userAgent = getRandom((ua) => { return ua.browserName === 'Chrome' && ua.browserVersion > '20'; - }) ?? (await browser.userAgent()); + }) ?? config.browser.userAgent; logger.debug('user agent', userAgent);