-
-
Notifications
You must be signed in to change notification settings - Fork 1.3k
/
index.ts
70 lines (62 loc) · 1.65 KB
/
index.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import puppeteer from 'puppeteer-extra';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import adblockerPlugin from 'puppeteer-extra-plugin-adblocker';
import {Config} from './config';
import {Store, Stores} from './store/model';
import {Logger} from './logger';
import {sendNotification} from './notification';
import {lookup} from './store';
import async from 'async';
puppeteer.use(stealthPlugin());
puppeteer.use(adblockerPlugin({blockTrackers: true}));
/**
* Starts the bot.
*/
async function main() {
const browser = await puppeteer.launch({
headless: Config.browser.isHeadless,
defaultViewport: {
height: Config.page.height,
width: Config.page.width
}
});
const q = async.queue<Store>(async (store: Store, cb) => {
setTimeout(async () => {
try {
Logger.debug(`↗ scraping initialized - ${store.name}`);
await lookup(browser, store);
} catch (error) {
// Ignoring errors; more than likely due to rate limits
Logger.error(error);
} finally {
cb();
q.push(store);
}
}, Config.browser.rateLimitTimeout);
}, Stores.length);
for (const store of Stores) {
Logger.debug(store.links);
q.push(store);
if (Stores.length === 1) {
q.push(store);
} // Keep from completely draining
}
await q.drain();
await browser.close();
}
/**
* Send test email.
*/
if (Config.notifications.test) {
sendNotification('http://test.com/', {brand: 'THE BEST BRAND', model: 'VENTUS', oosLabels: [], url: '', cartUrl: ''});
}
/**
* Will continually run until user interferes.
*/
try {
void main();
} catch (error) {
// Ignoring errors; more than likely due to rate limits
Logger.error(error);
void main();
}