diff --git a/.env-example b/.env-example index b52a73ba26..939a35b8cf 100644 --- a/.env-example +++ b/.env-example @@ -16,3 +16,5 @@ STORES="bestbuy,bandh,nvidia" SCREENSHOT="true" TELEGRAM_ACCESS_TOKEN="" TELEGRAM_CHAT_ID="1234" +HEADLESS="true" +LOG_LEVEL="info" \ No newline at end of file diff --git a/README.md b/README.md index fb7203ad7b..9270593a27 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,8 @@ Here is a list of variables that you can use to customize your newly copied `.en |:---:|---|---| | `EMAIL_USERNAME` | Gmail address | E.g.: `jensen.robbed.us@gmail.com` | | `EMAIL_PASSWORD` | Gmail password | See below if you have MFA | +| `HEADLESS` | Puppeteer to run headless or not (Debugging Purposes) | +| `LOG_LEVEL` | [Logging levels](https://github.com/winstonjs/winston#logging-levels) (Debugging Purposes) | | `NOTIFICATION_TEST` | Test all the notifications configured | Default: `false` | | `OPEN_BROWSER` | Toggle for whether or not the browser should open when item is found | Default: `true` | | `PAGE_TIMEOUT` | Navigation Timeout in milliseconds | `0` for infinite, default: `30000` | @@ -80,7 +82,6 @@ Here is a list of variables that you can use to customize your newly copied `.en | `SCREENSHOT` | Capture screenshot of page if a card is found | Default: `true` | | `TELEGRAM_ACCESS_TOKEN` | Telegram access token | | `TELEGRAM_CHAT_ID` | Telegram chat ID | - > :point_right: If you have multi-factor authentication (MFA), you will need to create an [app password](https://myaccount.google.com/apppasswords) and use this instead of your Gmail password. #### Supported stores diff --git a/package-lock.json b/package-lock.json index c6c42730a0..a08070da0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -123,6 +123,45 @@ "to-fast-properties": "^2.0.0" } }, + "@cliqz/adblocker": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/@cliqz/adblocker/-/adblocker-1.18.3.tgz", + "integrity": "sha512-fkGky+ffAsXw9WIS+cV9zm8EMzdjRKU/uO196yCFHYICByZyREBie3lMNNKQ6RVSUeEVFOx1JlEKkY9Bze/9xQ==", + "requires": { + "@remusao/guess-url-type": "^1.1.2", + "@remusao/small": "^1.1.2", + "@remusao/smaz": "^1.7.1", + "@types/chrome": "^0.0.123", + "@types/firefox-webext-browser": "^78.0.0", + "tldts-experimental": "^5.6.21" + }, + "dependencies": { + "@types/chrome": { + "version": "0.0.123", + "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.123.tgz", + "integrity": "sha512-fG6GPreuSY+Z+0e3dtBz5MJ5qyZ2feOZISG8udxBiuwUYqykK1q4NxkjfzL2F5I05LqK2ojP7ZR08Gcfo3ubHQ==", + "requires": { + "@types/filesystem": "*", + "@types/har-format": "*" + } + } + } + }, + "@cliqz/adblocker-content": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/@cliqz/adblocker-content/-/adblocker-content-1.18.3.tgz", + "integrity": "sha512-mCLlGg4B8P2VWtJpSAJStR9HeRNt5Jo4D0MIOdXIkdSFjCWcXUSwqlUtu5GBvA8iFp9cGgHC/EYeyUW1SbuvYg==" + }, + "@cliqz/adblocker-puppeteer": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/@cliqz/adblocker-puppeteer/-/adblocker-puppeteer-1.18.3.tgz", + "integrity": "sha512-2JkMzGeC2+s2t7oZHZLiBt7b3RTyo5kC3Uewih9CapRo/3xWUugIqBvmik0Q+Pr1/DE3x7YB36GuyLwxbt1yZg==", + "requires": { + "@cliqz/adblocker": "^1.18.3", + "@cliqz/adblocker-content": "^1.18.3", + "tldts-experimental": "^5.6.21" + } + }, "@dabh/diagnostics": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", @@ -167,6 +206,43 @@ "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==", "dev": true }, + "@remusao/guess-url-type": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@remusao/guess-url-type/-/guess-url-type-1.2.0.tgz", + "integrity": "sha512-alnTonifD/Ii/0pI9EA5nVgdk/eOihU4OOYMIXq4U4cS0NocnaYCozqV4OVkmArPPnz9s4ap4GM1ODftBpBW0w==" + }, + "@remusao/small": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@remusao/small/-/small-1.2.0.tgz", + "integrity": "sha512-18Bwa/EjqQ5WfdERqmG3YgOohO7J2sS8+v31JgmYnEg3wAtcAOPVBRkD24IzVS0eJOQk1P2Yd++aP0ldirk7MQ==" + }, + "@remusao/smaz": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@remusao/smaz/-/smaz-1.9.0.tgz", + "integrity": "sha512-HMMPam5jLhP0ymtMUQ8sm2p9zwDJwHD09krORXN/l/TR+NlSCdU2gSAoVNr9idD9OmMGfeXPFQYCofEUZfjbTQ==", + "requires": { + "@remusao/smaz-compress": "^1.9.0", + "@remusao/smaz-decompress": "^1.9.0" + } + }, + "@remusao/smaz-compress": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@remusao/smaz-compress/-/smaz-compress-1.9.0.tgz", + "integrity": "sha512-PAze3aYCcUfX+a6E6sVMoxVtUkeQgX+oiY6DqbiRkNtUqzjtcl9JVyEAWGbBEgOuv2jdEATAlyIf0W18NKDEnw==", + "requires": { + "@remusao/trie": "^1.4.0" + } + }, + "@remusao/smaz-decompress": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@remusao/smaz-decompress/-/smaz-decompress-1.9.0.tgz", + "integrity": "sha512-7uXEX8cSMWy+ai7j8sJpVQuY+CHj2e5D+PjxY//4wbAJlw1a/X+CYPt7BuxLBzpVoioB5Y7++1USjCkrw0pl8g==" + }, + "@remusao/trie": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@remusao/trie/-/trie-1.4.0.tgz", + "integrity": "sha512-mIr0m4/xj6qxHtJjAFb4I8tXXmjTniUYTB2Hv+xK5hXf/YWocEPlJ+V31bv5HJwo6ly64DUnZDBeBxolT3WE7w==" + }, "@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", @@ -224,6 +300,20 @@ "defer-to-connect": "^1.0.1" } }, + "@types/async": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@types/async/-/async-3.2.3.tgz", + "integrity": "sha512-deXFjLZc1h6SOh3hicVgD+S2EAkhSBGX/vdlD4nTzCjjOFQ+bfNiXocQ21xJjFAUwqaCeyvOQMgrnbg4QEV63A==", + "dev": true + }, + "@types/chrome": { + "version": "0.0.91", + "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.91.tgz", + "integrity": "sha512-vNvo9lJkp1AvViWrUwe1bxhoMwr5dRZWlgr1DTuaNkz97LsG56lDX1sceWeZir2gRACJ5vdHtoRdVAvm8C75Ug==", + "requires": { + "@types/filesystem": "*" + } + }, "@types/color-name": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", @@ -241,6 +331,24 @@ "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", "dev": true }, + "@types/filesystem": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/filesystem/-/filesystem-0.0.29.tgz", + "integrity": "sha512-85/1KfRedmfPGsbK8YzeaQUyV1FQAvMPMTuWFQ5EkLd2w7szhNO96bk3Rh/SKmOfd9co2rCLf0Voy4o7ECBOvw==", + "requires": { + "@types/filewriter": "*" + } + }, + "@types/filewriter": { + "version": "0.0.28", + "resolved": "https://registry.npmjs.org/@types/filewriter/-/filewriter-0.0.28.tgz", + "integrity": "sha1-wFTor02d11205jq8dviFFocU1LM=" + }, + "@types/firefox-webext-browser": { + "version": "78.0.1", + "resolved": "https://registry.npmjs.org/@types/firefox-webext-browser/-/firefox-webext-browser-78.0.1.tgz", + "integrity": "sha512-0d7oiI9K6Y4efP4Crl3JB88zYl7vaRdLtumqz8v6axMF8RCnK0NaGUjL4DnyQ7GLPo98b+s0BSRalaxAXgvPAQ==" + }, "@types/glob": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz", @@ -251,6 +359,11 @@ "@types/node": "*" } }, + "@types/har-format": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.4.tgz", + "integrity": "sha512-iUxzm1meBm3stxUMzRqgOVHjj4Kgpgu5w9fm4X7kPRfSgVRzythsucEN7/jtOo8SQzm+HfcxWWzJS0mJDH/3DQ==" + }, "@types/is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz", @@ -325,7 +438,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-3.0.2.tgz", "integrity": "sha512-JRuHPSbHZBadOxxFwpyZPeRlpPTTeMbQneMdpFd8LXdyNfFSiX950CGewdm69g/ipzEAXAmMyFF1WOWJOL/nKw==", - "dev": true, "requires": { "@types/node": "*" } @@ -526,8 +638,7 @@ "arr-union": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", - "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=", - "dev": true + "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=" }, "array-find": { "version": "1.0.0", @@ -1174,6 +1285,28 @@ "integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==", "dev": true }, + "clone-deep": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz", + "integrity": "sha1-TnPdCen7lxzDhnDF3O2cGJZIHMY=", + "requires": { + "for-own": "^0.1.3", + "is-plain-object": "^2.0.1", + "kind-of": "^3.0.2", + "lazy-cache": "^1.0.3", + "shallow-clone": "^0.1.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, "clone-response": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", @@ -1485,6 +1618,11 @@ "core-assert": "^0.2.0" } }, + "deepmerge": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz", + "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==" + }, "defer-to-connect": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", @@ -2854,8 +2992,15 @@ "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", - "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=", - "dev": true + "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=" + }, + "for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha1-UmXGgaTylNq78XyVCbZ2OqhFEM4=", + "requires": { + "for-in": "^1.0.1" + } }, "form-data": { "version": "2.5.1", @@ -3297,8 +3442,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "dev": true + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "is-callable": { "version": "1.2.1", @@ -3374,8 +3518,7 @@ "is-extendable": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", - "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=", - "dev": true + "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=" }, "is-extglob": { "version": "2.1.1", @@ -3497,7 +3640,6 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", - "dev": true, "requires": { "isobject": "^3.0.1" } @@ -3599,8 +3741,7 @@ "isobject": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", - "dev": true + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" }, "js-tokens": { "version": "4.0.0", @@ -3701,6 +3842,11 @@ "package-json": "^6.3.0" } }, + "lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=" + }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -3991,6 +4137,26 @@ } } }, + "merge-deep": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.2.tgz", + "integrity": "sha512-T7qC8kg4Zoti1cFd8Cr0M+qaZfOwjlPDEdZIIPPB2JZctjaPM4fX+i7HOId69tAti2fvO6X5ldfYUONDODsrkA==", + "requires": { + "arr-union": "^3.1.0", + "clone-deep": "^0.2.4", + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, "merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -4197,6 +4363,22 @@ } } }, + "mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha1-T7lJRB2rGCVA8f4DW6YOGUel5X4=", + "requires": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "dependencies": { + "for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha1-2Hc5COMSVhCZUrH9ubP6hn0ndeE=" + } + } + }, "mkdirp": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", @@ -4256,6 +4438,11 @@ "tslib": "^1.10.0" } }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, "node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -4912,6 +5099,48 @@ "ws": "^7.2.3" } }, + "puppeteer-extra": { + "version": "3.1.15", + "resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.1.15.tgz", + "integrity": "sha512-TFKcluoNSYCT3xmZjDcqOkBuxZePbwvaL5mrW5Gvp5c9QsJEei5TYixoenMQaB3QZuRW0Aura4yyjVrJDNlWFA==", + "requires": { + "@types/debug": "^4.1.0", + "@types/puppeteer": "*", + "debug": "^4.1.1", + "deepmerge": "^4.2.2" + } + }, + "puppeteer-extra-plugin": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.1.7.tgz", + "integrity": "sha512-In3o89X06Q4w1wt0RsAvvdRIp7BzWttVHh24srzlfcBvsGq+zCqhPuwbOYbxnwemtNLaiD/hcYGLvNMEqeUo/Q==", + "requires": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "merge-deep": "^3.0.1" + } + }, + "puppeteer-extra-plugin-adblocker": { + "version": "2.11.6", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.11.6.tgz", + "integrity": "sha512-tlCmcsdfRr57lzP1tT07hsEHGtv9pYO+1rzV1AtzX3+Tci1M4Y3ANwjrrYCiAx5Sbf5MF6apqaofBvTZBwfPbg==", + "requires": { + "@cliqz/adblocker-puppeteer": "^1.4.0", + "@types/chrome": "0.0.91", + "debug": "^4.1.1", + "node-fetch": "^2.6.0", + "puppeteer-extra-plugin": "^3.1.7" + } + }, + "puppeteer-extra-plugin-stealth": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.6.1.tgz", + "integrity": "sha512-KPoqjvYdnX8BpcjuXZKethad11WY1UfVzckSmOpBCCtdYic1s6esPhenLTvBEfZQ5XAT61yLK8jQtNgkML0QVg==", + "requires": { + "debug": "^4.1.1", + "puppeteer-extra-plugin": "^3.1.7" + } + }, "pushover-notifications": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/pushover-notifications/-/pushover-notifications-1.2.2.tgz", @@ -5297,6 +5526,32 @@ "safe-buffer": "^5.0.1" } }, + "shallow-clone": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", + "integrity": "sha1-WQnodLp3EG1zrEFM/sH/yofZcGA=", + "requires": { + "is-extendable": "^0.1.1", + "kind-of": "^2.0.1", + "lazy-cache": "^0.2.3", + "mixin-object": "^2.0.1" + }, + "dependencies": { + "kind-of": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "integrity": "sha1-AY7HpM5+OobLkUG+UZ0kyPqpgbU=", + "requires": { + "is-buffer": "^1.0.2" + } + }, + "lazy-cache": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz", + "integrity": "sha1-f+3fLctu23fRHvHRF6tf/fCrG2U=" + } + } + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -5862,6 +6117,19 @@ "setimmediate": "^1.0.4" } }, + "tldts-core": { + "version": "5.6.55", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.6.55.tgz", + "integrity": "sha512-xAUqjkPfwfnCoRxds3xrMi6j1nuR1F4iuB9dqE30sgi0P+7c4NfnbwQTk80Vu1CL15ZyRZeZ+yAwWo2rWxkC9w==" + }, + "tldts-experimental": { + "version": "5.6.55", + "resolved": "https://registry.npmjs.org/tldts-experimental/-/tldts-experimental-5.6.55.tgz", + "integrity": "sha512-GKfOgP1XxHgVcGp9fIA/dPzr2cSUpS5hCpJBvkpTZ8hwOj5UX7mSUSO8tag7udWdmIDh6FdJhIPCzmDaM9n7WQ==", + "requires": { + "tldts-core": "^5.6.55" + } + }, "to-absolute-glob": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/to-absolute-glob/-/to-absolute-glob-2.0.2.tgz", diff --git a/package.json b/package.json index 67ef98312f..f5b4f78b96 100644 --- a/package.json +++ b/package.json @@ -21,16 +21,21 @@ }, "homepage": "https://github.com/jef/nvidia-snatcher#readme", "dependencies": { + "async": "^3.2.0", "dotenv": "^8.2.0", "messaging-api-telegram": "^1.0.0", "nodemailer": "^6.4.11", "open": "^7.2.1", "puppeteer": "^5.3.0", + "puppeteer-extra": "^3.1.15", + "puppeteer-extra-plugin-adblocker": "^2.11.6", + "puppeteer-extra-plugin-stealth": "^2.6.1", "pushover-notifications": "^1.2.2", "winston": "^3.3.3" }, "devDependencies": { "@slack/web-api": "^5.12.0", + "@types/async": "^3.2.3", "@types/node": "^14.11.1", "@types/nodemailer": "^6.4.0", "@types/puppeteer": "^3.0.2", diff --git a/src/config.ts b/src/config.ts index bb895a830f..9c138ac4a5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -34,27 +34,36 @@ const notifications = { accessToken: process.env.TELEGRAM_ACCESS_TOKEN ?? '', chatId: process.env.TELEGRAM_CHAT_ID ?? '' }, - test: process.env.NOTIFICATION_TEST ?? 'false' + test: process.env.NOTIFICATION_TEST === 'true' }; const page = { - capture: process.env.SCREENSHOT ?? 'true', + capture: process.env.SCREENSHOT === 'true', width: 1920, height: 1080, navigationTimeout: Number(process.env.PAGE_TIMEOUT) ?? 30000, userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36' }; -const openBrowser = process.env.OPEN_BROWSER ?? 'true'; const rateLimitTimeout = Number(process.env.RATE_LIMIT_TIMEOUT) ?? 5000; + const stores = process.env.STORES ? process.env.STORES.split(',') : ['nvidia']; + +const openBrowser = process.env.OPEN_BROWSER === 'true'; + +const isHeadless = process.env.HEADLESS === 'true'; + const showOnlyBrands = process.env.SHOW_ONLY_BRANDS ? process.env.SHOW_ONLY_BRANDS.split(',') : []; +const logLevel = process.env.LOG_LEVEL ?? 'info'; + export const Config = { + isHeadless, + logLevel, notifications, - rateLimitTimeout, - page, - stores, openBrowser, - showOnlyBrands + page, + rateLimitTimeout, + showOnlyBrands, + stores }; diff --git a/src/index.ts b/src/index.ts index d5c6447d70..62ee12b52a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,33 +1,57 @@ +import puppeteer from 'puppeteer-extra'; +import stealthPlugin from 'puppeteer-extra-plugin-stealth'; +import adblockerPlugin from 'puppeteer-extra-plugin-adblocker'; import {Config} from './config'; -import {Stores} from './store/model'; +import {Store, Stores} from './store/model'; import {Logger} from './logger'; import {sendNotification} from './notification'; import {lookup} from './store'; -import puppeteer from 'puppeteer'; +import async from 'async'; + +puppeteer.use(stealthPlugin()); +puppeteer.use(adblockerPlugin({blockTrackers: true})); /** * Starts the bot. */ async function main() { - const results = []; - const browser = await puppeteer.launch(); + const browser = await puppeteer.launch({ + headless: Config.isHeadless, + defaultViewport: { + height: Config.page.height, + width: Config.page.width + } + }); + + const q = async.queue(async (store: Store, cb) => { + setTimeout(async () => { + try { + Logger.debug(`↗ Scraping Initialized - ${store.name}`); + await lookup(browser, store); + } catch (error) { + // Ignoring errors; more than likely due to rate limits + Logger.error(error); + } finally { + cb(); + q.push(store); + } + }, Config.rateLimitTimeout); + }, Stores.length); for (const store of Stores) { Logger.debug(store.links); - results.push(lookup(browser, store)); + q.push(store); } - await Promise.all(results); - await browser.close(); + await q.drain(); - Logger.info('↗ trying stores again'); - setTimeout(main, Config.rateLimitTimeout); + await browser.close(); } /** * Send test email. */ -if (Config.notifications.test === 'true') { +if (Config.notifications.test) { sendNotification('test'); } diff --git a/src/logger.ts b/src/logger.ts index 220921bb20..88c1d07579 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,4 +1,5 @@ import winston, {format} from 'winston'; +import {Config} from './config'; const prettyJson = format.printf(info => { const timestamp = new Date().toLocaleTimeString(); @@ -11,7 +12,7 @@ const prettyJson = format.printf(info => { }); export const Logger = winston.createLogger({ - level: process.env.LOG_LEVEL ?? 'info', + level: Config.logLevel, format: format.combine( format.colorize(), format.prettyPrint(), diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 6fc0b5a91b..aa4f46e001 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -1,4 +1,4 @@ -import puppeteer from 'puppeteer'; +import {Browser, Response} from 'puppeteer'; import {Config} from '../config'; import {Logger} from '../logger'; import open from 'open'; @@ -23,11 +23,10 @@ function filterBrand(brand: string) { * Responsible for looking up information about a each product within * a `Store`. It's important that we ignore `no-await-in-loop` here * because we don't want to get rate limited within the same store. - * - * @param browser Current browser in use. + * @param browser Puppeteer browser. * @param store Vendor of graphics cards. */ -export async function lookup(browser: puppeteer.Browser, store: Store) { +export async function lookup(browser: Browser, store: Store) { /* eslint-disable no-await-in-loop */ for (const link of store.links) { if (!filterBrand(link.brand)) { @@ -37,14 +36,10 @@ export async function lookup(browser: puppeteer.Browser, store: Store) { const page = await browser.newPage(); page.setDefaultNavigationTimeout(Config.page.navigationTimeout); await page.setUserAgent(Config.page.userAgent); - await page.setViewport({ - height: Config.page.height, - width: Config.page.width - }); const graphicsCard = `${link.brand} ${link.model}`; - let response: puppeteer.Response | null; + let response: Response | null; try { response = await page.goto(link.url, {waitUntil: 'networkidle0'}); } catch { @@ -68,14 +63,14 @@ export async function lookup(browser: puppeteer.Browser, store: Store) { Logger.info(`🚀🚀🚀 [${store.name}] ${graphicsCard} IN STOCK 🚀🚀🚀`); Logger.info(link.url); - if (Config.page.capture === 'true') { + if (Config.page.capture) { Logger.debug('ℹ saving screenshot'); await page.screenshot({path: `success-${Date.now()}.png`}); } const givenUrl = link.cartUrl ? link.cartUrl : link.url; - if (Config.openBrowser === 'true') { + if (Config.openBrowser) { await open(givenUrl); } diff --git a/src/store/model/index.ts b/src/store/model/index.ts index fb842be48a..4728be9e1b 100644 --- a/src/store/model/index.ts +++ b/src/store/model/index.ts @@ -9,7 +9,7 @@ import {Evga} from './evga'; import {MicroCenter} from './microcenter'; import {NewEgg} from './newegg'; import {Nvidia} from './nvidia'; - +import {Store} from './store'; const masterList = new Map([ [Amazon.name, Amazon], @@ -29,6 +29,6 @@ for (const name of Config.stores) { list.set(name, masterList.get(name)); } -export const Stores = Array.from(list.values()); +export const Stores = Array.from(list.values()) as Store[]; export * from './store';