From b868d1a4833a8ec5ac1c79481530d75cd0c4b01e Mon Sep 17 00:00:00 2001 From: Andrew Mackrodt Date: Fri, 25 Sep 2020 23:29:10 +0100 Subject: [PATCH] feat: enhanced lookup behaviour (#270) Co-authored-by: Jef LeCompte --- .env-example | 2 + README.md | 6 + nodemon.json | 8 + package-lock.json | 411 ++++++++++++++++++++++++++++++-- package.json | 11 +- src/adblocker.ts | 4 +- src/config.ts | 2 + src/index.ts | 8 + src/logger.ts | 43 ++++ src/store/fetch-links.ts | 58 +++++ src/store/filter.ts | 2 +- src/store/includes-labels.ts | 82 +++++++ src/store/lookup.ts | 118 ++++++--- src/store/model/helpers/card.ts | 48 ++++ src/store/model/store.ts | 22 +- src/util.ts | 31 ++- tsconfig.json | 2 +- 17 files changed, 788 insertions(+), 70 deletions(-) create mode 100644 nodemon.json create mode 100644 src/store/fetch-links.ts create mode 100644 src/store/model/helpers/card.ts diff --git a/.env-example b/.env-example index bc76bf2415..90143f5960 100644 --- a/.env-example +++ b/.env-example @@ -20,6 +20,8 @@ PUSHBULLET="" PUSHOVER_TOKEN="" PUSHOVER_USER="" PUSHOVER_PRIORITY="" +PAGE_BACKOFF_MIN="" +PAGE_BACKOFF_MAX="" PAGE_SLEEP_MIN="" PAGE_SLEEP_MAX="" SHOW_ONLY_BRANDS="" diff --git a/README.md b/README.md index 14848baab2..3ad8418949 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,10 @@ At any point you want the program to stop, use Ctrl + C. > :point_right: Please visit the [wiki](https://github.com/jef/nvidia-snatcher/wiki) if you need more help with installation. +### Developer notes + +The command `npm run dev` can be used instead of `npm run start` to automatically restart the project when filesystem changes are detected in the `src/` folder or `.env` file. + ### Customization To customize `nvidia-snatcher`, make a copy of `.env-example` as `.env` and make any changes to your liking. _All environment variables are **optional**._ @@ -83,6 +87,8 @@ Here is a list of variables that you can use to customize your newly copied `.en | `PUSHOVER_TOKEN` | Pushover access token | Generate at https://pushover.net/apps/build | | | `PUSHOVER_USER` | Pushover username | | | `PUSHOVER_PRIORITY` | Pushover message priority | +| `PAGE_BACKOFF_MIN` | Minimum backoff time between retrying requests for the same store when a forbidden response is received | Default: `10000` | +| `PAGE_BACKOFF_MAX` | Maximum backoff time between retrying requests for the same store when a forbidden response is received | Default: `3600000` | | `PAGE_SLEEP_MIN` | Minimum sleep time between queries of the same store | In milliseconds, default: `5000` | | `PAGE_SLEEP_MAX` | Maximum sleep time between queries of the same store | In milliseconds, default: `10000` | | `SCREENSHOT` | Capture screenshot of page if a card is found | Default: `true` | diff --git a/nodemon.json b/nodemon.json new file mode 100644 index 0000000000..30d6b818da --- /dev/null +++ b/nodemon.json @@ -0,0 +1,8 @@ +{ + "exec": "ts-node --files ./src/index", + "ext": "ts", + "watch": [ + "src/", + ".env" + ] +} diff --git a/package-lock.json b/package-lock.json index 8d943fd928..f83c92e333 100644 --- a/package-lock.json +++ b/package-lock.json @@ -313,7 +313,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/@slack/logger/-/logger-2.0.0.tgz", "integrity": "sha512-OkIJpiU2fz6HOJujhlhfIGrc8hB4ibqtf7nnbJQDerG0BqwZCfmgtK5sWzZ0TkXVRBKD5MpLrTmCYyMxoMCgPw==", - "dev": true, "requires": { "@types/node": ">=8.9.0" } @@ -321,14 +320,12 @@ "@slack/types": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@slack/types/-/types-1.9.0.tgz", - "integrity": "sha512-RmwgMWqOtzd2JPXdiaD/tyrDD0vtjjRDFdxN1I3tAxwBbg4aryzDUVqFc8na16A+3Xik/UN8X1hvVTw8J4EB9w==", - "dev": true + "integrity": "sha512-RmwgMWqOtzd2JPXdiaD/tyrDD0vtjjRDFdxN1I3tAxwBbg4aryzDUVqFc8na16A+3Xik/UN8X1hvVTw8J4EB9w==" }, "@slack/web-api": { "version": "5.12.0", "resolved": "https://registry.npmjs.org/@slack/web-api/-/web-api-5.12.0.tgz", "integrity": "sha512-ygSnNHVid7PltGo7W36f2SNVHyliemkzxn9uSwgnWNF7CHmWBKWAylU/eoDml9l5K7akMOxbousiurOw4XqOFg==", - "dev": true, "requires": { "@slack/logger": ">=1.0.0 <3.0.0", "@slack/types": "^1.7.0", @@ -346,8 +343,7 @@ "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", - "dev": true + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" } } }, @@ -372,6 +368,15 @@ "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==", "dev": true }, + "@types/cheerio": { + "version": "0.22.22", + "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.22.tgz", + "integrity": "sha512-05DYX4zU96IBfZFY+t3Mh88nlwSMtmmzSYaQkKN48T495VV1dkHSah6qYyDTN5ngaS0i0VonH37m+RuzSM0YiA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/chrome": { "version": "0.0.91", "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.91.tgz", @@ -433,7 +438,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz", "integrity": "sha512-jkZatu4QVbR60mpIzjINmtS1ZF4a/FqdTUTBeQDVOQ2PYyidtwFKr0B5G6ERukKwliq+7mIXvxyppwzG5EgRYg==", - "dev": true, "requires": { "@types/node": "*" } @@ -499,8 +503,7 @@ "@types/p-queue": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/@types/p-queue/-/p-queue-2.3.2.tgz", - "integrity": "sha512-eKAv5Ql6k78dh3ULCsSBxX6bFNuGjTmof5Q/T6PiECDq0Yf8IIn46jCyp3RJvCi8owaEmm3DZH1PEImjBMd/vQ==", - "dev": true + "integrity": "sha512-eKAv5Ql6k78dh3ULCsSBxX6bFNuGjTmof5Q/T6PiECDq0Yf8IIn46jCyp3RJvCi8owaEmm3DZH1PEImjBMd/vQ==" }, "@types/parse-json": { "version": "4.0.0", @@ -531,8 +534,7 @@ "@types/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", - "dev": true + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" }, "@types/tough-cookie": { "version": "4.0.0", @@ -674,6 +676,12 @@ "eslint-visitor-keys": "^1.1.0" } }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, "acorn": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", @@ -749,6 +757,22 @@ "color-convert": "^1.9.0" } }, + "anymatch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", + "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "dev": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -1042,6 +1066,12 @@ "tweetnacl": "^0.14.3" } }, + "binary-extensions": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", + "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", + "dev": true + }, "bl": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", @@ -1058,6 +1088,11 @@ "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==", "dev": true }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, "boxen": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", @@ -1294,6 +1329,12 @@ "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true + }, "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", @@ -1422,6 +1463,70 @@ } } }, + "cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, + "chokidar": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz", + "integrity": "sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A==", + "dev": true, + "requires": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.1.2", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.4.0" + }, + "dependencies": { + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + } + } + }, "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -1756,6 +1861,22 @@ "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", "dev": true }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==" + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -1913,6 +2034,12 @@ "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz", "integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg==" }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true + }, "diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", @@ -1962,7 +2089,6 @@ "version": "1.1.8", "resolved": "https://registry.npmjs.org/discord-webhook-node/-/discord-webhook-node-1.1.8.tgz", "integrity": "sha512-3u0rrwywwYGc6HrgYirN/9gkBYqmdpvReyQjapoXARAHi0P0fIyf3W5tS5i3U3cc7e44E+e7dIHYUeec7yWaug==", - "dev": true, "requires": { "form-data": "^3.0.0", "node-fetch": "^2.6.0" @@ -1972,7 +2098,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -1990,12 +2115,43 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", "dev": true }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dot-case": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.3.tgz", @@ -2105,6 +2261,11 @@ "ansi-colors": "^4.1.1" } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, "env-editor": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/env-editor/-/env-editor-0.4.1.tgz", @@ -2853,8 +3014,7 @@ "eventemitter3": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", - "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==", - "dev": true + "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==" }, "events": { "version": "3.2.0", @@ -3182,8 +3342,7 @@ "find-exec": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/find-exec/-/find-exec-1.0.1.tgz", - "integrity": "sha512-4o6QkGkpg3xK5s/47rdK9LDZRsE4JR1mrXnaAOXBngG6UKeIDJXfwtNCAkljgyy6VRh75D3FFaB0tii9vDEtIA==", - "dev": true + "integrity": "sha512-4o6QkGkpg3xK5s/47rdK9LDZRsE4JR1mrXnaAOXBngG6UKeIDJXfwtNCAkljgyy6VRh75D3FFaB0tii9vDEtIA==" }, "find-root": { "version": "1.1.0", @@ -3272,7 +3431,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", @@ -3310,6 +3468,13 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, + "fsevents": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz", + "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==", + "dev": true, + "optional": true + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -3597,6 +3762,19 @@ "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", "dev": true }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + } + }, "http-cache-semantics": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", @@ -3718,6 +3896,12 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true }, + "ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=", + "dev": true + }, "import-fresh": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", @@ -3827,6 +4011,15 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", @@ -4457,6 +4650,12 @@ } } }, + "make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, "map-cache": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", @@ -4984,6 +5183,71 @@ "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.11.tgz", "integrity": "sha512-BVZBDi+aJV4O38rxsUh164Dk1NCqgh6Cm0rQSb9SK/DHGll/DrCMnycVDD7msJgZCnmVa8ASo8EZzR7jsgTukQ==" }, + "nodemon": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.4.tgz", + "integrity": "sha512-Ltced+hIfTmaS28Zjv1BM552oQ3dbwPqI4+zI0SLgq+wpJhSyqgYude/aZa/3i31VCQWMfXJVxvu86abcam3uQ==", + "dev": true, + "requires": { + "chokidar": "^3.2.2", + "debug": "^3.2.6", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.0.4", + "pstree.remy": "^1.1.7", + "semver": "^5.7.1", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.2", + "update-notifier": "^4.0.0" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=", + "dev": true, + "requires": { + "abbrev": "1" + } + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -5004,12 +5268,26 @@ } } }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true + }, "normalize-url": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", "dev": true }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "requires": { + "boolbase": "~1.0.0" + } + }, "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", @@ -5238,8 +5516,7 @@ "p-queue": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-2.4.2.tgz", - "integrity": "sha512-n8/y+yDJwBjoLQe1GSJbbaYQLTI7QHNZI2+rpmCDbe++WLf9HC3gf6iqj5yfPAV71W4UF3ql5W1+UBPXoXTxng==", - "dev": true + "integrity": "sha512-n8/y+yDJwBjoLQe1GSJbbaYQLTI7QHNZI2+rpmCDbe++WLf9HC3gf6iqj5yfPAV71W4UF3ql5W1+UBPXoXTxng==" }, "p-reduce": { "version": "2.1.0", @@ -5251,7 +5528,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.2.0.tgz", "integrity": "sha512-jPH38/MRh263KKcq0wBNOGFJbm+U6784RilTmHjB/HM9kH9V8WlCpVUcdOmip9cjXOh6MxZ5yk1z2SjDUJfWmA==", - "dev": true, "requires": { "@types/retry": "^0.12.0", "retry": "^0.12.0" @@ -5322,6 +5598,14 @@ "lines-and-columns": "^1.1.6" } }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, "pascal-case": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.1.tgz", @@ -5424,7 +5708,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/play-sound/-/play-sound-1.1.3.tgz", "integrity": "sha512-lqEzgtNAdfg2VUXItOtu5bTyWcqeFmnJmgvc8iHEeEOBEJdurqiGYfmCOzIpSBcwrs7XeSpvHv+Rw9dzRPc4aw==", - "dev": true, "requires": { "find-exec": "1.0.1" } @@ -5518,6 +5801,12 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, + "pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true + }, "public-encrypt": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", @@ -5881,6 +6170,15 @@ "util-deprecate": "^1.0.1" } }, + "readdirp": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz", + "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, "redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -6070,8 +6368,7 @@ "retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", - "dev": true + "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=" }, "rimraf": { "version": "3.0.2", @@ -6404,6 +6701,24 @@ "urix": "^0.1.0" } }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + } + } + }, "source-map-url": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", @@ -6872,6 +7187,15 @@ "repeat-string": "^1.6.1" } }, + "touch": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", + "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==", + "dev": true, + "requires": { + "nopt": "~1.0.10" + } + }, "tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", @@ -6900,6 +7224,19 @@ "tslib": "^1.9.3" } }, + "ts-node": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.0.0.tgz", + "integrity": "sha512-/TqB4SnererCDR/vb4S/QvSZvzQMJN8daAslg7MeaiHvD8rDZsSfXmNeNumyZZzMned72Xoq/isQljYSt8Ynfg==", + "dev": true, + "requires": { + "arg": "^4.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.17", + "yn": "3.1.1" + } + }, "tsconfig-paths": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", @@ -7016,6 +7353,26 @@ "integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=", "dev": true }, + "undefsafe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz", + "integrity": "sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A==", + "dev": true, + "requires": { + "debug": "^2.2.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + } + } + }, "union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", @@ -7518,6 +7875,12 @@ "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" } + }, + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true } } } diff --git a/package.json b/package.json index 36ea81b51d..00929961d9 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "src/index.ts", "scripts": { "build": "rimraf ./build && tsc", + "dev": "nodemon --config nodemon.json", "lint": "xo", "lint:fix": "xo --fix", "start": "npm run build && node build/index.js", @@ -22,12 +23,16 @@ }, "homepage": "https://github.com/jef/nvidia-snatcher#readme", "dependencies": { + "@slack/web-api": "^5.12.0", "chalk": "^4.1.0", + "cheerio": "^1.0.0-rc.3", + "discord-webhook-node": "^1.1.8", "dotenv": "^8.2.0", "messaging-api-telegram": "^1.0.1", "node-notifier": "^8.0.0", "nodemailer": "^6.4.11", "open": "^7.2.1", + "play-sound": "^1.1.3", "puppeteer": "^5.3.1", "puppeteer-extra": "^3.1.15", "puppeteer-extra-plugin-adblocker": "^2.11.6", @@ -38,17 +43,17 @@ "winston": "^3.3.3" }, "devDependencies": { - "@slack/web-api": "^5.12.0", "@types/async": "^3.2.3", + "@types/cheerio": "^0.22.22", "@types/node": "^14.11.2", "@types/node-notifier": "^8.0.0", "@types/nodemailer": "^6.4.0", "@types/puppeteer": "^3.0.2", "@types/twitter": "^1.7.0", - "discord-webhook-node": "^1.1.8", "husky": "^4.3.0", - "play-sound": "^1.1.3", + "nodemon": "^2.0.4", "rimraf": "^3.0.2", + "ts-node": "^9.0.0", "typescript": "^4.0.2", "xo": "^0.33.1" }, diff --git a/src/adblocker.ts b/src/adblocker.ts index f8e7099ca4..fe9f68bb36 100644 --- a/src/adblocker.ts +++ b/src/adblocker.ts @@ -7,5 +7,7 @@ export const adBlocker = new PuppeteerExtraPluginAdblocker({ export async function disableBlockerInPage(page: Page) { const blockerObject = await adBlocker.getBlocker(); - await blockerObject.disableBlockingInPage(page); + if (blockerObject.isBlockingEnabled(page)) { + await blockerObject.disableBlockingInPage(page); + } } diff --git a/src/config.ts b/src/config.ts index 88daf25271..39d68f62c0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -49,7 +49,9 @@ function envOrNumber(environment: string | undefined, number?: number): number { const browser = { isHeadless: envOrBoolean(process.env.HEADLESS), isTrusted: envOrBoolean(process.env.BROWSER_TRUSTED, false), + maxBackoff: envOrNumber(process.env.PAGE_BACKOFF_MAX, 3600000), maxSleep: envOrNumber(process.env.PAGE_SLEEP_MAX, 10000), + minBackoff: envOrNumber(process.env.PAGE_BACKOFF_MIN, 10000), minSleep: envOrNumber(process.env.PAGE_SLEEP_MIN, 5000), open: envOrBoolean(process.env.OPEN_BROWSER) }; diff --git a/src/index.ts b/src/index.ts index fec007372a..f04bd7ce45 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ import {Config} from './config'; import {Logger} from './logger'; import {Stores} from './store/model'; import {adBlocker} from './adblocker'; +import {fetchLinks} from './store/fetch-links'; import {getSleepTime} from './util'; import puppeteer from 'puppeteer-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; @@ -37,14 +38,21 @@ async function main() { headless: Config.browser.isHeadless }); + const promises = []; for (const store of Stores) { Logger.debug(store.links); if (store.setupAction !== undefined) { store.setupAction(browser); } + if (store.linksBuilder) { + promises.push(fetchLinks(store, browser)); + } + setTimeout(tryLookupAndLoop, getSleepTime(), browser, store); } + + await Promise.all(promises); } /** diff --git a/src/logger.ts b/src/logger.ts index 9aab48ed0c..7a097f7254 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -26,6 +26,27 @@ export const Logger = winston.createLogger({ }); export const Print = { + backoff(link: Link, store: Store, delay: number, color?: boolean): string { + if (color) { + return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`); + } + + return `✖ ${buildProductString(link, store)} :: REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`; + }, + badStatusCode(link: Link, store: Store, statusCode: number, color?: boolean): string { + if (color) { + return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`STATUS CODE ERROR ${statusCode}`); + } + + return `✖ ${buildProductString(link, store)} :: STATUS CODE ERROR ${statusCode}`; + }, + bannedSeller(link: Link, store: Store, color?: boolean): string { + if (color) { + return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('BANNED SELLER'); + } + + return `✖ ${buildProductString(link, store)} :: BANNED SELLER`; + }, captcha(link: Link, store: Store, color?: boolean): string { if (color) { return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('CAPTCHA'); @@ -47,6 +68,20 @@ export const Print = { return `ℹ ${buildProductString(link, store)} :: IN STOCK, WAITING`; }, + message(message: string, topic: string, store: Store, color?: boolean): string { + if (color) { + return '✖ ' + buildSetupString(topic, store, true) + ' :: ' + chalk.yellow(message); + } + + return `✖ ${buildSetupString(topic, store)} :: ${message}`; + }, + noResponse(link: Link, store: Store, color?: boolean): string { + if (color) { + return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('NO RESPONSE'); + } + + return `✖ ${buildProductString(link, store)} :: NO RESPONSE`; + }, outOfStock(link: Link, store: Store, color?: boolean): string { if (color) { return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.red('OUT OF STOCK'); @@ -63,6 +98,14 @@ export const Print = { } }; +function buildSetupString(topic: string, store: Store, color?: boolean): string { + if (color) { + return chalk.cyan(`[${store.name}]`) + chalk.grey(` [setup (${topic})]`); + } + + return `[${store.name}] [setup (${topic})]`; +} + function buildProductString(link: Link, store: Store, color?: boolean): string { if (color) { return chalk.cyan(`[${store.name}]`) + chalk.grey(` [${link.brand} (${link.series})] ${link.model}`); diff --git a/src/store/fetch-links.ts b/src/store/fetch-links.ts new file mode 100644 index 0000000000..2dcb4a37a9 --- /dev/null +++ b/src/store/fetch-links.ts @@ -0,0 +1,58 @@ +import {Link, Series, Store} from './model'; +import {Logger, Print} from '../logger'; +import {Browser} from 'puppeteer'; +import cheerio from 'cheerio'; +import {filterSeries} from './filter'; +import {usingResponse} from '../util'; + +function addNewLinks(store: Store, links: Link[], series: Series) { + if (links.length === 0) { + Logger.error(Print.message('NO STORE LINKS FOUND', series, store, true)); + + return; + } + + const existingUrls = new Set(store.links.map(link => link.url)); + const newLinks = links.filter(link => !existingUrls.has(link.url)); + + if (newLinks.length === 0) { + return; + } + + Logger.info(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true)); + Logger.debug(JSON.stringify(newLinks, null, 2)); + + store.links = store.links.concat(newLinks); +} + +export async function fetchLinks(store: Store, browser: Browser) { + if (!store.linksBuilder) { + return; + } + + const promises = []; + + for (const {series, url} of store.linksBuilder.urls) { + if (!filterSeries(series)) { + continue; + } + + Logger.info(Print.message('DETECTING STORE LINKS', series, store, true)); + + promises.push(usingResponse(browser, url, async response => { + const text = await response?.text(); + + if (!text) { + Logger.error(Print.message('NO RESPONSE', series, store, true)); + return; + } + + const docElement = cheerio.load(text).root(); + const links = store.linksBuilder!.builder(docElement, series); + + addNewLinks(store, links, series); + })); + } + + await Promise.all(promises); +} diff --git a/src/store/filter.ts b/src/store/filter.ts index c1d43bc15d..d44fbe6c51 100644 --- a/src/store/filter.ts +++ b/src/store/filter.ts @@ -40,7 +40,7 @@ function filterModel(model: Link['model']): boolean { * * @param series The series of the GPU */ -function filterSeries(series: Link['series']): boolean { +export function filterSeries(series: Link['series']): boolean { if (Config.store.showOnlySeries.length === 0) { return true; } diff --git a/src/store/includes-labels.ts b/src/store/includes-labels.ts index 4c768d2808..462ba8a313 100644 --- a/src/store/includes-labels.ts +++ b/src/store/includes-labels.ts @@ -1,3 +1,85 @@ +import {Element, LabelQuery} from './model'; +import {Logger} from '../logger'; +import {Page} from 'puppeteer'; + +export type Selector = { + requireVisible: boolean; + selector: string; + type: 'innerHTML' | 'outerHTML' | 'textContent'; +}; + +function isElementArray(query: LabelQuery): query is Element[] { + return Array.isArray(query) && query.length > 0 && typeof query[0] === 'object'; +} + +function getQueryAsElementArray(query: LabelQuery, defaultContainer: string): Array> { + if (isElementArray(query)) { + return query.map(x => ({ + container: x.container ?? defaultContainer, + text: x.text + })); + } + + if (Array.isArray(query)) { + return [{ + container: defaultContainer, + text: query + }]; + } + + return [{ + container: query.container ?? defaultContainer, + text: query.text + }]; +} + +export async function pageIncludesLabels(page: Page, query: LabelQuery, options: Selector) { + const elementQueries = getQueryAsElementArray(query, options.selector); + + const resolved = await Promise.all(elementQueries.map(async query => { + const selector = {...options, selector: query.container}; + const contents = await extractPageContents(page, selector) ?? ''; + + if (!contents) { + return false; + } + + Logger.debug(contents); + + return includesLabels(contents, query.text); + })); + + return resolved.includes(true); +} + +export async function extractPageContents(page: Page, selector: Selector): Promise { + const content = await page.evaluate((options: Selector) => { + // eslint-disable-next-line no-undef + const element: globalThis.HTMLElement | null = document.querySelector(options.selector); + + if (!element) { + return null; + } + + if (options.requireVisible && !(element.offsetWidth > 0 && element.offsetHeight > 0)) { + return null; + } + + switch (options.type) { + case 'innerHTML': + return element.innerHTML; + case 'outerHTML': + return element.outerHTML; + case 'textContent': + return element.textContent; + default: + return 'Error: selector.type is unknown'; + } + }, selector); + + return content; +} + /** * Checks if DOM has any related text. * diff --git a/src/store/lookup.ts b/src/store/lookup.ts index f14a43cdff..c70613886d 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -1,15 +1,23 @@ import {Browser, Page, Response} from 'puppeteer'; import {Link, Store} from './model'; import {Logger, Print} from '../logger'; +import {Selector, pageIncludesLabels} from './includes-labels'; import {closePage, delay, getSleepTime} from '../util'; import {Config} from '../config'; +import {disableBlockerInPage} from '../adblocker'; import {filterStoreLink} from './filter'; -import {includesLabels} from './includes-labels'; import open from 'open'; import {sendNotification} from '../notification'; +type Backoff = { + count: number; + time: number; +}; + const inStock: Record = {}; +const storeBackoff: Record = {}; + /** * Responsible for looking up information about a each product within * a `Store`. It's important that we ignore `no-await-in-loop` here @@ -34,6 +42,14 @@ async function lookup(browser: Browser, store: Store) { page.setDefaultNavigationTimeout(Config.page.navigationTimeout); await page.setUserAgent(Config.page.userAgent); + if (store.disableAdBlocker) { + try { + await disableBlockerInPage(page); + } catch (error) { + Logger.error(error); + } + } + try { await lookupCard(browser, store, page, link); } catch (error) { @@ -49,7 +65,41 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link const givenWaitFor = store.waitUntil ? store.waitUntil : 'networkidle0'; const response: Response | null = await page.goto(link.url, {waitUntil: givenWaitFor}); - if (await lookupCardInStock(store, page)) { + if (!response) { + Logger.debug(Print.noResponse(link, store, true)); + } + + let backoff = storeBackoff[store.name]; + + if (!backoff) { + backoff = {count: 0, time: Config.browser.minBackoff}; + storeBackoff[store.name] = backoff; + } + + if (response?.status() === 403) { + Logger.warn(Print.backoff(link, store, backoff.time, true)); + await delay(backoff.time); + backoff.count++; + backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff); + return; + } + + if (response?.status() === 429) { + Logger.warn(Print.rateLimit(link, store, true)); + return; + } + + if ((response?.status() || 200) >= 400) { + Logger.warn(Print.badStatusCode(link, store, response!.status(), true)); + return; + } + + if (backoff.count > 0) { + backoff.count--; + backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff); + } + + if (await lookupCardInStock(store, page, link)) { const givenUrl = link.cartUrl ? link.cartUrl : link.url; Logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`); @@ -77,48 +127,48 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link link.screenshot = `success-${Date.now()}.png`; await page.screenshot({path: link.screenshot}); } - - return; - } - - if (await lookupPageHasCaptcha(store, page)) { - Logger.warn(Print.captcha(link, store, true)); - await delay(getSleepTime()); - return; } - - if (response && response.status() === 429) { - Logger.warn(Print.rateLimit(link, store, true)); - return; - } - - Logger.info(Print.outOfStock(link, store, true)); } -async function lookupCardInStock(store: Store, page: Page) { - const stockHandle = await page.$(store.labels.inStock.container); +async function lookupCardInStock(store: Store, page: Page, link: Link) { + const baseOptions: Selector = { + requireVisible: false, + selector: store.labels.container ?? 'body', + type: 'textContent' + }; - const visible = await page.evaluate(element => element && element.offsetWidth > 0 && element.offsetHeight > 0, stockHandle); - if (!visible) { - return false; - } - - const stockContent = await page.evaluate(element => element.outerHTML, stockHandle); + if (store.labels.inStock) { + const options = {...baseOptions, requireVisible: true, type: 'outerHTML' as const}; - Logger.debug(stockContent); + if (!await pageIncludesLabels(page, store.labels.inStock, options)) { + Logger.info(Print.outOfStock(link, store, true)); + return false; + } + } - return includesLabels(stockContent, store.labels.inStock.text); -} + if (store.labels.outOfStock) { + if (await pageIncludesLabels(page, store.labels.outOfStock, baseOptions)) { + Logger.info(Print.outOfStock(link, store, true)); + return false; + } + } -async function lookupPageHasCaptcha(store: Store, page: Page) { - if (!store.labels.captcha) { - return false; + if (store.labels.bannedSeller) { + if (await pageIncludesLabels(page, store.labels.bannedSeller, baseOptions)) { + Logger.warn(Print.bannedSeller(link, store, true)); + return false; + } } - const captchaHandle = await page.$(store.labels.captcha.container); - const captchaContent = await page.evaluate(element => element.textContent, captchaHandle); + if (store.labels.captcha) { + if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) { + Logger.warn(Print.captcha(link, store, true)); + await delay(getSleepTime()); + return false; + } + } - return includesLabels(captchaContent, store.labels.captcha.text); + return true; } export async function tryLookupAndLoop(browser: Browser, store: Store) { diff --git a/src/store/model/helpers/card.ts b/src/store/model/helpers/card.ts new file mode 100644 index 0000000000..5015265aea --- /dev/null +++ b/src/store/model/helpers/card.ts @@ -0,0 +1,48 @@ +export interface Card { + brand: string; + model: string; +} + +export function parseCard(name: string): Card | null { + name = name.replace(/[^\w ]+/g, '').trim(); + name = name.replace(/\bgraphics card\b/gi, '').trim(); + name = name.replace(/\b\w+ fan\b/gi, '').trim(); + name = name.replace(/\s{2,}/g, ' '); + + let model = name.split(' '); + const brand = model.shift(); + + if (!brand) { + return null; + } + + // Some vendors have oc at the beginning of the product name, + // store whether the card contains the term "oc" and remove + // it during filtering, then add it to the end of the name. + let isOC = false; + + /* eslint-disable @typescript-eslint/prefer-regexp-exec */ + model = model.filter(word => { + if (word.toLowerCase() === 'oc') { + isOC = true; + return false; + } + + return !word.match(/^(nvidia|geforce|rtx|amp[ae]re|graphics|card|gpu|pci-?e(xpress)?|ray-?tracing|ray|tracing|core|boost)$/i) && + !word.match(/^(\d+(?:gb?|mhz)?|gb|mhz|g?ddr(\d+x?)?)$/i); + }); + /* eslint-enable @typescript-eslint/prefer-regexp-exec */ + + if (isOC) { + model.push('OC'); + } + + if (model.length === 0) { + return null; + } + + return { + brand: brand.toLowerCase(), + model: model.join(' ').toLowerCase().replace(/ gaming\b/g, '').trim() + }; +} diff --git a/src/store/model/store.ts b/src/store/model/store.ts index 515d1e368f..1b4bd82d27 100644 --- a/src/store/model/store.ts +++ b/src/store/model/store.ts @@ -1,13 +1,15 @@ import {Browser, LoadEvent} from 'puppeteer'; export type Element = { - container: string; + container?: string; text: string[]; }; +export type Series = 'test:series' | '3070' | '3080' | '3090'; + export type Link = { - brand: 'test:brand' | 'asus' | 'evga' | 'gigabyte' | 'pny' | 'msi' | 'nvidia' | 'zotac'; - series: 'test:series' | '3070' | '3080' | '3090'; + brand: 'test:brand' | 'asus' | 'evga' | 'gigabyte' | 'inno3d' | 'kfa2' | 'palit' | 'pny' | 'msi' | 'nvidia' | 'zotac'; + series: Series; model: string; url: string; cartUrl?: string; @@ -15,13 +17,23 @@ export type Link = { screenshot?: string; }; +export type LabelQuery = Element[] | Element | string[]; + export type Labels = { - captcha?: Element; - inStock: Element; + bannedSeller?: LabelQuery; + captcha?: LabelQuery; + container?: string; + inStock?: LabelQuery; + outOfStock?: LabelQuery; }; export type Store = { + disableAdBlocker?: boolean; links: Link[]; + linksBuilder?: { + builder: (docElement: cheerio.Cheerio, series: Series) => Link[]; + urls: Array<{series: Series; url: string}>; + }; labels: Labels; name: string; setupAction?: (browser: Browser) => void; diff --git a/src/util.ts b/src/util.ts index 0bd824bf29..b164701f71 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,5 +1,6 @@ +import {Browser, Page, Response} from 'puppeteer'; import {Config} from './config'; -import {Page} from 'puppeteer'; +import {Logger} from './logger'; import {disableBlockerInPage} from './adblocker'; export function getSleepTime() { @@ -12,6 +13,34 @@ export async function delay(ms: number) { }); } +export async function usingResponse( + browser: Browser, + url: string, + cb: (response: (Response | null), page: Page, browser: Browser) => Promise +): Promise { + return usingPage(browser, async (page, browser) => { + const response = await page.goto(url, {waitUntil: 'domcontentloaded'}); + + return cb(response, page, browser); + }); +} + +export async function usingPage(browser: Browser, cb: (page: Page, browser: Browser) => Promise): Promise { + const page = await browser.newPage(); + page.setDefaultNavigationTimeout(Config.page.navigationTimeout); + await page.setUserAgent(Config.page.userAgent); + + try { + return await cb(page, browser); + } finally { + try { + await closePage(page); + } catch (error) { + Logger.error(error); + } + } +} + export async function closePage(page: Page) { await disableBlockerInPage(page); await page.close(); diff --git a/tsconfig.json b/tsconfig.json index 5205510ef6..15390b1625 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "es5", "module": "commonjs", - "lib": ["es6"], + "lib": ["es6", "dom"], "allowJs": true, "outDir": "build", "rootDir": "src",