diff --git a/.env-example b/.env-example
index bc76bf2415..90143f5960 100644
--- a/.env-example
+++ b/.env-example
@@ -20,6 +20,8 @@ PUSHBULLET=""
PUSHOVER_TOKEN=""
PUSHOVER_USER=""
PUSHOVER_PRIORITY=""
+PAGE_BACKOFF_MIN=""
+PAGE_BACKOFF_MAX=""
PAGE_SLEEP_MIN=""
PAGE_SLEEP_MAX=""
SHOW_ONLY_BRANDS=""
diff --git a/README.md b/README.md
index 14848baab2..3ad8418949 100644
--- a/README.md
+++ b/README.md
@@ -56,6 +56,10 @@ At any point you want the program to stop, use Ctrl + C.
> :point_right: Please visit the [wiki](https://github.com/jef/nvidia-snatcher/wiki) if you need more help with installation.
+### Developer notes
+
+The command `npm run dev` can be used instead of `npm run start` to automatically restart the project when filesystem changes are detected in the `src/` folder or `.env` file.
+
### Customization
To customize `nvidia-snatcher`, make a copy of `.env-example` as `.env` and make any changes to your liking. _All environment variables are **optional**._
@@ -83,6 +87,8 @@ Here is a list of variables that you can use to customize your newly copied `.en
| `PUSHOVER_TOKEN` | Pushover access token | Generate at https://pushover.net/apps/build | |
| `PUSHOVER_USER` | Pushover username | |
| `PUSHOVER_PRIORITY` | Pushover message priority |
+| `PAGE_BACKOFF_MIN` | Minimum backoff time between retrying requests for the same store when a forbidden response is received | Default: `10000` |
+| `PAGE_BACKOFF_MAX` | Maximum backoff time between retrying requests for the same store when a forbidden response is received | Default: `3600000` |
| `PAGE_SLEEP_MIN` | Minimum sleep time between queries of the same store | In milliseconds, default: `5000` |
| `PAGE_SLEEP_MAX` | Maximum sleep time between queries of the same store | In milliseconds, default: `10000` |
| `SCREENSHOT` | Capture screenshot of page if a card is found | Default: `true` |
diff --git a/nodemon.json b/nodemon.json
new file mode 100644
index 0000000000..30d6b818da
--- /dev/null
+++ b/nodemon.json
@@ -0,0 +1,8 @@
+{
+ "exec": "ts-node --files ./src/index",
+ "ext": "ts",
+ "watch": [
+ "src/",
+ ".env"
+ ]
+}
diff --git a/package-lock.json b/package-lock.json
index 8d943fd928..f83c92e333 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -313,7 +313,6 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@slack/logger/-/logger-2.0.0.tgz",
"integrity": "sha512-OkIJpiU2fz6HOJujhlhfIGrc8hB4ibqtf7nnbJQDerG0BqwZCfmgtK5sWzZ0TkXVRBKD5MpLrTmCYyMxoMCgPw==",
- "dev": true,
"requires": {
"@types/node": ">=8.9.0"
}
@@ -321,14 +320,12 @@
"@slack/types": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@slack/types/-/types-1.9.0.tgz",
- "integrity": "sha512-RmwgMWqOtzd2JPXdiaD/tyrDD0vtjjRDFdxN1I3tAxwBbg4aryzDUVqFc8na16A+3Xik/UN8X1hvVTw8J4EB9w==",
- "dev": true
+ "integrity": "sha512-RmwgMWqOtzd2JPXdiaD/tyrDD0vtjjRDFdxN1I3tAxwBbg4aryzDUVqFc8na16A+3Xik/UN8X1hvVTw8J4EB9w=="
},
"@slack/web-api": {
"version": "5.12.0",
"resolved": "https://registry.npmjs.org/@slack/web-api/-/web-api-5.12.0.tgz",
"integrity": "sha512-ygSnNHVid7PltGo7W36f2SNVHyliemkzxn9uSwgnWNF7CHmWBKWAylU/eoDml9l5K7akMOxbousiurOw4XqOFg==",
- "dev": true,
"requires": {
"@slack/logger": ">=1.0.0 <3.0.0",
"@slack/types": "^1.7.0",
@@ -346,8 +343,7 @@
"is-stream": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
- "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
- "dev": true
+ "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
}
}
},
@@ -372,6 +368,15 @@
"integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==",
"dev": true
},
+ "@types/cheerio": {
+ "version": "0.22.22",
+ "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.22.tgz",
+ "integrity": "sha512-05DYX4zU96IBfZFY+t3Mh88nlwSMtmmzSYaQkKN48T495VV1dkHSah6qYyDTN5ngaS0i0VonH37m+RuzSM0YiA==",
+ "dev": true,
+ "requires": {
+ "@types/node": "*"
+ }
+ },
"@types/chrome": {
"version": "0.0.91",
"resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.91.tgz",
@@ -433,7 +438,6 @@
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz",
"integrity": "sha512-jkZatu4QVbR60mpIzjINmtS1ZF4a/FqdTUTBeQDVOQ2PYyidtwFKr0B5G6ERukKwliq+7mIXvxyppwzG5EgRYg==",
- "dev": true,
"requires": {
"@types/node": "*"
}
@@ -499,8 +503,7 @@
"@types/p-queue": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/@types/p-queue/-/p-queue-2.3.2.tgz",
- "integrity": "sha512-eKAv5Ql6k78dh3ULCsSBxX6bFNuGjTmof5Q/T6PiECDq0Yf8IIn46jCyp3RJvCi8owaEmm3DZH1PEImjBMd/vQ==",
- "dev": true
+ "integrity": "sha512-eKAv5Ql6k78dh3ULCsSBxX6bFNuGjTmof5Q/T6PiECDq0Yf8IIn46jCyp3RJvCi8owaEmm3DZH1PEImjBMd/vQ=="
},
"@types/parse-json": {
"version": "4.0.0",
@@ -531,8 +534,7 @@
"@types/retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
- "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",
- "dev": true
+ "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="
},
"@types/tough-cookie": {
"version": "4.0.0",
@@ -674,6 +676,12 @@
"eslint-visitor-keys": "^1.1.0"
}
},
+ "abbrev": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+ "dev": true
+ },
"acorn": {
"version": "7.4.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
@@ -749,6 +757,22 @@
"color-convert": "^1.9.0"
}
},
+ "anymatch": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
+ "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==",
+ "dev": true,
+ "requires": {
+ "normalize-path": "^3.0.0",
+ "picomatch": "^2.0.4"
+ }
+ },
+ "arg": {
+ "version": "4.1.3",
+ "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
+ "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
+ "dev": true
+ },
"argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@@ -1042,6 +1066,12 @@
"tweetnacl": "^0.14.3"
}
},
+ "binary-extensions": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz",
+ "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
+ "dev": true
+ },
"bl": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
@@ -1058,6 +1088,11 @@
"integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==",
"dev": true
},
+ "boolbase": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+ "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+ },
"boxen": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz",
@@ -1294,6 +1329,12 @@
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
},
+ "buffer-from": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
+ "dev": true
+ },
"buffer-xor": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
@@ -1422,6 +1463,70 @@
}
}
},
+ "cheerio": {
+ "version": "1.0.0-rc.3",
+ "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
+ "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
+ "requires": {
+ "css-select": "~1.2.0",
+ "dom-serializer": "~0.1.1",
+ "entities": "~1.1.1",
+ "htmlparser2": "^3.9.1",
+ "lodash": "^4.15.0",
+ "parse5": "^3.0.1"
+ }
+ },
+ "chokidar": {
+ "version": "3.4.2",
+ "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz",
+ "integrity": "sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A==",
+ "dev": true,
+ "requires": {
+ "anymatch": "~3.1.1",
+ "braces": "~3.0.2",
+ "fsevents": "~2.1.2",
+ "glob-parent": "~5.1.0",
+ "is-binary-path": "~2.1.0",
+ "is-glob": "~4.0.1",
+ "normalize-path": "~3.0.0",
+ "readdirp": "~3.4.0"
+ },
+ "dependencies": {
+ "braces": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+ "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+ "dev": true,
+ "requires": {
+ "fill-range": "^7.0.1"
+ }
+ },
+ "fill-range": {
+ "version": "7.0.1",
+ "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+ "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+ "dev": true,
+ "requires": {
+ "to-regex-range": "^5.0.1"
+ }
+ },
+ "is-number": {
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+ "dev": true
+ },
+ "to-regex-range": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+ "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+ "dev": true,
+ "requires": {
+ "is-number": "^7.0.0"
+ }
+ }
+ }
+ },
"chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
@@ -1756,6 +1861,22 @@
"integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==",
"dev": true
},
+ "css-select": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+ "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+ "requires": {
+ "boolbase": "~1.0.0",
+ "css-what": "2.1",
+ "domutils": "1.5.1",
+ "nth-check": "~1.0.1"
+ }
+ },
+ "css-what": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
+ "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
+ },
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@@ -1913,6 +2034,12 @@
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz",
"integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg=="
},
+ "diff": {
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
+ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+ "dev": true
+ },
"diffie-hellman": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
@@ -1962,7 +2089,6 @@
"version": "1.1.8",
"resolved": "https://registry.npmjs.org/discord-webhook-node/-/discord-webhook-node-1.1.8.tgz",
"integrity": "sha512-3u0rrwywwYGc6HrgYirN/9gkBYqmdpvReyQjapoXARAHi0P0fIyf3W5tS5i3U3cc7e44E+e7dIHYUeec7yWaug==",
- "dev": true,
"requires": {
"form-data": "^3.0.0",
"node-fetch": "^2.6.0"
@@ -1972,7 +2098,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
"integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
- "dev": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
@@ -1990,12 +2115,43 @@
"esutils": "^2.0.2"
}
},
+ "dom-serializer": {
+ "version": "0.1.1",
+ "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
+ "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
+ "requires": {
+ "domelementtype": "^1.3.0",
+ "entities": "^1.1.1"
+ }
+ },
"domain-browser": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
"integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==",
"dev": true
},
+ "domelementtype": {
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
+ "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
+ },
+ "domhandler": {
+ "version": "2.4.2",
+ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+ "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+ "requires": {
+ "domelementtype": "1"
+ }
+ },
+ "domutils": {
+ "version": "1.5.1",
+ "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+ "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+ "requires": {
+ "dom-serializer": "0",
+ "domelementtype": "1"
+ }
+ },
"dot-case": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.3.tgz",
@@ -2105,6 +2261,11 @@
"ansi-colors": "^4.1.1"
}
},
+ "entities": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+ "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+ },
"env-editor": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/env-editor/-/env-editor-0.4.1.tgz",
@@ -2853,8 +3014,7 @@
"eventemitter3": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz",
- "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==",
- "dev": true
+ "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q=="
},
"events": {
"version": "3.2.0",
@@ -3182,8 +3342,7 @@
"find-exec": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/find-exec/-/find-exec-1.0.1.tgz",
- "integrity": "sha512-4o6QkGkpg3xK5s/47rdK9LDZRsE4JR1mrXnaAOXBngG6UKeIDJXfwtNCAkljgyy6VRh75D3FFaB0tii9vDEtIA==",
- "dev": true
+ "integrity": "sha512-4o6QkGkpg3xK5s/47rdK9LDZRsE4JR1mrXnaAOXBngG6UKeIDJXfwtNCAkljgyy6VRh75D3FFaB0tii9vDEtIA=="
},
"find-root": {
"version": "1.1.0",
@@ -3272,7 +3431,6 @@
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz",
"integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==",
- "dev": true,
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
@@ -3310,6 +3468,13 @@
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
+ "fsevents": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz",
+ "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==",
+ "dev": true,
+ "optional": true
+ },
"function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
@@ -3597,6 +3762,19 @@
"integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
"dev": true
},
+ "htmlparser2": {
+ "version": "3.10.1",
+ "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
+ "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
+ "requires": {
+ "domelementtype": "^1.3.1",
+ "domhandler": "^2.3.0",
+ "domutils": "^1.5.1",
+ "entities": "^1.1.1",
+ "inherits": "^2.0.1",
+ "readable-stream": "^3.1.1"
+ }
+ },
"http-cache-semantics": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
@@ -3718,6 +3896,12 @@
"integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
"dev": true
},
+ "ignore-by-default": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
+ "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=",
+ "dev": true
+ },
"import-fresh": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
@@ -3827,6 +4011,15 @@
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
"integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
},
+ "is-binary-path": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
+ "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+ "dev": true,
+ "requires": {
+ "binary-extensions": "^2.0.0"
+ }
+ },
"is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
@@ -4457,6 +4650,12 @@
}
}
},
+ "make-error": {
+ "version": "1.3.6",
+ "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
+ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
+ "dev": true
+ },
"map-cache": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz",
@@ -4984,6 +5183,71 @@
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.11.tgz",
"integrity": "sha512-BVZBDi+aJV4O38rxsUh164Dk1NCqgh6Cm0rQSb9SK/DHGll/DrCMnycVDD7msJgZCnmVa8ASo8EZzR7jsgTukQ=="
},
+ "nodemon": {
+ "version": "2.0.4",
+ "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.4.tgz",
+ "integrity": "sha512-Ltced+hIfTmaS28Zjv1BM552oQ3dbwPqI4+zI0SLgq+wpJhSyqgYude/aZa/3i31VCQWMfXJVxvu86abcam3uQ==",
+ "dev": true,
+ "requires": {
+ "chokidar": "^3.2.2",
+ "debug": "^3.2.6",
+ "ignore-by-default": "^1.0.1",
+ "minimatch": "^3.0.4",
+ "pstree.remy": "^1.1.7",
+ "semver": "^5.7.1",
+ "supports-color": "^5.5.0",
+ "touch": "^3.1.0",
+ "undefsafe": "^2.0.2",
+ "update-notifier": "^4.0.0"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "3.2.6",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+ "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+ "dev": true,
+ "requires": {
+ "ms": "^2.1.1"
+ }
+ },
+ "has-flag": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+ "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+ "dev": true
+ },
+ "ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "dev": true
+ },
+ "semver": {
+ "version": "5.7.1",
+ "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+ "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+ "dev": true
+ },
+ "supports-color": {
+ "version": "5.5.0",
+ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+ "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+ "dev": true,
+ "requires": {
+ "has-flag": "^3.0.0"
+ }
+ }
+ }
+ },
+ "nopt": {
+ "version": "1.0.10",
+ "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz",
+ "integrity": "sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=",
+ "dev": true,
+ "requires": {
+ "abbrev": "1"
+ }
+ },
"normalize-package-data": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz",
@@ -5004,12 +5268,26 @@
}
}
},
+ "normalize-path": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
+ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
+ "dev": true
+ },
"normalize-url": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
"integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==",
"dev": true
},
+ "nth-check": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
+ "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
+ "requires": {
+ "boolbase": "~1.0.0"
+ }
+ },
"oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -5238,8 +5516,7 @@
"p-queue": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/p-queue/-/p-queue-2.4.2.tgz",
- "integrity": "sha512-n8/y+yDJwBjoLQe1GSJbbaYQLTI7QHNZI2+rpmCDbe++WLf9HC3gf6iqj5yfPAV71W4UF3ql5W1+UBPXoXTxng==",
- "dev": true
+ "integrity": "sha512-n8/y+yDJwBjoLQe1GSJbbaYQLTI7QHNZI2+rpmCDbe++WLf9HC3gf6iqj5yfPAV71W4UF3ql5W1+UBPXoXTxng=="
},
"p-reduce": {
"version": "2.1.0",
@@ -5251,7 +5528,6 @@
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.2.0.tgz",
"integrity": "sha512-jPH38/MRh263KKcq0wBNOGFJbm+U6784RilTmHjB/HM9kH9V8WlCpVUcdOmip9cjXOh6MxZ5yk1z2SjDUJfWmA==",
- "dev": true,
"requires": {
"@types/retry": "^0.12.0",
"retry": "^0.12.0"
@@ -5322,6 +5598,14 @@
"lines-and-columns": "^1.1.6"
}
},
+ "parse5": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+ "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+ "requires": {
+ "@types/node": "*"
+ }
+ },
"pascal-case": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.1.tgz",
@@ -5424,7 +5708,6 @@
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/play-sound/-/play-sound-1.1.3.tgz",
"integrity": "sha512-lqEzgtNAdfg2VUXItOtu5bTyWcqeFmnJmgvc8iHEeEOBEJdurqiGYfmCOzIpSBcwrs7XeSpvHv+Rw9dzRPc4aw==",
- "dev": true,
"requires": {
"find-exec": "1.0.1"
}
@@ -5518,6 +5801,12 @@
"resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
"integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
},
+ "pstree.remy": {
+ "version": "1.1.8",
+ "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",
+ "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==",
+ "dev": true
+ },
"public-encrypt": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz",
@@ -5881,6 +6170,15 @@
"util-deprecate": "^1.0.1"
}
},
+ "readdirp": {
+ "version": "3.4.0",
+ "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz",
+ "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==",
+ "dev": true,
+ "requires": {
+ "picomatch": "^2.2.1"
+ }
+ },
"redent": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz",
@@ -6070,8 +6368,7 @@
"retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
- "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=",
- "dev": true
+ "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs="
},
"rimraf": {
"version": "3.0.2",
@@ -6404,6 +6701,24 @@
"urix": "^0.1.0"
}
},
+ "source-map-support": {
+ "version": "0.5.19",
+ "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz",
+ "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==",
+ "dev": true,
+ "requires": {
+ "buffer-from": "^1.0.0",
+ "source-map": "^0.6.0"
+ },
+ "dependencies": {
+ "source-map": {
+ "version": "0.6.1",
+ "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+ "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+ "dev": true
+ }
+ }
+ },
"source-map-url": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz",
@@ -6872,6 +7187,15 @@
"repeat-string": "^1.6.1"
}
},
+ "touch": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz",
+ "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==",
+ "dev": true,
+ "requires": {
+ "nopt": "~1.0.10"
+ }
+ },
"tough-cookie": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
@@ -6900,6 +7224,19 @@
"tslib": "^1.9.3"
}
},
+ "ts-node": {
+ "version": "9.0.0",
+ "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.0.0.tgz",
+ "integrity": "sha512-/TqB4SnererCDR/vb4S/QvSZvzQMJN8daAslg7MeaiHvD8rDZsSfXmNeNumyZZzMned72Xoq/isQljYSt8Ynfg==",
+ "dev": true,
+ "requires": {
+ "arg": "^4.1.0",
+ "diff": "^4.0.1",
+ "make-error": "^1.1.1",
+ "source-map-support": "^0.5.17",
+ "yn": "3.1.1"
+ }
+ },
"tsconfig-paths": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz",
@@ -7016,6 +7353,26 @@
"integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=",
"dev": true
},
+ "undefsafe": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz",
+ "integrity": "sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A==",
+ "dev": true,
+ "requires": {
+ "debug": "^2.2.0"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "dev": true,
+ "requires": {
+ "ms": "2.0.0"
+ }
+ }
+ }
+ },
"union-value": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
@@ -7518,6 +7875,12 @@
"buffer-crc32": "~0.2.3",
"fd-slicer": "~1.1.0"
}
+ },
+ "yn": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
+ "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
+ "dev": true
}
}
}
diff --git a/package.json b/package.json
index 36ea81b51d..00929961d9 100644
--- a/package.json
+++ b/package.json
@@ -5,6 +5,7 @@
"main": "src/index.ts",
"scripts": {
"build": "rimraf ./build && tsc",
+ "dev": "nodemon --config nodemon.json",
"lint": "xo",
"lint:fix": "xo --fix",
"start": "npm run build && node build/index.js",
@@ -22,12 +23,16 @@
},
"homepage": "https://github.com/jef/nvidia-snatcher#readme",
"dependencies": {
+ "@slack/web-api": "^5.12.0",
"chalk": "^4.1.0",
+ "cheerio": "^1.0.0-rc.3",
+ "discord-webhook-node": "^1.1.8",
"dotenv": "^8.2.0",
"messaging-api-telegram": "^1.0.1",
"node-notifier": "^8.0.0",
"nodemailer": "^6.4.11",
"open": "^7.2.1",
+ "play-sound": "^1.1.3",
"puppeteer": "^5.3.1",
"puppeteer-extra": "^3.1.15",
"puppeteer-extra-plugin-adblocker": "^2.11.6",
@@ -38,17 +43,17 @@
"winston": "^3.3.3"
},
"devDependencies": {
- "@slack/web-api": "^5.12.0",
"@types/async": "^3.2.3",
+ "@types/cheerio": "^0.22.22",
"@types/node": "^14.11.2",
"@types/node-notifier": "^8.0.0",
"@types/nodemailer": "^6.4.0",
"@types/puppeteer": "^3.0.2",
"@types/twitter": "^1.7.0",
- "discord-webhook-node": "^1.1.8",
"husky": "^4.3.0",
- "play-sound": "^1.1.3",
+ "nodemon": "^2.0.4",
"rimraf": "^3.0.2",
+ "ts-node": "^9.0.0",
"typescript": "^4.0.2",
"xo": "^0.33.1"
},
diff --git a/src/adblocker.ts b/src/adblocker.ts
index f8e7099ca4..fe9f68bb36 100644
--- a/src/adblocker.ts
+++ b/src/adblocker.ts
@@ -7,5 +7,7 @@ export const adBlocker = new PuppeteerExtraPluginAdblocker({
export async function disableBlockerInPage(page: Page) {
const blockerObject = await adBlocker.getBlocker();
- await blockerObject.disableBlockingInPage(page);
+ if (blockerObject.isBlockingEnabled(page)) {
+ await blockerObject.disableBlockingInPage(page);
+ }
}
diff --git a/src/config.ts b/src/config.ts
index 88daf25271..39d68f62c0 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -49,7 +49,9 @@ function envOrNumber(environment: string | undefined, number?: number): number {
const browser = {
isHeadless: envOrBoolean(process.env.HEADLESS),
isTrusted: envOrBoolean(process.env.BROWSER_TRUSTED, false),
+ maxBackoff: envOrNumber(process.env.PAGE_BACKOFF_MAX, 3600000),
maxSleep: envOrNumber(process.env.PAGE_SLEEP_MAX, 10000),
+ minBackoff: envOrNumber(process.env.PAGE_BACKOFF_MIN, 10000),
minSleep: envOrNumber(process.env.PAGE_SLEEP_MIN, 5000),
open: envOrBoolean(process.env.OPEN_BROWSER)
};
diff --git a/src/index.ts b/src/index.ts
index fec007372a..f04bd7ce45 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -2,6 +2,7 @@ import {Config} from './config';
import {Logger} from './logger';
import {Stores} from './store/model';
import {adBlocker} from './adblocker';
+import {fetchLinks} from './store/fetch-links';
import {getSleepTime} from './util';
import puppeteer from 'puppeteer-extra';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
@@ -37,14 +38,21 @@ async function main() {
headless: Config.browser.isHeadless
});
+ const promises = [];
for (const store of Stores) {
Logger.debug(store.links);
if (store.setupAction !== undefined) {
store.setupAction(browser);
}
+ if (store.linksBuilder) {
+ promises.push(fetchLinks(store, browser));
+ }
+
setTimeout(tryLookupAndLoop, getSleepTime(), browser, store);
}
+
+ await Promise.all(promises);
}
/**
diff --git a/src/logger.ts b/src/logger.ts
index 9aab48ed0c..7a097f7254 100644
--- a/src/logger.ts
+++ b/src/logger.ts
@@ -26,6 +26,27 @@ export const Logger = winston.createLogger({
});
export const Print = {
+ backoff(link: Link, store: Store, delay: number, color?: boolean): string {
+ if (color) {
+ return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`);
+ }
+
+ return `✖ ${buildProductString(link, store)} :: REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`;
+ },
+ badStatusCode(link: Link, store: Store, statusCode: number, color?: boolean): string {
+ if (color) {
+ return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`STATUS CODE ERROR ${statusCode}`);
+ }
+
+ return `✖ ${buildProductString(link, store)} :: STATUS CODE ERROR ${statusCode}`;
+ },
+ bannedSeller(link: Link, store: Store, color?: boolean): string {
+ if (color) {
+ return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('BANNED SELLER');
+ }
+
+ return `✖ ${buildProductString(link, store)} :: BANNED SELLER`;
+ },
captcha(link: Link, store: Store, color?: boolean): string {
if (color) {
return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('CAPTCHA');
@@ -47,6 +68,20 @@ export const Print = {
return `ℹ ${buildProductString(link, store)} :: IN STOCK, WAITING`;
},
+ message(message: string, topic: string, store: Store, color?: boolean): string {
+ if (color) {
+ return '✖ ' + buildSetupString(topic, store, true) + ' :: ' + chalk.yellow(message);
+ }
+
+ return `✖ ${buildSetupString(topic, store)} :: ${message}`;
+ },
+ noResponse(link: Link, store: Store, color?: boolean): string {
+ if (color) {
+ return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('NO RESPONSE');
+ }
+
+ return `✖ ${buildProductString(link, store)} :: NO RESPONSE`;
+ },
outOfStock(link: Link, store: Store, color?: boolean): string {
if (color) {
return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.red('OUT OF STOCK');
@@ -63,6 +98,14 @@ export const Print = {
}
};
+function buildSetupString(topic: string, store: Store, color?: boolean): string {
+ if (color) {
+ return chalk.cyan(`[${store.name}]`) + chalk.grey(` [setup (${topic})]`);
+ }
+
+ return `[${store.name}] [setup (${topic})]`;
+}
+
function buildProductString(link: Link, store: Store, color?: boolean): string {
if (color) {
return chalk.cyan(`[${store.name}]`) + chalk.grey(` [${link.brand} (${link.series})] ${link.model}`);
diff --git a/src/store/fetch-links.ts b/src/store/fetch-links.ts
new file mode 100644
index 0000000000..2dcb4a37a9
--- /dev/null
+++ b/src/store/fetch-links.ts
@@ -0,0 +1,58 @@
+import {Link, Series, Store} from './model';
+import {Logger, Print} from '../logger';
+import {Browser} from 'puppeteer';
+import cheerio from 'cheerio';
+import {filterSeries} from './filter';
+import {usingResponse} from '../util';
+
+function addNewLinks(store: Store, links: Link[], series: Series) {
+ if (links.length === 0) {
+ Logger.error(Print.message('NO STORE LINKS FOUND', series, store, true));
+
+ return;
+ }
+
+ const existingUrls = new Set(store.links.map(link => link.url));
+ const newLinks = links.filter(link => !existingUrls.has(link.url));
+
+ if (newLinks.length === 0) {
+ return;
+ }
+
+ Logger.info(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true));
+ Logger.debug(JSON.stringify(newLinks, null, 2));
+
+ store.links = store.links.concat(newLinks);
+}
+
+export async function fetchLinks(store: Store, browser: Browser) {
+ if (!store.linksBuilder) {
+ return;
+ }
+
+ const promises = [];
+
+ for (const {series, url} of store.linksBuilder.urls) {
+ if (!filterSeries(series)) {
+ continue;
+ }
+
+ Logger.info(Print.message('DETECTING STORE LINKS', series, store, true));
+
+ promises.push(usingResponse(browser, url, async response => {
+ const text = await response?.text();
+
+ if (!text) {
+ Logger.error(Print.message('NO RESPONSE', series, store, true));
+ return;
+ }
+
+ const docElement = cheerio.load(text).root();
+ const links = store.linksBuilder!.builder(docElement, series);
+
+ addNewLinks(store, links, series);
+ }));
+ }
+
+ await Promise.all(promises);
+}
diff --git a/src/store/filter.ts b/src/store/filter.ts
index c1d43bc15d..d44fbe6c51 100644
--- a/src/store/filter.ts
+++ b/src/store/filter.ts
@@ -40,7 +40,7 @@ function filterModel(model: Link['model']): boolean {
*
* @param series The series of the GPU
*/
-function filterSeries(series: Link['series']): boolean {
+export function filterSeries(series: Link['series']): boolean {
if (Config.store.showOnlySeries.length === 0) {
return true;
}
diff --git a/src/store/includes-labels.ts b/src/store/includes-labels.ts
index 4c768d2808..462ba8a313 100644
--- a/src/store/includes-labels.ts
+++ b/src/store/includes-labels.ts
@@ -1,3 +1,85 @@
+import {Element, LabelQuery} from './model';
+import {Logger} from '../logger';
+import {Page} from 'puppeteer';
+
+export type Selector = {
+ requireVisible: boolean;
+ selector: string;
+ type: 'innerHTML' | 'outerHTML' | 'textContent';
+};
+
+function isElementArray(query: LabelQuery): query is Element[] {
+ return Array.isArray(query) && query.length > 0 && typeof query[0] === 'object';
+}
+
+function getQueryAsElementArray(query: LabelQuery, defaultContainer: string): Array> {
+ if (isElementArray(query)) {
+ return query.map(x => ({
+ container: x.container ?? defaultContainer,
+ text: x.text
+ }));
+ }
+
+ if (Array.isArray(query)) {
+ return [{
+ container: defaultContainer,
+ text: query
+ }];
+ }
+
+ return [{
+ container: query.container ?? defaultContainer,
+ text: query.text
+ }];
+}
+
+export async function pageIncludesLabels(page: Page, query: LabelQuery, options: Selector) {
+ const elementQueries = getQueryAsElementArray(query, options.selector);
+
+ const resolved = await Promise.all(elementQueries.map(async query => {
+ const selector = {...options, selector: query.container};
+ const contents = await extractPageContents(page, selector) ?? '';
+
+ if (!contents) {
+ return false;
+ }
+
+ Logger.debug(contents);
+
+ return includesLabels(contents, query.text);
+ }));
+
+ return resolved.includes(true);
+}
+
+export async function extractPageContents(page: Page, selector: Selector): Promise {
+ const content = await page.evaluate((options: Selector) => {
+ // eslint-disable-next-line no-undef
+ const element: globalThis.HTMLElement | null = document.querySelector(options.selector);
+
+ if (!element) {
+ return null;
+ }
+
+ if (options.requireVisible && !(element.offsetWidth > 0 && element.offsetHeight > 0)) {
+ return null;
+ }
+
+ switch (options.type) {
+ case 'innerHTML':
+ return element.innerHTML;
+ case 'outerHTML':
+ return element.outerHTML;
+ case 'textContent':
+ return element.textContent;
+ default:
+ return 'Error: selector.type is unknown';
+ }
+ }, selector);
+
+ return content;
+}
+
/**
* Checks if DOM has any related text.
*
diff --git a/src/store/lookup.ts b/src/store/lookup.ts
index f14a43cdff..c70613886d 100644
--- a/src/store/lookup.ts
+++ b/src/store/lookup.ts
@@ -1,15 +1,23 @@
import {Browser, Page, Response} from 'puppeteer';
import {Link, Store} from './model';
import {Logger, Print} from '../logger';
+import {Selector, pageIncludesLabels} from './includes-labels';
import {closePage, delay, getSleepTime} from '../util';
import {Config} from '../config';
+import {disableBlockerInPage} from '../adblocker';
import {filterStoreLink} from './filter';
-import {includesLabels} from './includes-labels';
import open from 'open';
import {sendNotification} from '../notification';
+type Backoff = {
+ count: number;
+ time: number;
+};
+
const inStock: Record = {};
+const storeBackoff: Record = {};
+
/**
* Responsible for looking up information about a each product within
* a `Store`. It's important that we ignore `no-await-in-loop` here
@@ -34,6 +42,14 @@ async function lookup(browser: Browser, store: Store) {
page.setDefaultNavigationTimeout(Config.page.navigationTimeout);
await page.setUserAgent(Config.page.userAgent);
+ if (store.disableAdBlocker) {
+ try {
+ await disableBlockerInPage(page);
+ } catch (error) {
+ Logger.error(error);
+ }
+ }
+
try {
await lookupCard(browser, store, page, link);
} catch (error) {
@@ -49,7 +65,41 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
const givenWaitFor = store.waitUntil ? store.waitUntil : 'networkidle0';
const response: Response | null = await page.goto(link.url, {waitUntil: givenWaitFor});
- if (await lookupCardInStock(store, page)) {
+ if (!response) {
+ Logger.debug(Print.noResponse(link, store, true));
+ }
+
+ let backoff = storeBackoff[store.name];
+
+ if (!backoff) {
+ backoff = {count: 0, time: Config.browser.minBackoff};
+ storeBackoff[store.name] = backoff;
+ }
+
+ if (response?.status() === 403) {
+ Logger.warn(Print.backoff(link, store, backoff.time, true));
+ await delay(backoff.time);
+ backoff.count++;
+ backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff);
+ return;
+ }
+
+ if (response?.status() === 429) {
+ Logger.warn(Print.rateLimit(link, store, true));
+ return;
+ }
+
+ if ((response?.status() || 200) >= 400) {
+ Logger.warn(Print.badStatusCode(link, store, response!.status(), true));
+ return;
+ }
+
+ if (backoff.count > 0) {
+ backoff.count--;
+ backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff);
+ }
+
+ if (await lookupCardInStock(store, page, link)) {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
Logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`);
@@ -77,48 +127,48 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
link.screenshot = `success-${Date.now()}.png`;
await page.screenshot({path: link.screenshot});
}
-
- return;
- }
-
- if (await lookupPageHasCaptcha(store, page)) {
- Logger.warn(Print.captcha(link, store, true));
- await delay(getSleepTime());
- return;
}
-
- if (response && response.status() === 429) {
- Logger.warn(Print.rateLimit(link, store, true));
- return;
- }
-
- Logger.info(Print.outOfStock(link, store, true));
}
-async function lookupCardInStock(store: Store, page: Page) {
- const stockHandle = await page.$(store.labels.inStock.container);
+async function lookupCardInStock(store: Store, page: Page, link: Link) {
+ const baseOptions: Selector = {
+ requireVisible: false,
+ selector: store.labels.container ?? 'body',
+ type: 'textContent'
+ };
- const visible = await page.evaluate(element => element && element.offsetWidth > 0 && element.offsetHeight > 0, stockHandle);
- if (!visible) {
- return false;
- }
-
- const stockContent = await page.evaluate(element => element.outerHTML, stockHandle);
+ if (store.labels.inStock) {
+ const options = {...baseOptions, requireVisible: true, type: 'outerHTML' as const};
- Logger.debug(stockContent);
+ if (!await pageIncludesLabels(page, store.labels.inStock, options)) {
+ Logger.info(Print.outOfStock(link, store, true));
+ return false;
+ }
+ }
- return includesLabels(stockContent, store.labels.inStock.text);
-}
+ if (store.labels.outOfStock) {
+ if (await pageIncludesLabels(page, store.labels.outOfStock, baseOptions)) {
+ Logger.info(Print.outOfStock(link, store, true));
+ return false;
+ }
+ }
-async function lookupPageHasCaptcha(store: Store, page: Page) {
- if (!store.labels.captcha) {
- return false;
+ if (store.labels.bannedSeller) {
+ if (await pageIncludesLabels(page, store.labels.bannedSeller, baseOptions)) {
+ Logger.warn(Print.bannedSeller(link, store, true));
+ return false;
+ }
}
- const captchaHandle = await page.$(store.labels.captcha.container);
- const captchaContent = await page.evaluate(element => element.textContent, captchaHandle);
+ if (store.labels.captcha) {
+ if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) {
+ Logger.warn(Print.captcha(link, store, true));
+ await delay(getSleepTime());
+ return false;
+ }
+ }
- return includesLabels(captchaContent, store.labels.captcha.text);
+ return true;
}
export async function tryLookupAndLoop(browser: Browser, store: Store) {
diff --git a/src/store/model/helpers/card.ts b/src/store/model/helpers/card.ts
new file mode 100644
index 0000000000..5015265aea
--- /dev/null
+++ b/src/store/model/helpers/card.ts
@@ -0,0 +1,48 @@
+export interface Card {
+ brand: string;
+ model: string;
+}
+
+export function parseCard(name: string): Card | null {
+ name = name.replace(/[^\w ]+/g, '').trim();
+ name = name.replace(/\bgraphics card\b/gi, '').trim();
+ name = name.replace(/\b\w+ fan\b/gi, '').trim();
+ name = name.replace(/\s{2,}/g, ' ');
+
+ let model = name.split(' ');
+ const brand = model.shift();
+
+ if (!brand) {
+ return null;
+ }
+
+ // Some vendors have oc at the beginning of the product name,
+ // store whether the card contains the term "oc" and remove
+ // it during filtering, then add it to the end of the name.
+ let isOC = false;
+
+ /* eslint-disable @typescript-eslint/prefer-regexp-exec */
+ model = model.filter(word => {
+ if (word.toLowerCase() === 'oc') {
+ isOC = true;
+ return false;
+ }
+
+ return !word.match(/^(nvidia|geforce|rtx|amp[ae]re|graphics|card|gpu|pci-?e(xpress)?|ray-?tracing|ray|tracing|core|boost)$/i) &&
+ !word.match(/^(\d+(?:gb?|mhz)?|gb|mhz|g?ddr(\d+x?)?)$/i);
+ });
+ /* eslint-enable @typescript-eslint/prefer-regexp-exec */
+
+ if (isOC) {
+ model.push('OC');
+ }
+
+ if (model.length === 0) {
+ return null;
+ }
+
+ return {
+ brand: brand.toLowerCase(),
+ model: model.join(' ').toLowerCase().replace(/ gaming\b/g, '').trim()
+ };
+}
diff --git a/src/store/model/store.ts b/src/store/model/store.ts
index 515d1e368f..1b4bd82d27 100644
--- a/src/store/model/store.ts
+++ b/src/store/model/store.ts
@@ -1,13 +1,15 @@
import {Browser, LoadEvent} from 'puppeteer';
export type Element = {
- container: string;
+ container?: string;
text: string[];
};
+export type Series = 'test:series' | '3070' | '3080' | '3090';
+
export type Link = {
- brand: 'test:brand' | 'asus' | 'evga' | 'gigabyte' | 'pny' | 'msi' | 'nvidia' | 'zotac';
- series: 'test:series' | '3070' | '3080' | '3090';
+ brand: 'test:brand' | 'asus' | 'evga' | 'gigabyte' | 'inno3d' | 'kfa2' | 'palit' | 'pny' | 'msi' | 'nvidia' | 'zotac';
+ series: Series;
model: string;
url: string;
cartUrl?: string;
@@ -15,13 +17,23 @@ export type Link = {
screenshot?: string;
};
+export type LabelQuery = Element[] | Element | string[];
+
export type Labels = {
- captcha?: Element;
- inStock: Element;
+ bannedSeller?: LabelQuery;
+ captcha?: LabelQuery;
+ container?: string;
+ inStock?: LabelQuery;
+ outOfStock?: LabelQuery;
};
export type Store = {
+ disableAdBlocker?: boolean;
links: Link[];
+ linksBuilder?: {
+ builder: (docElement: cheerio.Cheerio, series: Series) => Link[];
+ urls: Array<{series: Series; url: string}>;
+ };
labels: Labels;
name: string;
setupAction?: (browser: Browser) => void;
diff --git a/src/util.ts b/src/util.ts
index 0bd824bf29..b164701f71 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -1,5 +1,6 @@
+import {Browser, Page, Response} from 'puppeteer';
import {Config} from './config';
-import {Page} from 'puppeteer';
+import {Logger} from './logger';
import {disableBlockerInPage} from './adblocker';
export function getSleepTime() {
@@ -12,6 +13,34 @@ export async function delay(ms: number) {
});
}
+export async function usingResponse(
+ browser: Browser,
+ url: string,
+ cb: (response: (Response | null), page: Page, browser: Browser) => Promise
+): Promise {
+ return usingPage(browser, async (page, browser) => {
+ const response = await page.goto(url, {waitUntil: 'domcontentloaded'});
+
+ return cb(response, page, browser);
+ });
+}
+
+export async function usingPage(browser: Browser, cb: (page: Page, browser: Browser) => Promise): Promise {
+ const page = await browser.newPage();
+ page.setDefaultNavigationTimeout(Config.page.navigationTimeout);
+ await page.setUserAgent(Config.page.userAgent);
+
+ try {
+ return await cb(page, browser);
+ } finally {
+ try {
+ await closePage(page);
+ } catch (error) {
+ Logger.error(error);
+ }
+ }
+}
+
export async function closePage(page: Page) {
await disableBlockerInPage(page);
await page.close();
diff --git a/tsconfig.json b/tsconfig.json
index 5205510ef6..15390b1625 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -2,7 +2,7 @@
"compilerOptions": {
"target": "es5",
"module": "commonjs",
- "lib": ["es6"],
+ "lib": ["es6", "dom"],
"allowJs": true,
"outDir": "build",
"rootDir": "src",