diff --git a/package-lock.json b/package-lock.json index d1254346a7..32b2c2d7b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -590,12 +590,28 @@ "defer-to-connect": "^1.0.1" } }, + "@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" + }, "@types/async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@types/async/-/async-3.2.4.tgz", "integrity": "sha512-2c14HTaCn6tlb7bnqJGeH5JJGHZmyrkDD3EYe1NYBYjuY4+2EmMg8pjau2HvVz8+ANmO+DieV95H8PlzbK383g==", "dev": true }, + "@types/cacheable-request": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.1.tgz", + "integrity": "sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ==", + "requires": { + "@types/http-cache-semantics": "*", + "@types/keyv": "*", + "@types/node": "*", + "@types/responselike": "*" + } + }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", @@ -683,6 +699,11 @@ "resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.5.tgz", "integrity": "sha512-IG8AE1m2pWtPqQ7wXhFhy6Q59bwwnLwO36v5Rit2FrbXCIp8Sk8E2PfUCreyrdo17STwFSKDAkitVuVYbpEHvQ==" }, + "@types/http-cache-semantics": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz", + "integrity": "sha512-c3Xy026kOF7QOTn00hbIllV1dLR9hG9NkSrLQgCVs8NF6sBU+VGWjD3wLPhmh1TYAc7ugCFsvHYMN4VcBN1U1A==" + }, "@types/is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz", @@ -703,6 +724,14 @@ "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=", "dev": true }, + "@types/keyv": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.1.tgz", + "integrity": "sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw==", + "requires": { + "@types/node": "*" + } + }, "@types/lodash": { "version": "4.14.164", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.164.tgz", @@ -798,6 +827,14 @@ "form-data": "^2.5.0" } }, + "@types/responselike": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", + "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==", + "requires": { + "@types/node": "*" + } + }, "@types/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", @@ -1927,6 +1964,11 @@ "unset-value": "^1.0.0" } }, + "cacheable-lookup": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz", + "integrity": "sha512-W+JBqF9SWe18A72XFzN/V/CULFzPm7sBXzzR6ekkE+3tLG72wFZrBiBZhrZuDoYexop4PHJVdFAKb/Nj9+tm9w==" + }, "cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", @@ -2198,7 +2240,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", - "dev": true, "requires": { "mimic-response": "^1.0.0" } @@ -4696,8 +4737,40 @@ "http-cache-semantics": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", - "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", - "dev": true + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" + }, + "http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "requires": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } + }, + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } }, "http-signature": { "version": "1.2.0", @@ -4709,6 +4782,22 @@ "sshpk": "^1.7.0" } }, + "http2-wrapper": { + "version": "1.0.0-beta.5.2", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.0-beta.5.2.tgz", + "integrity": "sha512-xYz9goEyBnC8XwXDTuC/MZ6t+MrKVQZOk4s7+PaDkwIsQd8IwqvM+0M6bA/2lvG8GHXcPdf+MejTUeO2LCPCeQ==", + "requires": { + "quick-lru": "^5.1.1", + "resolve-alpn": "^1.0.0" + }, + "dependencies": { + "quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==" + } + } + }, "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", @@ -4834,6 +4923,11 @@ "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", "dev": true }, + "ip": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + }, "ircv3": { "version": "0.26.5", "resolved": "https://registry.npmjs.org/ircv3/-/ircv3-0.26.5.tgz", @@ -5893,8 +5987,7 @@ "mimic-response": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "dev": true + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" }, "min-indent": { "version": "1.0.1", @@ -6437,8 +6530,7 @@ "normalize-url": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", - "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", - "dev": true + "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==" }, "npm": { "version": "7.0.12", @@ -8953,6 +9045,159 @@ } } }, + "puppeteer-page-proxy": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/puppeteer-page-proxy/-/puppeteer-page-proxy-1.2.8.tgz", + "integrity": "sha512-dbidfqSV67UiN0Yk5xjXHCMZ+iXpMicWUwHS1PtvM1pFUhyRWWtq+bz7J/DO4wJ8hYof7wWk7dVIhtQsQ/Xl/A==", + "requires": { + "got": "^11.5.1", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "socks-proxy-agent": "^5.0.0", + "tough-cookie": "^4.0.0" + }, + "dependencies": { + "@sindresorhus/is": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.0.0.tgz", + "integrity": "sha512-FyD2meJpDPjyNQejSjvnhpgI/azsQkA4lGbuu5BQZfjvJ9cbRZXzeWL2HceCekW4lixO9JPesIIQkSoLjeJHNQ==" + }, + "@szmarczak/http-timer": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz", + "integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==", + "requires": { + "defer-to-connect": "^2.0.0" + } + }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } + }, + "cacheable-request": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.1.tgz", + "integrity": "sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==", + "requires": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^4.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^2.0.0" + } + }, + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "requires": { + "mimic-response": "^3.1.0" + } + }, + "defer-to-connect": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.0.tgz", + "integrity": "sha512-bYL2d05vOSf1JEZNx5vSAtPuBMkX8K9EUutg7zlKvTqKXHt7RhWJFbmd7qakVuf13i+IkGmp6FwSsONOf6VYIg==" + }, + "got": { + "version": "11.8.0", + "resolved": "https://registry.npmjs.org/got/-/got-11.8.0.tgz", + "integrity": "sha512-k9noyoIIY9EejuhaBNLyZ31D5328LeqnyPNXJQb2XlJZcKakLqN5m6O/ikhq/0lw56kUYS54fVm+D1x57YC9oQ==", + "requires": { + "@sindresorhus/is": "^4.0.0", + "@szmarczak/http-timer": "^4.0.5", + "@types/cacheable-request": "^6.0.1", + "@types/responselike": "^1.0.0", + "cacheable-lookup": "^5.0.3", + "cacheable-request": "^7.0.1", + "decompress-response": "^6.0.0", + "http2-wrapper": "^1.0.0-beta.5.2", + "lowercase-keys": "^2.0.0", + "p-cancelable": "^2.0.0", + "responselike": "^2.0.0" + } + }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + } + }, + "json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" + }, + "keyv": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz", + "integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==", + "requires": { + "json-buffer": "3.0.1" + } + }, + "lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" + }, + "mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "p-cancelable": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz", + "integrity": "sha512-wvPXDmbMmu2ksjkB4Z3nZWTSkJEb9lqVdMaCKpZUGJG9TMiNp9XcbG3fn9fPKjem04fJMJnXoyFPk2FmgiaiNg==" + }, + "responselike": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz", + "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==", + "requires": { + "lowercase-keys": "^2.0.0" + } + }, + "tough-cookie": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.1.2" + } + }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + } + } + }, "pushover-notifications": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/pushover-notifications/-/pushover-notifications-1.2.2.tgz", @@ -9313,6 +9558,11 @@ "path-parse": "^1.0.6" } }, + "resolve-alpn": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.0.0.tgz", + "integrity": "sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA==" + }, "resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", @@ -9630,6 +9880,11 @@ } } }, + "smart-buffer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.1.0.tgz", + "integrity": "sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw==" + }, "snake-case": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-3.0.3.tgz", @@ -9756,6 +10011,48 @@ "kind-of": "^3.2.0" } }, + "socks": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.5.0.tgz", + "integrity": "sha512-00OqQHp5SCbwm9ecOMJj9aQtMSjwi1uVuGQoxnpKCS50VKZcOZ8z11CTKypmR8sEy7nZimy/qXY7rYJYbRlXmA==", + "requires": { + "ip": "^1.1.5", + "smart-buffer": "^4.1.0" + } + }, + "socks-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-5.0.0.tgz", + "integrity": "sha512-lEpa1zsWCChxiynk+lCycKuC502RxDWLKJZoIhnxrWNjLSDGYRFflHA1/228VkRcnv9TIb8w98derGbpKxJRgA==", + "requires": { + "agent-base": "6", + "debug": "4", + "socks": "^2.3.3" + }, + "dependencies": { + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } + }, + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "source-list-map": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", diff --git a/package.json b/package.json index f0eb6f5c44..565c1fa1d1 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "puppeteer-extra-plugin-adblocker": "^2.11.9", "puppeteer-extra-plugin-block-resources": "^2.2.7", "puppeteer-extra-plugin-stealth": "^2.6.5", + "puppeteer-page-proxy": "^1.2.8", "pushover-notifications": "^1.2.2", "twilio": "^3.52.0", "twitch": "^4.3.2", diff --git a/src/adblocker.ts b/src/adblocker.ts index fe9f68bb36..7fc58dbbbe 100644 --- a/src/adblocker.ts +++ b/src/adblocker.ts @@ -5,9 +5,20 @@ export const adBlocker = new PuppeteerExtraPluginAdblocker({ blockTrackers: true }); -export async function disableBlockerInPage(page: Page) { +export async function enableBlockerInPage(page: Page) { const blockerObject = await adBlocker.getBlocker(); if (blockerObject.isBlockingEnabled(page)) { - await blockerObject.disableBlockingInPage(page); + return; + } + + await blockerObject.enableBlockingInPage(page); +} + +export async function disableBlockerInPage(page: Page) { + const blockerObject = await adBlocker.getBlocker(); + if (!blockerObject.isBlockingEnabled(page)) { + return; } + + await blockerObject.disableBlockingInPage(page); } diff --git a/src/config.ts b/src/config.ts index 2c8b59479e..099e807912 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,6 +2,7 @@ import {banner} from './banner'; import {config as config_} from 'dotenv'; import path from 'path'; +import {readFileSync} from 'fs'; config_({path: path.resolve(__dirname, '../.env')}); @@ -354,6 +355,16 @@ const store = { ]), stores: envOrArray(process.env.STORES, ['nvidia']).map((entry) => { const [name, minPageSleep, maxPageSleep] = entry.match(/[^:]+/g) ?? []; + + let proxyList; + try { + proxyList = readFileSync(`${name}.proxies`) + .toString() + .trim() + .split('\n') + .map((x) => x.trim()); + } catch {} + return { maxPageSleep: envOrNumberMax( minPageSleep, @@ -365,7 +376,8 @@ const store = { maxPageSleep, browser.minSleep ), - name: envOrString(name) + name: envOrString(name), + proxyList }; }) }; diff --git a/src/index.ts b/src/index.ts index 573f3708b1..cb87c0567a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,25 +1,14 @@ import {startAPIServer, stopAPIServer} from './web'; import {Browser} from 'puppeteer'; -import {adBlocker} from './adblocker'; import {config} from './config'; import {getSleepTime} from './util'; import {logger} from './logger'; import puppeteer from 'puppeteer-extra'; -import resourceBlock from 'puppeteer-extra-plugin-block-resources'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import {storeList} from './store/model'; import {tryLookupAndLoop} from './store'; puppeteer.use(stealthPlugin()); -if (config.browser.lowBandwidth) { - puppeteer.use( - resourceBlock({ - blockedTypes: new Set(['image', 'font'] as const) - }) - ); -} else { - puppeteer.use(adBlocker); -} let browser: Browser | undefined; diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 7e94fe0443..8a74ce6d8f 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -1,4 +1,4 @@ -import {Browser, Page, Response} from 'puppeteer'; +import {Browser, Page, PageEventObj, Request, Response} from 'puppeteer'; import {Link, Store, getStores} from './model'; import {Print, logger} from '../logger'; import {Selector, cardPrice, pageIncludesLabels} from './includes-labels'; @@ -9,18 +9,109 @@ import { getSleepTime, isStatusCodeInRange } from '../util'; +import {disableBlockerInPage, enableBlockerInPage} from '../adblocker'; import {config} from '../config'; -import {disableBlockerInPage} from '../adblocker'; import {fetchLinks} from './fetch-links'; import {filterStoreLink} from './filter'; import open from 'open'; import {processBackoffDelay} from './model/helpers/backoff'; import {sendNotification} from '../notification'; +import useProxy from 'puppeteer-page-proxy'; const inStock: Record = {}; const linkBuilderLastRunTimes: Record = {}; +function nextProxy(store: Store) { + if (!store.proxyList) { + return; + } + + if (store.currentProxyIndex === undefined) { + store.currentProxyIndex = 0; + } + + store.currentProxyIndex++; + if (store.currentProxyIndex >= store.proxyList.length) { + store.currentProxyIndex = 0; + } + + logger.info( + `ℹ [${store.name}] Next proxy index: ${store.currentProxyIndex} / Count: ${store.proxyList.length}` + ); + + return store.proxyList[store.currentProxyIndex]; +} + +async function handleLowBandwidth(request: Request) { + if (!config.browser.lowBandwidth) { + return false; + } + + const typ = request.resourceType(); + if (typ === 'font' || typ === 'image') { + try { + await request.abort(); + } catch {} + + return true; + } + + return false; +} + +async function handleProxy(request: Request, proxy?: string) { + if (!proxy) { + return false; + } + + try { + await useProxy(request, proxy); + } catch (error: unknown) { + logger.error(error); + try { + await request.abort(); + } catch {} + } + + return true; +} + +async function handleAdBlock(request: Request, adBlockRequestHandler: any) { + if (!adBlockRequestHandler) { + return false; + } + + return new Promise((resolve) => { + const continueFunc = async () => { + resolve(false); + }; + + const abortFunc = async () => { + try { + await request.abort(); + } catch {} + + resolve(true); + }; + + const requestProxy = new Proxy(request, { + get(target, prop, receiver) { + if (prop === 'continue') { + return continueFunc; + } + + if (prop === 'abort') { + return abortFunc; + } + + return Reflect.get(target, prop, receiver); + } + }); + adBlockRequestHandler(requestProxy); + }); +} + /** * Responsible for looking up information about a each product within * a `Store`. It's important that we ignore `no-await-in-loop` here @@ -34,6 +125,20 @@ async function lookup(browser: Browser, store: Store) { return; } + if (store.linksBuilder) { + logger.info(`[${store.name}] Running linksBuilder...`); + const lastRunTime = linkBuilderLastRunTimes[store.name] ?? -1; + const ttl = store.linksBuilder.ttl ?? Number.MAX_SAFE_INTEGER; + if (lastRunTime === -1 || Date.now() - lastRunTime > ttl) { + try { + await fetchLinks(store, browser); + linkBuilderLastRunTimes[store.name] = Date.now(); + } catch (error: unknown) { + logger.error(error); + } + } + } + /* eslint-disable no-await-in-loop */ for (const link of store.links) { if (!filterStoreLink(link)) { @@ -45,23 +150,62 @@ async function lookup(browser: Browser, store: Store) { continue; } - const context = config.browser.isIncognito + const proxy = nextProxy(store); + + const useAdBlock = !config.browser.lowBandwidth && !store.disableAdBlocker; + const customContext = config.browser.isIncognito; + + const context = customContext ? await browser.createIncognitoBrowserContext() : browser.defaultBrowserContext(); - const page = config.browser.isIncognito - ? await context.newPage() - : await browser.newPage(); + const page = await context.newPage(); + page.setDefaultNavigationTimeout(config.page.timeout); await page.setUserAgent(getRandomUserAgent()); - if (store.disableAdBlocker) { - try { - await disableBlockerInPage(page); - } catch (error: unknown) { - logger.error(error); - } + let adBlockRequestHandler: any; + let pageProxy; + if (useAdBlock) { + const onProxyFunc = (event: keyof PageEventObj, handler: any) => { + if (event !== 'request') { + page.on(event, handler); + return; + } + + adBlockRequestHandler = handler; + }; + + pageProxy = new Proxy(page, { + get(target, prop, receiver) { + if (prop === 'on') { + return onProxyFunc; + } + + return Reflect.get(target, prop, receiver); + } + }); + await enableBlockerInPage(pageProxy); } + await page.setRequestInterception(true); + page.on('request', async (request) => { + if (await handleLowBandwidth(request)) { + return; + } + + if (await handleAdBlock(request, adBlockRequestHandler)) { + return; + } + + if (await handleProxy(request, proxy)) { + return; + } + + try { + await request.continue(); + } catch {} + }); + let statusCode = 0; try { @@ -74,7 +218,11 @@ async function lookup(browser: Browser, store: Store) { ); const client = await page.target().createCDPSession(); await client.send('Network.clearBrowserCookies'); - await client.send('Network.clearBrowserCache'); + // Await client.send('Network.clearBrowserCache'); + } + + if (pageProxy) { + await disableBlockerInPage(pageProxy); } // Must apply backoff before closing the page, e.g. if CloudFlare is @@ -82,7 +230,7 @@ async function lookup(browser: Browser, store: Store) { // before redirecting to the next page await processBackoffDelay(store, link, statusCode); await closePage(page); - if (config.browser.isIncognito) { + if (customContext) { await context.close(); } } @@ -223,19 +371,6 @@ export async function tryLookupAndLoop(browser: Browser, store: Store) { return; } - if (getStores().has(store.name) && store.linksBuilder) { - const lastRunTime = linkBuilderLastRunTimes[store.name] ?? -1; - const ttl = store.linksBuilder.ttl ?? Number.MAX_SAFE_INTEGER; - if (lastRunTime === -1 || Date.now() - lastRunTime > ttl) { - try { - await fetchLinks(store, browser); - linkBuilderLastRunTimes[store.name] = Date.now(); - } catch (error: unknown) { - logger.error((error as Error).message); - } - } - } - logger.debug(`[${store.name}] Starting lookup...`); try { await lookup(browser, store); diff --git a/src/store/model/index.ts b/src/store/model/index.ts index 8ffc80c80a..2a70b8d5fd 100644 --- a/src/store/model/index.ts +++ b/src/store/model/index.ts @@ -236,6 +236,7 @@ export function updateStores() { stores.set(storeData.name, store); store.minPageSleep = storeData.minPageSleep; store.maxPageSleep = storeData.maxPageSleep; + store.proxyList = storeData.proxyList; } else { logger.warn(`No store named ${storeData.name}, skipping.`); } diff --git a/src/store/model/store.ts b/src/store/model/store.ts index 5fb0ceccf5..e46de35743 100644 --- a/src/store/model/store.ts +++ b/src/store/model/store.ts @@ -1,4 +1,4 @@ -import {Browser, LoadEvent} from 'puppeteer'; +import {Browser, BrowserContext, LoadEvent} from 'puppeteer'; export type Element = { container?: string; @@ -178,4 +178,7 @@ export type Store = { waitUntil?: LoadEvent; minPageSleep?: number; maxPageSleep?: number; + + proxyList?: string[]; + currentProxyIndex?: number; };