From fd294d2baa06a1c0a68852497889a0412dea492e Mon Sep 17 00:00:00 2001 From: xninjax Date: Fri, 2 Oct 2020 10:59:06 -0600 Subject: [PATCH] feat: max price filtering (#383) --- .env-example | 1 + .gitignore | 7 +++++-- README.md | 3 ++- src/config.ts | 1 + src/logger.ts | 7 +++++++ src/store/includes-labels.ts | 21 ++++++++++++++++++++- src/store/lookup.ts | 10 +++++++++- src/store/model/adorama.ts | 4 ++++ src/store/model/amazon-ca.ts | 4 ++++ src/store/model/amazon-de.ts | 4 ++++ src/store/model/amazon-nl.ts | 4 ++++ src/store/model/amazon.ts | 3 +++ src/store/model/bandh.ts | 4 ++++ src/store/model/bestbuy-ca.ts | 4 ++++ src/store/model/bestbuy.ts | 4 ++++ src/store/model/microcenter.ts | 4 ++++ src/store/model/newegg-ca.ts | 4 ++++ src/store/model/newegg.ts | 4 ++++ src/store/model/officedepot.ts | 4 ++++ src/store/model/pny.ts | 5 +++++ src/store/model/store.ts | 6 ++++++ src/store/model/zotac.ts | 4 ++++ 22 files changed, 107 insertions(+), 5 deletions(-) diff --git a/.env-example b/.env-example index 2e30c06b83..7787fcbe18 100644 --- a/.env-example +++ b/.env-example @@ -13,6 +13,7 @@ HEADLESS="" IN_STOCK_WAIT_TIME="" LOG_LEVEL="" LOW_BANDWIDTH="" +MAX_PRICE=“” MICROCENTER_LOCATION="" NVIDIA_ADD_TO_CART_ATTEMPTS="" NVIDIA_SESSION_TTL="" diff --git a/.gitignore b/.gitignore index 3a2d306c44..35529ada26 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,13 @@ build/ node_modules/ -.env +.env* +.*env +!.env-example success-*.png *.wav *.mp3 *.flac -*.exe \ No newline at end of file +*.exe +desktop.ini diff --git a/README.md b/README.md index 9a878d839e..588aeefcb1 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Here is a list of variables that you can use to customize your newly copied `.en | `LOW_BANDWIDTH` | Blocks images/fonts to reduce traffic | Disables ad blocker, default: `false` | | `MICROCENTER_LOCATION` | Specific MicroCenter location to search | Default : `web` | | `NVIDIA_ADD_TO_CART_ATTEMPTS` | The maximum number of times the `nvidia-api` add to cart feature will be attempted before failing | Default: `10` | -| `NVIDIA_SESSION_TTL` | The time in seconds to keep the cart active while using `nvidia-api` | Default: `60000` | +| `NVIDIA_SESSION_TTL` | The time in milliseconds to keep the cart active while using `nvidia-api` | Default: `60000` | | `OPEN_BROWSER` | Toggle for whether or not the browser should open when item is found | Default: `true` | | `PAGE_TIMEOUT` | Navigation Timeout in milliseconds | `0` for infinite, default: `30000` | | `PHONE_NUMBER` | 10 digit phone number | E.g.: `1234567890`, email configuration required | @@ -104,6 +104,7 @@ Here is a list of variables that you can use to customize your newly copied `.en | `SLACK_CHANNEL` | Slack channel for posting | E.g.: `update`, no need for `#` | | `SLACK_TOKEN` | Slack API token | | | `STORES` | [Supported stores](#supported-stores) you want to be scraped | Comma separated, default: `nvidia` | +| `MAX_PRICE` | Maximum price allowed for a match, applies to all cards (does not apply to these sites: Nvidia, Asus, EVGA) | Default: leave empty for no limit, otherwise enter a price (enter whole dollar amounts only, avoid use of: dollar symbols, commas, and periods.) e.g.: `1234` - Cards above `1234` will be skipped. | | `COUNTRY` | [Supported country](#supported-countries) you want to be scraped | Currently only used by Nvidia, default: `usa` | | `SCREENSHOT` | Capture screenshot of page if a card is found | Default: `true` | | `TELEGRAM_ACCESS_TOKEN` | Telegram access token | | diff --git a/src/config.ts b/src/config.ts index a5ef543c7c..78f2c9dc62 100644 --- a/src/config.ts +++ b/src/config.ts @@ -140,6 +140,7 @@ const proxy = { const store = { country: envOrString(process.env.COUNTRY, 'usa'), + maxPrice: envOrNumber(process.env.MAX_PRICE), microCenterLocation: envOrString(process.env.MICROCENTER_LOCATION, 'web'), showOnlyBrands: envOrArray(process.env.SHOW_ONLY_BRANDS), showOnlyModels: envOrArray(process.env.SHOW_ONLY_MODELS), diff --git a/src/logger.ts b/src/logger.ts index 2183b3f43a..24c8670753 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -74,6 +74,13 @@ export const Print = { return `ℹ ${buildProductString(link, store)} :: IN STOCK, WAITING`; }, + maxPrice(link: Link, store: Store, price: number, color?: boolean): string { + if (color) { + return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`PRICE ${price} EXCEEDS LIMIT ${Config.store.maxPrice}`); + } + + return `✖ ${buildProductString(link, store)} :: PRICE ${price} EXCEEDS LIMIT ${Config.store.maxPrice}`; + }, message(message: string, topic: string, store: Store, color?: boolean): string { if (color) { return '✖ ' + buildSetupString(topic, store, true) + ' :: ' + chalk.yellow(message); diff --git a/src/store/includes-labels.ts b/src/store/includes-labels.ts index 462ba8a313..0f84f066c1 100644 --- a/src/store/includes-labels.ts +++ b/src/store/includes-labels.ts @@ -1,4 +1,4 @@ -import {Element, LabelQuery} from './model'; +import {Element, LabelQuery, Pricing} from './model'; import {Logger} from '../logger'; import {Page} from 'puppeteer'; @@ -90,3 +90,22 @@ export function includesLabels(domText: string, searchLabels: string[]): boolean const domTextLowerCase = domText.toLowerCase(); return searchLabels.some(label => domTextLowerCase.includes(label)); } + +export async function cardPriceLimit(page: Page, query: Pricing, max: number, options: Selector) { + if (!max) { + return null; + } + + const selector = {...options, selector: query.container}; + const cardPrice = await extractPageContents(page, selector); + + if (cardPrice) { + const priceSeperator = query.euroFormat ? /\./g : /,/g; + const cardpriceNumber = Number.parseFloat(cardPrice.replace(priceSeperator, '').match(/\d+/g)!.join('.')); + + Logger.debug(`Raw card price: ${cardPrice} | Limit: ${max}`); + return cardpriceNumber > max ? cardpriceNumber : null; + } + + return null; +} diff --git a/src/store/lookup.ts b/src/store/lookup.ts index fe0ab61b6d..1bb8c766dd 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -1,7 +1,7 @@ import {Browser, Page, Response} from 'puppeteer'; import {Link, Store} from './model'; import {Logger, Print} from '../logger'; -import {Selector, pageIncludesLabels} from './includes-labels'; +import {Selector, cardPriceLimit, pageIncludesLabels} from './includes-labels'; import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util'; import {Config} from '../config'; import {disableBlockerInPage} from '../adblocker'; @@ -145,6 +145,14 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { } } + if (store.labels.maxPrice) { + const priceLimit = await cardPriceLimit(page, store.labels.maxPrice, Config.store.maxPrice, baseOptions); + if (priceLimit) { + Logger.info(Print.maxPrice(link, store, priceLimit, true)); + return false; + } + } + if (store.labels.captcha) { if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) { Logger.warn(Print.captcha(link, store, true)); diff --git a/src/store/model/adorama.ts b/src/store/model/adorama.ts index b320c7d8a7..91b3d1bbe6 100644 --- a/src/store/model/adorama.ts +++ b/src/store/model/adorama.ts @@ -9,6 +9,10 @@ export const Adorama: Store = { inStock: { container: '.buy-section.purchase', text: ['add to cart'] + }, + maxPrice: { + container: '.your-price', + euroFormat: false } }, links: [ diff --git a/src/store/model/amazon-ca.ts b/src/store/model/amazon-ca.ts index d652a8950b..843780e977 100644 --- a/src/store/model/amazon-ca.ts +++ b/src/store/model/amazon-ca.ts @@ -9,6 +9,10 @@ export const AmazonCa: Store = { inStock: { container: '#desktop_buybox', text: ['add to cart'] + }, + maxPrice: { + container: 'span[class*="PriceString"]', + euroFormat: false } }, links: [ diff --git a/src/store/model/amazon-de.ts b/src/store/model/amazon-de.ts index 9e4be850f9..09405d6875 100644 --- a/src/store/model/amazon-de.ts +++ b/src/store/model/amazon-de.ts @@ -9,6 +9,10 @@ export const AmazonDe: Store = { inStock: { container: '#desktop_buybox', text: ['in den einkaufswagen'] + }, + maxPrice: { + container: 'span[class*="PriceString"]', + euroFormat: true } }, links: [ diff --git a/src/store/model/amazon-nl.ts b/src/store/model/amazon-nl.ts index 6a83b9a092..86619c2e83 100644 --- a/src/store/model/amazon-nl.ts +++ b/src/store/model/amazon-nl.ts @@ -9,6 +9,10 @@ export const AmazonNl: Store = { inStock: { container: '#desktop_buybox', text: ['in winkelwagen plaatsen'] + }, + maxPrice: { + container: 'span[class*="PriceString"]', + euroFormat: true } }, links: [ diff --git a/src/store/model/amazon.ts b/src/store/model/amazon.ts index f9706a4f27..e3efd4046c 100644 --- a/src/store/model/amazon.ts +++ b/src/store/model/amazon.ts @@ -9,6 +9,9 @@ export const Amazon: Store = { inStock: { container: '#desktop_buybox', text: ['add to cart'] + }, + maxPrice: { + container: 'span[class*="PriceString"]' } }, links: [ diff --git a/src/store/model/bandh.ts b/src/store/model/bandh.ts index 15caa217ae..69ec9ec1b6 100644 --- a/src/store/model/bandh.ts +++ b/src/store/model/bandh.ts @@ -6,6 +6,10 @@ export const BAndH: Store = { inStock: { container: 'div[data-selenium="addToCartSection"]', text: ['add to cart'] + }, + maxPrice: { + container: 'div[data-selenium="pricingPrice"]', + euroFormat: false } }, links: [ diff --git a/src/store/model/bestbuy-ca.ts b/src/store/model/bestbuy-ca.ts index bccb9d15ca..e5b177ac5f 100644 --- a/src/store/model/bestbuy-ca.ts +++ b/src/store/model/bestbuy-ca.ts @@ -5,6 +5,10 @@ export const BestBuyCa: Store = { inStock: { container: '#root', text: ['available online'] + }, + maxPrice: { + container: 'div[class^="productPricingContainer"] span[class^="screenReaderOnly_"', + euroFormat: false } }, links: [ diff --git a/src/store/model/bestbuy.ts b/src/store/model/bestbuy.ts index b2745a30d9..b3a39eef06 100644 --- a/src/store/model/bestbuy.ts +++ b/src/store/model/bestbuy.ts @@ -5,6 +5,10 @@ export const BestBuy: Store = { inStock: { container: '.v-m-bottom-g', text: ['add to cart'] + }, + maxPrice: { + container: 'div[class="priceView-hero-price priceView-customer-price"] > span', + euroFormat: false } }, links: [ diff --git a/src/store/model/microcenter.ts b/src/store/model/microcenter.ts index fe5bbe1eb4..8a1ae624ea 100644 --- a/src/store/model/microcenter.ts +++ b/src/store/model/microcenter.ts @@ -43,6 +43,10 @@ export const MicroCenter: Store = { inStock: { container: '#cart-options', text: ['in stock'] + }, + maxPrice: { + container: 'span[id="pricing"]', + euroFormat: false } }, links: [ diff --git a/src/store/model/newegg-ca.ts b/src/store/model/newegg-ca.ts index 36f0218bfe..62f6996d71 100644 --- a/src/store/model/newegg-ca.ts +++ b/src/store/model/newegg-ca.ts @@ -9,6 +9,10 @@ export const NeweggCa: Store = { inStock: { container: '#landingpage-cart .btn-primary span', text: ['add to cart'] + }, + maxPrice: { + container: '#landingpage-price > div > div > ul > li.price-current > strong', + euroFormat: false } }, links: [ diff --git a/src/store/model/newegg.ts b/src/store/model/newegg.ts index 863a9517b3..b49f591c18 100644 --- a/src/store/model/newegg.ts +++ b/src/store/model/newegg.ts @@ -9,6 +9,10 @@ export const Newegg: Store = { inStock: { container: '#landingpage-cart .btn-primary span', text: ['add to cart'] + }, + maxPrice: { + container: '#landingpage-price > div > div > ul > li.price-current > strong', + euroFormat: false } }, links: [ diff --git a/src/store/model/officedepot.ts b/src/store/model/officedepot.ts index 921eefe795..65f2775d9b 100644 --- a/src/store/model/officedepot.ts +++ b/src/store/model/officedepot.ts @@ -9,6 +9,10 @@ export const OfficeDepot: Store = { inStock: { container: '#productPurchase', text: ['add to cart'] + }, + maxPrice: { + container: 'span[class^="price_column right"]', + euroFormat: false } }, links: [ diff --git a/src/store/model/pny.ts b/src/store/model/pny.ts index 96b5937b0f..ec4e44f997 100644 --- a/src/store/model/pny.ts +++ b/src/store/model/pny.ts @@ -5,7 +5,12 @@ export const Pny: Store = { inStock: { container: '#ctl01_lbtnAddToCart', text: ['add to cart'] + }, + maxPrice: { + container: 'span[itemprop="price"]', + euroFormat: false } + }, links: [ { diff --git a/src/store/model/store.ts b/src/store/model/store.ts index bddc9db17e..aa4beb5df9 100644 --- a/src/store/model/store.ts +++ b/src/store/model/store.ts @@ -5,6 +5,11 @@ export type Element = { text: string[]; }; +export type Pricing = { + container: string; + euroFormat?: boolean; +}; + export type Series = 'test:series' | '3070' | '3080' | '3090'; export type Link = { @@ -25,6 +30,7 @@ export type Labels = { container?: string; inStock?: LabelQuery; outOfStock?: LabelQuery; + maxPrice?: Pricing; }; export type StatusCodeRangeArray = Array<(number | [number, number])>; diff --git a/src/store/model/zotac.ts b/src/store/model/zotac.ts index 18815c291f..1123140774 100644 --- a/src/store/model/zotac.ts +++ b/src/store/model/zotac.ts @@ -6,6 +6,10 @@ export const Zotac: Store = { inStock: { container: '.add-to-cart-wrapper', text: ['add to cart'] + }, + maxPrice: { + container: 'div[class="product-shop"] span[class="price"]', + euroFormat: false } }, links: [