/
deal_spider.py
146 lines (117 loc) · 4.46 KB
/
deal_spider.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
##########
# Programmer: Chris Tapia
# Program Description: Web Crawler to scrape the latest deals for those that are too busy, sends email of those deals
# and can also send those emails at a more specific time.
# Date Modified: 12/06/18
##########
# if not yet done, need to "pip install bs4" and "pip install requests" from the command window
import os
import sys
import requests
from bs4 import BeautifulSoup
import send_email
import sched, time
from datetime import datetime
from threading import Timer
# displays the title and the price when ran
def ebay_spider():
ebay_list = [] # list that we use to place into file
url = "https://www.ebay.com/deals?_trkparms=%26clkid%3D8422823819386027280"
source_code = requests.get(url)
plain_text = source_code.text
soup = BeautifulSoup(plain_text, "html.parser") # parses through the content
for item in soup.find_all('span', {'class': ['ebayui-ellipsis-3', 'ebayui-ellipsis-2', 'first', 'itemtile-price-bold']}):
title = item.string # this grabs only the string
print(title)
ebay_list.append(str(title) + "\n") # creates list to put in a file
message = '\n'.join([str(i) for i in ebay_list]) # formats the list for file
file = open("C:\\ebay deals.txt", "w")
file.write(message)
file.close()
filename = "C:\\ebay deals.txt"
body = "Attached are the latest deals on Ebay."
send_email.send_email("Ebay Deals", body, filename)
# look at ebay_spider comments for better understanding
def slickdeals_spider():
slick_list = []
url = "https://slickdeals.net"
source_code = requests.get(url)
plain_text = source_code.text
soup = BeautifulSoup(plain_text, "html.parser")
for item in soup.find_all('a', {'class': 'itemTitle'}):
title = item.string
item_url = url + item.get('href')
print(title)
get_single_item(item_url)
print(item_url)
slick_list.append(str(title) + "\n" + str(get_single_item(item_url)) + "\n" + str(item_url) + "\n\n")
message = '\n'.join([str(i) for i in slick_list])
file = open("C:\\Slickdeals.txt", "w")
file.write(message)
file.close()
filename = "C:\\Slickdeals.txt"
body = "Attached are the latest deals across the web from Slickdeals.com"
send_email.send_email("Slickdeals Deals!", body, filename)
def get_single_item(item_url):
source_code = requests.get(item_url)
plain_text = source_code.text
soup = BeautifulSoup(plain_text, "html.parser")
for item_price in soup.find_all('div', {'id': 'dealPrice'}):
price = item_price.string
print(str(price))
# here is the second menu
def menu():
option = True
while option:
print ("""
1. Deals on ebay
2. Deals on Slickdeals
3. Exit
""")
option = input("What would you like to do? ")
if option == "1":
print("\nHere are the current deals on ebay")
ebay_spider()
elif option == "2":
print("\nHere are the deals on Slickdeals from across the web")
slickdeals_spider()
elif option == "3":
raise SystemExit
elif option != "":
print("\n Not Valid Choice Try again")
# these automated functions send the email at a later time
# from which you choose
def ebay_automate(h, m):
print("sending your email at: %s:%s" % (h, m))
x = datetime.today()
y = x.replace(day=x.day + 1, hour=h, minute=m, second=0, microsecond=0)
delta_t = y - x
secs = delta_t.seconds + 1
t = Timer(secs, lambda: ebay_spider()) # lambda allows our function to be called separate from the Timer
t.start()
def slickdeals_automate(h, m):
print("sending your email at: %s:%s" % (h, m))
x = datetime.today()
y = x.replace(day=x.day + 1, hour=h, minute=m, second=0, microsecond=0)
delta_t = y - x
secs = delta_t.seconds + 1
t = Timer(secs, lambda: slickdeals_spider()) # lambda allows our function to be called separate from the Timer
t.start()
if __name__ == "__main__"
# starting menu
print("Which would you like to do?")
choice = input("""
1. Run Menu
2. Send me email of deals at a later time
3. Exit
""")
if choice == "1":
menu()
elif choice == "2":
ebay_automate(15, 30) # here you can put when to send the email
slickdeals_automate(15, 30)
elif choice == "3":
sys.exit
elif choice != "":
print("\nNot a valid choice!")
## end code ##