/
Browser.py
159 lines (143 loc) · 6.8 KB
/
Browser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
#!/usr/bin/env python
# Download link: https://sites.google.com/a/chromium.org/chromedriver/downloads
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import (
TimeoutException, NoSuchElementException, StaleElementReferenceException, WebDriverException
)
from Tools import write_books, read_books, id_from_url
class Browser(Chrome):
OPTIONS = {"goog:chromeOptions": {
# Disable images loading
"prefs": {"profile.managed_default_content_settings.images": 2},
# Disable Chrome's GUI
"args": ["--headless", "--disable-gpu"]
}}
def __init__(self, edition_reviews=False):
Chrome.__init__(self, desired_capabilities=self.OPTIONS)
# Set page loading timeout to 30 seconds
self.set_page_load_timeout(30)
self.edition_reviews = edition_reviews
# Initialize browsing counters
self.rating = self.sort = self.fails = None
# Starts browser window
def start(self):
self.start_session(self.OPTIONS)
# Login to Goodreads.com
def login(self, email, password):
# Open goodreads.com main page
self.open()
# Fill login form and submit it
self.find_element_by_id("userSignInFormEmail").send_keys(email)
self.find_element_by_id("user_password").send_keys(password)
self.find_element_by_class_name("gr-button--signIn").click()
# General shortcut to open a GoodReads page
def open(self, sub_url='', keyword='', options=''):
# Try to open url until it succeeds
while True:
try:
self.get(f"https://www.goodreads.com{sub_url}{keyword}{options}")
break
# On connection timeout, loop again
except TimeoutException:
print("Reloading page...")
def open_book_editions(self, book_id):
self.open(f"/work/editions/{book_id}?expanded=true&sort=title")
# Shortcut to open GoodReads book page
def open_book_page(self, book_id):
self.sort = 0
self.rating = 5
self.open("/book/show/", book_id, f"?text_only=true&rating={self.rating}")
# Get book id from URL
url_id = id_from_url.match(self.current_url).group(1)
# If a book id redirect happened
if book_id != url_id:
# Update new book id in books.txt
ids = read_books()
ids[ids.index(book_id)] = url_id
write_books(ids)
raise ConnectionResetError(f"Redirect book id from {book_id} to {url_id}")
# If only this editions's reviews are required
if self.edition_reviews:
# Make sure to load them before scraping first page
self.switch_reviews_mode(book_id, same_mode=True)
self.are_reviews_loaded()
# Otherwise continue as normal
else: print(f"Rating: {self.rating} Stars, Sorted: {self._SORTS[self.sort].capitalize()}")
# Shortcut to open GoodReads books list or shelf
def open_page(self, keyword, browse):
# URL contains "show" unless it's an author page
method = "show" if browse != "author" else "list"
self.open(f"/{browse}/{method}/{keyword}")
# Shortcut to open a GoodReads search page for books lists
def open_list_search(self, keyword):
self.open("/search?q=", keyword, "&search_type=lists")
# Check if there's a next page
def goto_next_page(self):
try: # Try to find the next button
next_page = self.find_element_by_class_name("next_page")
# Check if button is click-able (i.e. it's anchor tag)
if next_page.tag_name == "a":
# Click the next page button
next_page.click()
return True
return False
# Return none if there isn't one
except NoSuchElementException:
print("WARNING: There is no next page!")
return None
except WebDriverException:
print("WARNING: Retrying to goto next page!")
self.implicitly_wait(1)
return self.goto_next_page()
def switch_reviews_mode(self, book_id, only_default=False, same_mode=False):
if not same_mode:
self.sort += 1
if self.sort > 2 or only_default:
self.sort = 0
self.rating -= 1
if self.rating < 1:
return False
print(f"Rating: {self.rating} Stars, Sorted: {self._SORTS[self.sort].capitalize()}")
self.execute_script(
'document.getElementById("reviews").insertAdjacentHTML("beforeend", \'<a data-remote="true" rel="nofollow"'
f'class="actionLinkLite loadingLink" data-keep-on-success="true" id="switch{self.rating}{self.sort}"' +
f'href="/book/reviews/{book_id}?text_only=true&rating={self.rating}&sort={self._SORTS[self.sort]}' +
('&edition_reviews=true' if self.edition_reviews else '') + '">Switch Mode</a>\');' +
f'document.getElementById("switch{self.rating}{self.sort}").click()'
)
# self.find_element_by_id(f"switch{self.rating}{self.sort}").click()
return True
# Return whether reviews were loaded
def are_reviews_loaded(self):
try: # Add a dummy "loading" tag to DOM
self.execute_script(
'document.getElementById("reviews").'
'insertAdjacentHTML("beforeend", \'<p id="load_reviews">loading</p>\');'
)
# Let the driver wait until the the dummy tag has disappeared
WebDriverWait(self, 12).until(ec.invisibility_of_element_located((By.ID, "load_reviews")))
self.fails = 0
# Return true if reviews are loaded and they're more that 0, otherwise return false
return len(self.find_element_by_id("bookReviews").find_elements_by_class_name("review")) > 0
except (TimeoutException, StaleElementReferenceException):
print("WARNING: Reviews Loading Timeout!")
self.fails += 1
# If reviews loading fails 3 times, raise an error
if not self.edition_reviews and self.fails == 3:
raise ConnectionError
return False
def editions_id(self):
try: # To find a the parent tag (to check whether page loaded)
editions = self.find_element_by_class_name("otherEditions")
except (NoSuchElementException, TimeoutException):
return None
try: # To find child link tag (to check whether it has other editions)
return id_from_url.match(
editions.find_element_by_tag_name("a").get_attribute("href")
).group(1)
except NoSuchElementException:
return False
_SORTS = ["default", "newest", "oldest"]