-
Notifications
You must be signed in to change notification settings - Fork 1
/
main.py
40 lines (33 loc) · 1.4 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
import utils
def main(countries=[], geotagged=False):
"""Function to initialize scraping of Craigslist housing
data. Uses static JSON files from /static dir and
parallel + scraping functions from /utils dir."""
all_locations = utils.craigslist_regions()
specific_locations = []
if countries:
for country in countries:
try:
specific_locations.extend(all_locations.get(country.lower()))
except TypeError:
print(f"{country} is not a valid country keyword.\n")
else:
# search every country
for country, _ in all_locations.items():
specific_locations.extend(all_locations.get(country.lower()))
for location in specific_locations:
# append geotag bool to location list
location.append(geotagged)
specific_locations_tuple = tuple(tuple(location) for location in specific_locations)
utils.map_processes(utils.scrape_housing, specific_locations_tuple)
if __name__ == "__main__":
countries = input(
"Input a list of appropriate countries.\nIf no list is provided, a global search will be conducted: "
)
if countries:
countries = eval(countries)
geotagged = input(
"Would you like to include geotags of your Craigslist posts [y/n]: "
)
geotagged = True if geotagged.lower() == "y" else False
main(countries=countries, geotagged=geotagged)