-
Notifications
You must be signed in to change notification settings - Fork 4
/
scrape_google_finance_ticker.py
151 lines (121 loc) · 6.88 KB
/
scrape_google_finance_ticker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
from itertools import zip_longest # https://docs.python.org/3/library/itertools.html#itertools.zip_longest
import nasdaqdatalink # https://docs.data.nasdaq.com/docs/python-installation
import requests, json, re
from parsel import Selector
def nasdaq_get_timeseries_data():
nasdaqdatalink.read_key(filename=".nasdaq_api_key") # create .env (.your_file_name) file locally and paste your Nasdaq API key.
# print(nasdaqdatalink.ApiConfig.api_key) # prints api key from the .nasdaq_api_key
timeseries_data = nasdaqdatalink.get("WIKI/GOOGL", collapse="monthly") # not sure what "WIKI" stands for
print(timeseries_data)
# nasdaq_get_timeseries_data()
def scrape_google_finance(ticker: str):
# https://docs.python-requests.org/en/master/user/quickstart/#passing-parameters-in-urls
params = {
"hl": "en" # language
}
# https://docs.python-requests.org/en/master/user/quickstart/#custom-headers
# https://www.whatismybrowser.com/detect/what-is-my-user-agent
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36",
}
html = requests.get(f"https://www.google.com/finance/quote/{ticker}", params=params, headers=headers, timeout=30)
selector = Selector(text=html.text)
# where all extracted data will be temporary located
ticker_data = {
"ticker_data": {},
"about_panel": {},
"news": {"items": []},
"finance_perfomance": {"table": []},
"people_also_search_for": {"items": []},
"interested_in": {"items": []}
}
# current price, quote, title extraction
ticker_data["ticker_data"]["current_price"] = selector.css(".AHmHk .fxKbKc::text").get()
ticker_data["ticker_data"]["quote"] = selector.css(".PdOqHc::text").get().replace(" • ",":")
ticker_data["ticker_data"]["title"] = selector.css(".zzDege::text").get()
# about panel extraction
about_panel_keys = selector.css(".gyFHrc .mfs7Fc::text").getall()
about_panel_values = selector.css(".gyFHrc .P6K39c").xpath("normalize-space()").getall()
for key, value in zip_longest(about_panel_keys, about_panel_values):
key_value = key.lower().replace(" ", "_")
ticker_data["about_panel"][key_value] = value
# description "about" extraction
ticker_data["about_panel"]["description"] = selector.css(".bLLb2d::text").get()
ticker_data["about_panel"]["extensions"] = selector.css(".w2tnNd::text").getall()
# news extarction
if selector.css(".yY3Lee").get():
for index, news in enumerate(selector.css(".yY3Lee"), start=1):
ticker_data["news"]["items"].append({
"position": index,
"title": news.css(".Yfwt5::text").get(),
"link": news.css(".z4rs2b a::attr(href)").get(),
"source": news.css(".sfyJob::text").get(),
"published": news.css(".Adak::text").get(),
"thumbnail": news.css("img.Z4idke::attr(src)").get()
})
else:
ticker_data["news"]["error"] = f"No news result from a {ticker}."
# finance perfomance table
if selector.css(".slpEwd .roXhBd").get():
fin_perf_col_2 = selector.css(".PFjsMe+ .yNnsfe::text").get() # e.g. Dec 2021
fin_perf_col_3 = selector.css(".PFjsMe~ .yNnsfe+ .yNnsfe::text").get() # e.g. Year/year change
for fin_perf in selector.css(".slpEwd .roXhBd"):
if fin_perf.css(".J9Jhg::text , .jU4VAc::text").get():
"""
if fin_perf.css().get() statement is needed, otherwise first value in a dict would be None:
"finance_perfomance": {
"table": [
{
"null": {
"Dec 2021": null,
"Year/year change": null
}
}
"""
perf_key = fin_perf.css(".J9Jhg::text , .jU4VAc::text").get() # e.g. Revenue, Net Income, Operating Income..
perf_value_col_1 = fin_perf.css(".QXDnM::text").get() # 60.3B, 26.40%..
perf_value_col_2 = fin_perf.css(".gEUVJe .JwB6zf::text").get() # 2.39%, -21.22%..
ticker_data["finance_perfomance"]["table"].append({
perf_key: {
fin_perf_col_2: perf_value_col_1, # dynamically add key and value from the second (2) column
fin_perf_col_3: perf_value_col_2 # dynamically add key and value from the third (3) column
}
})
else:
ticker_data["finance_perfomance"]["error"] = f"No 'finence perfomance table' for {ticker}."
# "you may be interested in" results
if selector.css(".HDXgAf .tOzDHb").get():
for index, other_interests in enumerate(selector.css(".HDXgAf .tOzDHb"), start=1):
ticker_data["interested_in"]["items"].append(discover_more_tickers(index, other_interests))
else:
ticker_data["interested_in"]["error"] = f"No 'you may be interested in` results for {ticker}"
# "people also search for" results
if selector.css(".HDXgAf+ div .tOzDHb").get():
for index, other_tickers in enumerate(selector.css(".HDXgAf+ div .tOzDHb"), start=1):
ticker_data["people_also_search_for"]["items"].append(discover_more_tickers(index, other_tickers))
else:
ticker_data["people_also_search_for"]["error"] = f"No 'people_also_search_for` in results for {ticker}"
return ticker_data
def discover_more_tickers(index: int, other_data: str):
"""
if price_change_formatted will start complaining,
check beforehand for None values with try/except and set it to 0, in this function.
however, re.search(r"\d{1}%|\d{1,10}\.\d{1,2}%" should make the job done.
"""
return {
"position": index,
"ticker": other_data.css(".COaKTb::text").get(),
"ticker_link": f'https://www.google.com/finance{other_data.attrib["href"].replace("./", "/")}',
"title": other_data.css(".RwFyvf::text").get(),
"price": other_data.css(".YMlKec::text").get(),
"price_change": other_data.css("[jsname=Fe7oBc]::attr(aria-label)").get(),
# https://regex101.com/r/BOFBlt/1
# Up by 100.99% -> 100.99%
"price_change_formatted": re.search(r"\d{1}%|\d{1,10}\.\d{1,2}%", other_data.css("[jsname=Fe7oBc]::attr(aria-label)").get()).group()
}
data = scrape_google_finance(ticker="GOOGL:NASDAQ")
print(json.dumps(data, indent=2, ensure_ascii=False))
# iterate over multiple tickers
# for ticker in ["DAX:INDEXDB", "GOOGL:NASDAQ", "MSFT:NASDAQ"]:
# data = scrape_google_finance(ticker=ticker)
# print(json.dumps(data["ticker_data"], indent=2, ensure_ascii=False))