Skip to content

Commit 3d050e8

Browse files
RandomProgramm3rRandomProgramm3r
committed
feat: Add async parser version and restructure for modularity
- Introduced async scraping logic in `async_.py` to support non-blocking operations. - Extracted shared functionality into `common.py` for reuse between sync and async versions. - Split original monolithic parser into separate modules under `src/scraper/`. - Moved data-related code to `data/steam_data.py` for better separation of concerns. - Added `format_response()` utility to format item data with game and currency names. - Updated requirements.txt.
1 parent 0140376 commit 3d050e8

File tree

9 files changed

+216
-69
lines changed

9 files changed

+216
-69
lines changed

‎data/__init__.py

Whitespace-only changes.

‎data.py renamed to ‎data/steam_data.py

File renamed without changes.

‎examples.py

Lines changed: 46 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,57 @@
1-
import data
2-
import scraper
1+
import asyncio
32

4-
if __name__ == '__main__':
5-
print(
6-
scraper.market_scraper(
3+
import data.steam_data
4+
import src.scraper.async_
5+
import src.scraper.sync
6+
7+
# sync
8+
print(
9+
src.scraper.sync.market_scraper_sync(
10+
'Dreams & Nightmares Case',
11+
data.steam_data.Apps.CS2.value,
12+
data.steam_data.Currency.USD.value,
13+
),
14+
)
15+
16+
17+
# async
18+
async def main():
19+
items = [
20+
(
721
'Dreams & Nightmares Case',
8-
data.Apps.CS2.value,
9-
data.Currency.USD.value,
22+
data.steam_data.Apps.CS2.value,
23+
data.steam_data.Currency.USD.value,
1024
),
11-
)
12-
print(
13-
scraper.market_scraper(
25+
(
1426
'Mann Co. Supply Crate Key',
15-
data.Apps.TEAM_FORTRESS_2.value,
16-
data.Currency.EUR.value,
27+
data.steam_data.Apps.TEAM_FORTRESS_2.value,
28+
data.steam_data.Currency.EUR.value,
1729
),
18-
)
19-
print(
20-
scraper.market_scraper(
30+
(
2131
'Doomsday Hoodie',
22-
data.Apps.PUBG.value,
23-
data.Currency.GBP.value,
32+
data.steam_data.Apps.PUBG.value,
33+
data.steam_data.Currency.GBP.value,
2434
),
25-
)
26-
print(
27-
scraper.market_scraper(
35+
(
2836
'AWP | Neo-Noir (Factory New)',
29-
data.Apps.CS2.value,
30-
data.Currency.USD.value,
37+
data.steam_data.Apps.CS2.value,
38+
data.steam_data.Currency.USD.value,
3139
),
32-
)
33-
print(
34-
scraper.market_scraper(
40+
(
3541
'Snowcamo Jacket',
36-
data.Apps.RUST.value,
37-
data.Currency.CHF.value,
42+
data.steam_data.Apps.RUST.value,
43+
data.steam_data.Currency.CHF.value,
3844
),
39-
)
45+
]
46+
47+
tasks = [
48+
src.scraper.async_.market_scraper_async(name, app_id, currency)
49+
for name, app_id, currency in items
50+
]
51+
results = await asyncio.gather(*tasks)
52+
for result in results:
53+
print(result)
54+
55+
56+
if __name__ == '__main__':
57+
asyncio.run(main())

‎requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
aiohttp==3.11.18
12
ruff==0.11.10

‎src/__init__.py

Whitespace-only changes.

‎src/scraper/__init__.py

Whitespace-only changes.

‎src/scraper/async_.py

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import asyncio
2+
import json
3+
4+
import aiohttp
5+
6+
import data.steam_data
7+
import src.scraper.common
8+
9+
10+
async def fetch_price(
11+
session: aiohttp.ClientSession,
12+
item_name: str,
13+
app_id: int,
14+
currency: int,
15+
) -> str:
16+
"""
17+
A request is sent asynchronously to the Steam Market and a formatted
18+
JSON response or error text is returned.
19+
"""
20+
encoded = src.scraper.common.encode_item_name(item_name)
21+
url = (
22+
f'{src.scraper.common.BASE_URL}'
23+
f'appid={app_id}'
24+
f'&currency={currency}'
25+
f'&market_hash_name={encoded}'
26+
)
27+
28+
try:
29+
async with session.get(url, timeout=5) as response:
30+
text = await response.text()
31+
parsed = json.loads(text)
32+
33+
return src.scraper.common.format_response(
34+
item_name,
35+
app_id,
36+
currency,
37+
parsed,
38+
)
39+
40+
except aiohttp.ClientResponseError as e:
41+
return f'Server error: {e.status}'
42+
except aiohttp.ClientConnectionError:
43+
return 'Network error. Please check your internet connection.'
44+
except asyncio.TimeoutError:
45+
return 'Timeout error.'
46+
except json.JSONDecodeError:
47+
return 'Error decoding JSON response from server.'
48+
49+
50+
async def market_scraper_async(
51+
item_name: str,
52+
app_id: int,
53+
currency: int = data.steam_data.Currency.USD.value,
54+
) -> str:
55+
"""
56+
Asynchronously fetch the market price for a given Steam item.
57+
58+
This coroutine validates the input parameters, opens an HTTP session,
59+
and delegates to the `fetch_price` helper to retrieve live pricing data
60+
from the Steam market.
61+
62+
Parameters:
63+
item_name (str): The exact name of the Steam marketplace item.
64+
app_id (int): The Steam application ID where the item is listed.
65+
currency (int, optional): The currency code for price conversion.
66+
Defaults to USD if not provided.
67+
68+
Returns:
69+
str: A formatted result, which may be a JSON string
70+
(via `format_response`) or an error message if the fetch fails.
71+
72+
Raises:
73+
ValueError: If any of `item_name`, `app_id`, or `currency` is invalid.
74+
aiohttp.ClientError: If the HTTP request to the Steam API fails.
75+
"""
76+
77+
src.scraper.common.validate_parameters(item_name, app_id, currency)
78+
async with aiohttp.ClientSession() as session:
79+
return await fetch_price(session, item_name, app_id, currency)

‎scraper.py renamed to ‎src/scraper/common.py

Lines changed: 34 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import enum
22
import json
3-
import urllib.error
43
import urllib.parse
5-
import urllib.request
64

7-
import data
5+
import data.steam_data
6+
7+
BASE_URL = 'https://steamcommunity.com/market/priceoverview/?'
88

99

1010
def validate_enum_value(
@@ -75,8 +75,8 @@ def validate_parameters(item_name: str, app_id: int, currency: int) -> None:
7575
if not item_name.strip():
7676
raise ValueError('Item name cannot consist solely of whitespace.')
7777

78-
validate_enum_value(app_id, data.Apps, 'app ID')
79-
validate_enum_value(currency, data.Currency, 'currency')
78+
validate_enum_value(app_id, data.steam_data.Apps, 'app ID')
79+
validate_enum_value(currency, data.steam_data.Currency, 'currency')
8080

8181

8282
def encode_item_name(item_name: str) -> str:
@@ -97,52 +97,45 @@ def encode_item_name(item_name: str) -> str:
9797
return urllib.parse.quote_plus(item_name.strip()).replace('~', '%7E')
9898

9999

100-
def market_scraper(
100+
def format_response(
101101
item_name: str,
102102
app_id: int,
103-
currency: int = data.Currency.USD.value,
103+
currency: int,
104+
result: dict | str,
104105
) -> str:
105106
"""
106-
Retrieves data from the Steam Community Market for the specified item.
107+
Construct a standardized response string for the Steam market data.
107108
108-
The function validates the input parameters, encodes the item name,
109-
constructs the URL, and performs an HTTP request. It returns a formatted
110-
JSON response or an error message.
109+
This function builds a JSON-formatted string when `result` is a dictionary,
110+
or returns the raw string otherwise. It enriches the output by inserting
111+
the game name, currency name, and item name into the result payload if
112+
they are not already present.
111113
112114
Args:
113-
item_name (str): The full name of the item.
114-
app_id (int): The application ID for the request.
115-
currency (int, optional): The currency code.
116-
Defaults to data.Currency.USD.value. (USD)
115+
item_name (str): The human-readable name of the item being queried.
116+
app_id (int): The Steam application ID associated with the item.
117+
currency (int): Numeric code representing the currency (per Steam API).
118+
result (dict | str): The price data or error message. If a dict,
119+
it will be pretty-printed as JSON; if a string, it will be returned
120+
unchanged.
117121
118122
Returns:
119-
str: A formatted JSON response or an error message.
120-
"""
121-
validate_parameters(item_name, app_id, currency)
122-
123-
encoded_name = encode_item_name(item_name)
123+
str: A JSON-formatted string including `game_name`, `currency_name`,
124+
`item_name`, and any other data present in `result`, or the raw
125+
string if `result` is not a dict.
124126
125-
base_url = 'https://steamcommunity.com/market/priceoverview/?'
126-
127-
url = (
128-
f'{base_url}'
129-
f'appid={app_id}'
130-
f'&currency={currency}'
131-
f'&market_hash_name={encoded_name}'
132-
)
133-
134-
try:
135-
with urllib.request.urlopen(url, timeout=5) as response:
136-
result = json.loads(response.read().decode())
127+
Raises:
128+
KeyError: If expected fields in the Steam data lookup are missing.
129+
TypeError: If `result` is not of type dict or str.
130+
"""
137131

138-
if isinstance(result, dict):
139-
return json.dumps(result, indent=4, ensure_ascii=False)
132+
game_name = data.steam_data.Apps(app_id).name
133+
currency_name = data.steam_data.Currency(currency).name
140134

141-
return result
135+
if isinstance(result, dict):
136+
result.setdefault('game_name', game_name)
137+
result.setdefault('currency_name', currency_name)
138+
result.setdefault('item_name', item_name)
139+
return json.dumps(result, indent=4, ensure_ascii=False)
142140

143-
except urllib.error.HTTPError as e:
144-
return f'Server error: {e.code}'
145-
except urllib.error.URLError:
146-
return 'Network error. Please check your internet connection.'
147-
except json.JSONDecodeError:
148-
return 'Error decoding JSON response from server.'
141+
return result

‎src/scraper/sync.py

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import json
2+
import urllib.error
3+
import urllib.request
4+
5+
import data.steam_data
6+
import src.scraper.common
7+
8+
9+
def market_scraper_sync(
10+
item_name: str,
11+
app_id: int,
12+
currency: int = data.steam_data.Currency.USD.value,
13+
) -> str:
14+
"""
15+
Retrieves data from the Steam Community Market for the specified item.
16+
17+
The function validates the input parameters, encodes the item name,
18+
constructs the URL, and performs an HTTP request. It returns a formatted
19+
JSON response or an error message.
20+
21+
Args:
22+
item_name (str): The full name of the item.
23+
app_id (int): The application ID for the request.
24+
currency (int, optional): The currency code.
25+
Defaults to data.Currency.USD.value. (USD)
26+
27+
Returns:
28+
str: A formatted JSON response or an error message.
29+
"""
30+
src.scraper.common.validate_parameters(item_name, app_id, currency)
31+
32+
encoded_name = src.scraper.common.encode_item_name(item_name)
33+
34+
url = (
35+
f'{src.scraper.common.BASE_URL}'
36+
f'appid={app_id}'
37+
f'&currency={currency}'
38+
f'&market_hash_name={encoded_name}'
39+
)
40+
41+
try:
42+
with urllib.request.urlopen(url, timeout=5) as response:
43+
parsed = json.loads(response.read().decode())
44+
return src.scraper.common.format_response(
45+
item_name,
46+
app_id,
47+
currency,
48+
parsed,
49+
)
50+
51+
except urllib.error.HTTPError as e:
52+
return f'Server error: {e.code}'
53+
except urllib.error.URLError:
54+
return 'Network error. Please check your internet connection.'
55+
except json.JSONDecodeError:
56+
return 'Error decoding JSON response from server.'

0 commit comments

Comments
 (0)