simple user-agent middleware for Ruia
pip install -U ruia-uaruia-ua will be automatically with a random User-Agent for your request's headers
from ruia import AttrField, TextField, Item, Spider
from ruia_ua import middleware
class HackerNewsItem(Item):
target_item = TextField(css_select='tr.athing')
title = TextField(css_select='a.storylink')
url = AttrField(css_select='a.storylink', attr='href')
class HackerNewsSpider(Spider):
start_urls = ['https://news.ycombinator.com/news?p=1', 'https://news.ycombinator.com/news?p=2']
async def parse(self, response):
# Do something...
print(response.url)
if __name__ == '__main__':
HackerNewsSpider.start(middleware=middleware)Enjoy it :)