Website crawler

tools/crawler/README.md

@@ -12,15 +12,13 @@ Crawled pages are post-processed in a multi-step pipeline consisting of:
 
 ## Requirements
 
-- Python
-- Readability
-- Scrapy
-- python-magic
+- Mamba / Conda
+- Node
 
 ## Installation
 
 ```
-pip install scrapy python-magic
+mamba create -f environment.yml
 ```
 
 ## Settings

tools/crawler/environment.yml

@@ -0,0 +1,10 @@
+name: scrapy
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.10
+  - python-magic=0.4
+  - pip=23.3.1
+  - pip:
+      - -r requirements.txt

tools/crawler/pipelines/readability_pipeline.py

@@ -15,7 +15,7 @@ def __init__(self):
         self.readability = Readability(port=6667)
 
     def process_item(self, item: GenericWebsiteItem, spider):
-        info = self.readability.parse(item["html"], item["url"])
+        info = self.readability.parse(item["raw_html"], item["url"])
         item["title"] = item["title"] if "title" in item else info["title"]
         item["html"] = info["content"]  # cleaned html
         item["text"] = info["textContent"]  # raw text, no html

tools/crawler/pipelines/write_pipeline.py

@@ -8,26 +8,9 @@ class WritePipeline:
     def process_item(self, item: GenericWebsiteItem, spider):
         # define relevant directories
         output_dir = Path(item["output_dir"])
-        html_output_dir = output_dir / "html"
         json_output_dir = output_dir / "json"
-        txt_output_dir = output_dir / "txt"
-        extracted_html_output_dir = output_dir / "extracted_html"
         raw_html_output_dir = output_dir / "raw_html"
 
-        # write html
-        if "html" in item:
-            html_output_dir.mkdir(parents=True, exist_ok=True)
-            with open(html_output_dir / f"{item['file_name']}.html", "w") as f:
-                f.write(item["html"])
-
-        # write html
-        if "extracted_html" in item:
-            extracted_html_output_dir.mkdir(parents=True, exist_ok=True)
-            with open(
-                extracted_html_output_dir / f"{item['file_name']}.html", "w"
-            ) as f:
-                f.write(item["extracted_html"])
-
         # write html
         if "raw_html" in item:
             raw_html_output_dir.mkdir(parents=True, exist_ok=True)
@@ -40,14 +23,8 @@ def process_item(self, item: GenericWebsiteItem, spider):
             data = {
                 key: value
                 for key, value in item.items()
-                if key != "output_dir" and key != "file_name"
+                if key != "output_dir" and key != "file_name" and key != "raw_html"
             }
             json.dump(data, f)
 
-        # write txt
-        if "text" in item:
-            txt_output_dir.mkdir(parents=True, exist_ok=True)
-            with open(txt_output_dir / f"{item['file_name']}.txt", "w") as f:
-                f.write(item["text"])
-
         return item

tools/crawler/requirements.txt

@@ -0,0 +1,9 @@
+beautifulsoup4==4.12.3
+lxml==5.2.1
+lxml_html_clean==0.1.1
+python-dateutil==2.9.0.post0
+Scrapy==2.11.1
+scrapy-selenium==0.0.7
+selenium==3.141.0
+urllib3==1.26.16
+webdriver-manager==4.0.1

tools/crawler/settings.py

@@ -1,15 +1,13 @@
 # scrapy-selenium settings
-from shutil import which
+# from shutil import which
 
-from webdriver_manager.chrome import ChromeDriverManager
+# from webdriver_manager.chrome import ChromeDriverManager
 
-SELENIUM_DRIVER_NAME = "chrome"
-SELENIUM_DRIVER_EXECUTABLE_PATH = which(ChromeDriverManager().install())
-SELENIUM_DRIVER_ARGUMENTS = [
-    "--headless=new"
-]  # '--headless' if using chrome instead of firefox
+# SELENIUM_DRIVER_NAME = "chrome"
+# SELENIUM_DRIVER_EXECUTABLE_PATH = which(ChromeDriverManager().install())
+# SELENIUM_DRIVER_ARGUMENTS = ["--headless=new"]
 
-# Scrapy settings for incel project
+# Scrapy settings
 #
 # For simplicity, this file contains only settings considered important or
 # commonly used. You can find more settings consulting the documentation:
@@ -64,7 +62,7 @@
 # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
 DOWNLOADER_MIDDLEWARES = {
     #   'incel.middlewares.IncelDownloaderMiddleware': 543,
-    "scrapy_selenium.SeleniumMiddleware": 800
+    # "scrapy_selenium.SeleniumMiddleware": 800
 }
 
 # Enable or disable extensions
@@ -82,9 +80,9 @@
     "crawler.pipelines.readability_pipeline.ReadabilityPipeline": 1,
     "crawler.pipelines.txtclean_pipeline.TXTCleanPipeline": 2,
     "crawler.pipelines.htmlclean_pipeline.HTMLCleanPipeline": 3,
-    "crawler.pipelines.extract_image_pipeline.ExtractImagePipeline": 4,
-    "crawler.pipelines.image_pipeline.MyImagesPipeline": 5,
-    "crawler.pipelines.replace_image_pipeline.ReplaceImagePipeline": 6,
+    # "crawler.pipelines.extract_image_pipeline.ExtractImagePipeline": 4,
+    # "crawler.pipelines.image_pipeline.MyImagesPipeline": 5,
+    # "crawler.pipelines.replace_image_pipeline.ReplaceImagePipeline": 6,
     "crawler.pipelines.write_pipeline.WritePipeline": 7,
 }
 IMAGES_STORE = "data/images"

tools/crawler/spiders/other/website.py

@@ -0,0 +1,54 @@
+from urllib.parse import urlparse
+
+import scrapy
+
+from crawler.spiders.spider_base import SpiderBase
+
+
+class WebsiteSpider(SpiderBase):
+    """
+    This Spiders finds and follows all <a href=""> links that are in-domain.
+    """
+
+    name = "website"
+    start_urls = ["https://www.uni-hamburg.de/"]
+    domain = "uni-hamburg.de"
+    visited_links = set()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def parse(self, response, **kwargs):
+        # find all links (a-tags) on the page
+        links = response.css("a::attr(href)").getall()
+
+        # filter out links that are not in-domain
+        valid_links = set()
+        for link in links:
+            # join with response url
+            link = response.urljoin(link)
+
+            # remove query parameters
+            link = urlparse(link)._replace(query="").geturl()
+
+            # filter out links that are not in-domain
+            if self.domain in link:
+                valid_links.add(link)
+
+        # filter out links that have already been visited
+        links_to_visit = valid_links - self.visited_links
+        self.visited_links.update(links_to_visit)
+
+        # follow links
+        for link in links_to_visit:
+            yield scrapy.Request(link, callback=self.parse)
+
+        # apply pipeline
+        item = self.init_item(response=response)
+        yield item
+
+    # save visited links in txt file
+    def close(self, reason):
+        with open("visited_links.txt", "w") as f:
+            for link in self.visited_links:
+                f.write(link + "\n")

tools/crawler/spiders/spider_base.py

@@ -40,8 +40,10 @@ def start_requests(self):
 
     def generate_filename(self, response: Response) -> str:
         parsed_url = urlparse(response.url)
-        article_slug = Path(parsed_url.path).stem
-        filename = slugify(f"{self.prefix}-{article_slug}")
+        article_slug = (
+            "" if parsed_url.path == "/" else Path(parsed_url.path).with_suffix("")
+        )
+        filename = slugify(f"{self.prefix}-{parsed_url.netloc}-{article_slug}")
         return filename
 
     def write_raw_response(
@@ -86,8 +88,8 @@ def init_item(
         except UnicodeDecodeError:
             item["raw_html"] = response.body
 
-        item["extracted_html"] = html if html else ""
-        item["html"] = html if html else item["raw_html"]
+        if html:
+            item["extracted_html"] = html
         item["output_dir"] = str(self.output_dir)
 
         for key, value in kwargs.items():

tools/crawler/spiders/utils.py

@@ -7,17 +7,17 @@
 def validate_output_dir(output_dir: Union[str, None]) -> Path:
     if output_dir is None:
         print(
-            "You have to provide an output directory with -a output_directory=/path/to/directory"
+            "You have to provide an output directory with -a output_dir=/path/to/directory"
         )
         exit()
 
-    output_dir = Path(output_dir)
-    if output_dir.is_file():
-        print(f"{output_dir} cannot be a file!")
+    output_path = Path(output_dir)
+    if output_path.is_file():
+        print(f"{output_path} cannot be a file!")
         exit()
-    output_dir.mkdir(parents=True, exist_ok=True)
+    output_path.mkdir(parents=True, exist_ok=True)
 
-    return output_dir
+    return output_path
 
 
 def slugify(value, allow_unicode=False):