final scraper (2).ipynb

{
  "metadata": {
    "language_info": {
      "codemirror_mode": {
        "name": "python",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8"
    },
    "kernelspec": {
      "name": "python",
      "display_name": "Python (Pyodide)",
      "language": "python"
    }
  },
  "nbformat_minor": 4,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "code",
      "source": "from bs4 import BeautifulSoup as bs\nimport requests",
      "metadata": {
        "trusted": true
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": "def CIM_SCRAPER(url, url_filter):\n    # initialize the home page\n    page = requests.get(url)\n    soup = bs(page.text, 'html.parser')\n\n    # define the two data stores needed\n    text_set = []\n    url_filtered = []\n    \n    # get all urls, filter the urls based on the url_filter condition and prevent duplicates\n    for link in soup.find_all('a'):\n        if link.get('href')!= None and link.get('href').startswith(url_filter) and link.get('href') not in url_filtered:\n                url_filtered.append(link.get('href'))\n                \n                page = requests.get(link.get('href'))\n                soup = bs(page.content, \"html.parser\")\n                for data in soup(['style', 'script']):\n                    # Remove tags\n                    data.decompose()\n                # return data by retrieving the tag content\n                text_set.append( ' '.join(soup.stripped_strings))\n     \n    return text_set",
      "metadata": {},
      "execution_count": null,
      "outputs": []
    }
  ]
}