scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "52d91389",
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Imports \n",
    "from googleapiclient.discovery import build\n",
    "import requests\n",
    "from requests.auth import HTTPBasicAuth\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import re\n",
    "from itertools import product\n",
    "\n",
    "import urllib\n",
    "from requests_html import HTML, HTMLSession\n",
    "\n",
    "from typing import List"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f343454f",
   "metadata": {},
   "source": [
    "#### Creating the Scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dfe80354",
   "metadata": {
    "code_folding": [
     0,
     2,
     9
    ]
   },
   "outputs": [],
   "source": [
    "class Scraper: \n",
    "    \n",
    "    def __init__(self, api_key: str, cse_id: str, headers: dict): \n",
    "        \n",
    "        # Grab args\n",
    "        self.api_key = api_key\n",
    "        self.cse_id = cse_id\n",
    "        self.headers = headers\n",
    "        \n",
    "    def search(self, keyword: str, skip_links: list = None): \n",
    "        \n",
    "        \"\"\"\n",
    "        Run the google search and get list of top 70 results.\n",
    "        -- \n",
    "        keyword: str -> phrase to search on google \n",
    "        skip_links: list -> list of links that we should skip such as \n",
    "        \"\"\"\n",
    "        \n",
    "        # Setup service operator \n",
    "        service = build(\"customsearch\", \"v1\", developerKey = self.api_key)\n",
    "        \n",
    "        # Make request \n",
    "        res_list = []\n",
    "        for start in [1, 11, 21, 31, 41, 51, 61]:\n",
    "            res = service.cse().list(q = keyword, cx = self.cse_id, start = start).execute()\n",
    "            results = [{k:v for k,v in res.items() if k in ['title', 'link', 'snippet']} for res in res['items']]\n",
    "            res_list.append(results)\n",
    "            \n",
    "        # Flatten List\n",
    "        res_list = [item for sublist in res_list for item in sublist]\n",
    "        \n",
    "        # Remove all that contain a skip link\n",
    "        res_list = [page for page in res_list if not any(substring in page['link'] for substring in skip_links)]\n",
    "        \n",
    "        \n",
    "        return res_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d6d75cac",
   "metadata": {
    "code_folding": [
     0,
     2,
     16,
     61,
     89,
     95,
     136
    ]
   },
   "outputs": [],
   "source": [
    "class SearchResult: \n",
    "    \n",
    "    def __init__(self, scraper: Scraper, url: str, lawyer_type: str, city: str): \n",
    "        \n",
    "        # Grab attributes\n",
    "        self.url = url\n",
    "        self.lawyer_type = lawyer_type.lower()\n",
    "        self.lawyer_type_one_word = self.lawyer_type.replace(\" \", \"\")\n",
    "        self.city = city.lower()\n",
    "        self.keyword = self.lawyer_type + ' Lawyer ' + self.city\n",
    "        self.keyword = self.keyword.lower()\n",
    "        \n",
    "        # Make request to get bs4 object \n",
    "        req = requests.get(self.url, headers = scraper.headers)\n",
    "        self.soup = BeautifulSoup(req.content, 'html.parser')\n",
    "\n",
    "    def get_word_counts(self): \n",
    "        \n",
    "        # Body text\n",
    "        body_text = re.sub(r'\\s+', ' ', self.soup.find('body').text).strip().lower().replace('\\n','')\n",
    "        body_text = re.sub(r'(?<=[.,?!])(?=[^\\s])', r' ', body_text)\n",
    "        \n",
    "        # Header text \n",
    "        try: \n",
    "            header_text = re.sub(r'\\s+', ' ', self.soup.find('header').text).strip().lower().replace('\\n','')\n",
    "            header_text = re.sub(r'(?<=[.,?!])(?=[^\\s])', r' ', header_text)\n",
    "            header_words = len(header_text.split())\n",
    "        except: \n",
    "            header_text, header_words = \"\", 0\n",
    "        \n",
    "            \n",
    "        # Footer text \n",
    "        try: \n",
    "            footer_text = re.sub(r'\\s+', ' ', self.soup.find('footer').text).strip().lower().replace('\\n','')\n",
    "            footer_text = re.sub(r'(?<=[.,?!])(?=[^\\s])', r' ', footer_text)\n",
    "            footer_words = len(footer_text.split())\n",
    "        except: \n",
    "            footer_text, footer_words = \"\", 0\n",
    "            \n",
    "            \n",
    "        # Remove header/footer text\n",
    "        text = body_text.replace(header_text, '').replace(footer_text, '').strip()\n",
    "        \n",
    "        \n",
    "        # Compute word counts \n",
    "        total_words = len(text.split())\n",
    "        \n",
    "        word_count_info = {'total_words': total_words, 'header_words': header_words, \n",
    "                           'footer_words': footer_words}\n",
    "                                  \n",
    "                                  \n",
    "\n",
    "        # Assign stuff\n",
    "        self.text = text\n",
    "        self.body_text = body_text\n",
    "        self.header_text = header_text\n",
    "        self.footer_text = footer_text              \n",
    "        \n",
    "        \n",
    "        return word_count_info\n",
    "        \n",
    "    def get_keyword_counts(self): \n",
    "        \n",
    "        \"\"\"\n",
    "        Computes the keyword counts in various spots within a webpage\n",
    "        \"\"\"\n",
    "    \n",
    "        # City counts\n",
    "        city_count = len(re.findall(self.city, self.body_text))\n",
    "        \n",
    "        # Lawyer/attorney count \n",
    "        lawyer_count = len(re.findall(r'lawyer|attorney', self.text))\n",
    "        \n",
    "        # Type count\n",
    "        type_count = len(re.findall(self.lawyer_type, self.text))\n",
    "        \n",
    "        # First keyword occurence\n",
    "        self.text = self.text.replace(self.lawyer_type, self.lawyer_type_one_word)\n",
    "        \n",
    "        keyword_in_text = self.lawyer_type_one_word in self.text\n",
    "        first_keyword_spot = self.text.split().index(self.lawyer_type_one_word)+1 if keyword_in_text else 'NA'\n",
    "        keyword_in_200 = (first_keyword_spot < 200) if keyword_in_text else 'NA'\n",
    "        \n",
    "        # Output as dict \n",
    "        keyword_count_info = {'keyword_count': type_count, 'city_count': city_count, \n",
    "                              'lawyer_attorney_count': lawyer_count, 'first_keyword_spot': first_keyword_spot, \n",
    "                              'keyword_in_200_words': keyword_in_200}\n",
    "        return keyword_count_info\n",
    "    \n",
    "    def get_alt_text(self, img): \n",
    "        try: \n",
    "            return img['alt']\n",
    "        except: \n",
    "            return 'NO ALT' \n",
    "        \n",
    "    def get_tag_info(self): \n",
    "        \n",
    "        # Title tag \n",
    "        title = self.soup.find('title').text.strip().replace('\\n', '')\n",
    "        keyword_in_title = self.lawyer_type in title.lower()\n",
    "        \n",
    "        # H1 \n",
    "        h1 = self.soup.find_all('h1')\n",
    "        h1_count = len(h1)\n",
    "        h1_text = [el.text.strip().replace('\\n', '') for el in h1]\n",
    "        keyword_in_h1 = [self.lawyer_type in el.lower() for el in h1_text]\n",
    "        \n",
    "        # H2 \n",
    "        h2 = self.soup.find_all('h2')\n",
    "        h2_count = len(h2)\n",
    "        h2_text = [el.text.strip().replace('\\n', '') for el in h2]\n",
    "        keyword_in_h2 = [self.lawyer_type in el.lower() for el in h2_text]\n",
    "        \n",
    "        # H3 \n",
    "        h3 = self.soup.find_all('h3')\n",
    "        h3_count = len(h3)\n",
    "        h3_text = [el.text.strip().replace('\\n', '') for el in h3]\n",
    "        keyword_in_h3 = [self.lawyer_type in el.lower() for el in h3_text]\n",
    "        \n",
    "        # Image \n",
    "        images = self.soup.find_all('img')\n",
    "        image_count = len(images)\n",
    "        alt_text = [self.get_alt_text(img) for img in images]\n",
    "        keyword_in_alt_text = [self.lawyer_type in el.lower() for el in alt_text]\n",
    "        \n",
    "        # Combine \n",
    "        tag_info = {'title': title, 'key_in_title': keyword_in_title, \n",
    "                    'h1_count': h1_count, 'h1_text': h1_text, 'key_in_h1': keyword_in_h1, \n",
    "                    'h2_count': h2_count, 'h2_text': h2_text, 'key_in_h2': keyword_in_h2, \n",
    "                    'h3_count': h3_count, 'h3_text': h3_text, 'key_in_h3': keyword_in_h3, \n",
    "                    'image_count': image_count, 'image_alt_text': alt_text, \n",
    "                    'key_in_alt_text': keyword_in_alt_text\n",
    "                   }\n",
    "        \n",
    "        return tag_info\n",
    "      \n",
    "    def get_all_info(self): \n",
    "        \n",
    "        info = {'url':  self.url}\n",
    "        \n",
    "        # Word Counts \n",
    "        info.update(self.get_word_counts())\n",
    "        \n",
    "        # Keyword counts \n",
    "        keyword_count_info = self.get_keyword_counts()\n",
    "        info.update(keyword_count_info)\n",
    "        \n",
    "        # Tags\n",
    "        tag_info = self.get_tag_info()\n",
    "        info.update(tag_info)\n",
    "        \n",
    "        # Check if wordpress site\n",
    "        wp_script_tags = self.soup.find_all('link', {'href': re.compile(r'wp-content')})\n",
    "        info.update({'is_wordpress': bool(len(wp_script_tags) > 0)})\n",
    "        \n",
    "        # Check for table of contents\n",
    "        info.update({'has_table_of_contents': bool('table of contents' in self.body_text)})\n",
    "        \n",
    "        return info"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27349128",
   "metadata": {},
   "source": [
    "#### Run Scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "87e0e383",
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Create keyword list \n",
    "type_list = ['Personal Injury','Car Accident', 'Motorcycle Accident', \n",
    "          'Truck Accident', 'Medical Malpractice',  'Wrongful Death']\n",
    "city_list = ['Los Angeles', 'New York', 'Miami', 'Washington DC', 'Denver', 'Chicago']\n",
    "# keyword_list = [f'{k} Lawyer {city}' for k in type_list for city in city_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1b0905b8",
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Scraper Config\n",
    "# SECRET_KEY = \"AIzaSyDjwQM14wUBM40e5xtL7Df3Qe4my03iuTA\"\n",
    "# SECRET_KEY = \"AIzaSyBXIgkYyhxNqvEqWndStUlffrQV8EsK-fo\"\n",
    "# SECRET_KEY = 'AIzaSyBg9Q9dVyr1I7kGebGnMH6Ad6Lcm9MnB70'\n",
    "# SECRET_KEY = 'AIzaSyD46wJwuzLjTN5ygFk0B20mLi03eC1RsIg'\n",
    "# SECRET_KEY = 'AIzaSyDjwQM14wUBM40e5xtL7Df3Qe4my03iuTA'\n",
    "# SECRET_KEY = 'AIzaSyAKs4tgD2y3i-RDWSrmu0wO3J93_MVMiy4'\n",
    "SECRET_KEY = 'AIzaSyDixV59gyiiysxc1sadDXO9W_3WSQS8Qzs'\n",
    "CSE_ID = \"e340b4c39a82947c4\"\n",
    "HEADERS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}\n",
    "\n",
    "skip_links = ['yelp.com', 'forbes.com', 'expertise.com', 'findlaw.com', \n",
    "              'superlawyers.com', 'martindale.com', 'justia.com', 'indeed.com']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "24a5b121",
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Main scraping loop\n",
    "for lawyer_type in type_list[5:6]: \n",
    "    for city in city_list: \n",
    "        \n",
    "        # Generate keyword\n",
    "        keyword = lawyer_type + ' Lawyer ' + city\n",
    "         \n",
    "        # Get list of searchable results \n",
    "        scraper = Scraper(SECRET_KEY, CSE_ID, HEADERS)\n",
    "        result_list = scraper.search(keyword, skip_links = skip_links)\n",
    "        \n",
    "        # Iterate throguh each result\n",
    "        data = []\n",
    "        for idx, result in enumerate(result_list): \n",
    "            url = result['link']\n",
    "            try: \n",
    "                res = SearchResult(scraper = scraper, url = url, city = city, lawyer_type = lawyer_type)\n",
    "                res_vals = res.get_all_info()\n",
    "                data.append(res_vals)\n",
    "                if idx % 10 == 0: \n",
    "                    print(idx)\n",
    "            except: \n",
    "                continue\n",
    "        \n",
    "        # Create dataframe\n",
    "        df = pd.DataFrame(data)\n",
    "        df = df.reset_index(drop = False).rename(columns = {'index': 'rank'})\n",
    "        df.insert(0, 'keyword', keyword)\n",
    "        df.to_csv(keyword.lower().replace(\" \", \"_\") + \".csv\", index = False)\n",
    "\n",
    "        print(f'-- {keyword} COMPLETE -- ')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96b8687a",
   "metadata": {},
   "source": [
    "#### Moz Api Connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "98920e52",
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def divide_chunks(l, n):\n",
    "    for i in range(0, len(l), n):\n",
    "        yield l[i:i + n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "7b20fb6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Los Angeles', 'New York', 'Miami', 'Washington DC', 'Denver', 'Chicago']"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "city_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "id": "bcca43a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in df\n",
    "FILE_PATH = 'data/medical_malpractice/medical_malpractice_lawyer_los_angeles.csv'\n",
    "df = pd.read_csv(FILE_PATH)\n",
    "url_list = df['url'].tolist()\n",
    "url_list_chunked = list(divide_chunks(url_list, 50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "id": "c3e2adaa",
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# Config \n",
    "MOZ_API_ID = \"mozscape-a618b377a7\"\n",
    "MOZ_SECRET_KEY = \"c396802e89a0e12ce55c0839b96151d0\"\n",
    "auth = (MOZ_API_ID, MOZ_SECRET_KEY)\n",
    "\n",
    "# Request params and auth\n",
    "url = 'https://lsapi.seomoz.com/v2/url_metrics'\n",
    "\n",
    "# Iterate through each chunk and create df \n",
    "chunk_df = pd.DataFrame()\n",
    "for URL_CHUNK in url_list_chunked: \n",
    "\n",
    "    # Make request\n",
    "    data = '{\"targets\":' + json.dumps(URL_CHUNK) + '}'\n",
    "    req = requests.post(url, data = data, auth=auth)\n",
    "    res = req.json()['results']\n",
    "    \n",
    "    # Extract DA, PA, Spam, backlinks, no follow backlinks\n",
    "    keys = ['domain_authority', 'page_authority', 'spam_score', \n",
    "            'pages_to_root_domain', 'nofollow_pages_to_root_domain']\n",
    "    d = [{k:v for k,v in item.items() if k in keys} for item in res]\n",
    "    chunk_df = pd.concat([chunk_df, pd.DataFrame(d)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 483,
   "id": "6956a041",
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_cols = ['domain_authority', 'page_authority', 'spam_score']\n",
    "df = df.drop(drop_cols, axis = 1)\n",
    "combined_df = pd.concat([df.reset_index(drop=True), chunk_df.reset_index(drop=True)], axis = 1)\n",
    "\n",
    "front_cols = ['keyword', 'rank', 'url', 'domain_authority', 'page_authority', 'spam_score', 'pages_to_root_domain', \n",
    "             'nofollow_pages_to_root_domain']\n",
    "cols = front_cols + [col for col in combined_df if col not in front_cols]\n",
    "combined_df[cols].to_csv(FILE_PATH, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cadc8a5b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}