index.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests as rq\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lxml.etree\n",
    "import cssselect as css\n",
    "from IPython.core.display import display, HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def safer_extract_text(html_root, css_pattern):\n",
    "    possible_hits = html_root.xpath(css.HTMLTranslator().css_to_xpath(css_pattern))\n",
    "    if len(possible_hits) > 0:\n",
    "        return possible_hits[0].text\n",
    "    else:\n",
    "        return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def safer_extract_string(html_root, css_pattern):\n",
    "    possible_hits = html_root.xpath(css.HTMLTranslator().css_to_xpath(css_pattern))\n",
    "    if len(possible_hits) > 0:\n",
    "        return possible_hits[0].xpath(\"string()\")\n",
    "    else:\n",
    "        return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_details(page_text):\n",
    "    html_root = lxml.etree.HTML(page_text)\n",
    "\n",
    "    name        = safer_extract_text(html_root, 'div.box-title span')\n",
    "    breed       = safer_extract_text(html_root, 'div.breed-dog p')\n",
    "    gender      = safer_extract_text(html_root, 'div.gender p')\n",
    "    age         = safer_extract_text(html_root, 'div.age p')\n",
    "    weight      = safer_extract_text(html_root, 'div.weight p')\n",
    "    location    = safer_extract_string(html_root, 'div.location p')\n",
    "    description = safer_extract_string(html_root, 'h2:contains(Meet) + div')\n",
    "    \n",
    "    age_match = re.fullmatch(r'(\\d+) (Month(s?)|Year(s?))', age)\n",
    "    if age_match: \n",
    "        if age_match.group(2).startswith('Month'):\n",
    "            age = int(age_match.group(1))\n",
    "        else :\n",
    "            age = int(age_match.group(1)) * 12\n",
    "    else:\n",
    "        age = np.nan\n",
    "    \n",
    "    weight_match = re.fullmatch(r'(.*)lbs', weight)\n",
    "    if weight_match:\n",
    "        weight = float(weight_match.group(1))\n",
    "    else:\n",
    "        weight = np.nan\n",
    "        \n",
    "    description = re.sub('\\s+', ' ',description)\n",
    "    \n",
    "    image_link = ''\n",
    "    possible_images = html_root.xpath(css.HTMLTranslator().css_to_xpath('img.img-circle'))\n",
    "    if possible_images:\n",
    "        image_link = possible_images[0].get('src')\n",
    "    \n",
    "    return {\n",
    "        'Name': name,\n",
    "        'Breed': breed,\n",
    "        'Gender': gender,\n",
    "        'Age': age,\n",
    "        'Weight (lbs)': weight,\n",
    "        'Location': location,\n",
    "        'Description':  description,\n",
    "        'Image Source': image_link\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'http://www.pawschicago.org/pet-available-for-adoption/showdog/ace-8/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/adler-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/adonis-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/alaric/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/alda/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/aldo-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/benita-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/bob-4/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/braxton-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/brody-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/bruno-7/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/burton/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/cailean-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/calvin-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/caramel-cassiopeia-cantelope/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/chestnut-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/coco-6/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/colby-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/cuatro/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/dakota-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/darby-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/duncan-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/elisa-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/ellen-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/estelle/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/evgeniy/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/faith-4/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/finch-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/fremont-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/fudge-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/furby/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/gypsy-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/hampton-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/harley-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/harry-5/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/haywood/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/hera-4/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/hero-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/hopper/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/houston-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/huntley/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/ian-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/ibis/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/ida-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/ilana-ibis-isadora-and-ida/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/jay-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/jenson/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/jolene-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/josie-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/joyce-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/jubilee/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/judge-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/kino-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/landon-4/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/leila/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/lola-5/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/lola-lilo-lulu-landon-and-lily/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/lolo-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/lucas-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/lucy-and-lux/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/merlock/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/merrill-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/mia-10/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/miriam/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/mowgli-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/nightingale-wren/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/nish-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/noble-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/owen-4/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/paddy-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/padme-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/pam-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/patton-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/pepper-4/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/petula-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/peyton-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/phil-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/poli/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/quincy-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/rae-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/raina-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/rani-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/rhianna/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/rickey/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/sasha-4/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/shelby-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/sid-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/sloan-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/smokey-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/snoopy-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/sydney-5/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/tabitha-3/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/tally-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/tally/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/tegan-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/tod-1/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/trooper/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/vedder/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/vella-2/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/velma/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/wolfe/',\n",
       " 'http://www.pawschicago.org/pet-available-for-adoption/showdog/woody-4/'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dogs_page = rq.get(r'http://www.pawschicago.org/our-work/pet-adoption/pets-available/#dogsResults')\n",
    "some_dog_urls = re.findall(r'http://www.pawschicago.org/pet-available-for-adoption/showdog/.*/', dogs_page.text)\n",
    "\n",
    "more_dogs_data_url = re.search('.*dogs-load-more.*data-url=\"(.*)\"', dogs_page.text).group(1)\n",
    "more_dogs_result = rq.get(more_dogs_data_url)\n",
    "more_dog_urls = [dog['url'] for dog in more_dogs_result.json()['pets']]\n",
    "\n",
    "dog_urls = set(some_dog_urls + more_dog_urls)\n",
    "dog_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dog_page_texts = [p for p in map (lambda u: (rq.get(u).text, u), dog_urls)]\n",
    "#dog_page_texts = map (lambda u: rq.get(u).text, list(dog_urls)[:6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#df = pd.DataFrame(map(parse_details, dog_page_texts))\n",
    "details_dicts = []\n",
    "for pair in dog_page_texts:\n",
    "    details = parse_details(pair[0])\n",
    "    details['URL'] = pair[1]\n",
    "    details_dicts.append(details)\n",
    "df = pd.DataFrame(details_dicts)\n",
    "#df.set_index('Name',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_restrictions(description_text):\n",
    "    first_restriction = ''\n",
    "    second_restriction = ''\n",
    "    restriction_match = re.search(r'would do best (.*?)\\.', description_text)\n",
    "    if restriction_match:\n",
    "        first_restriction = restriction_match.group(1)\n",
    "    second_restriction_match = re.search(r'[Ww]ould also do best (.*?)\\.', description_text)\n",
    "    if second_restriction_match:\n",
    "        second_restriction = second_restriction_match.group(1)\n",
    "    return first_restriction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_name_link(row):\n",
    "    return r'<a href=\"' + row['URL'] + r'\">' + row['Name'] + r'</a>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_image_link(row):\n",
    "    return r'<a href=\"' + row['URL'] + r'\">' + r'<img src=\"' + row['Image Source'] + r'\"></a>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Restriction'] = df['Description'].apply(get_restrictions)\n",
    "df['Image Link'] = df.apply(make_image_link, axis=1)\n",
    "df['Name Link'] = df.apply(make_name_link, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def age_formater(age):\n",
    "    if np.isnan(age):\n",
    "        return 'NaN'\n",
    "    elif age < 12:\n",
    "        return str(int(age)) + \" Months\"\n",
    "    else:\n",
    "        return str(int(age / 12)) + \" Years\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def name_formatter(name):\n",
    "    if name: return '<a href=\"' + df.loc[name,'URL'] +'\">' + name + '</a>'\n",
    "    else: return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def image_formatter(image_link):\n",
    "    if image_link: return '<img src=\"'+image_link+'\">'\n",
    "    else: return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"dogs.html\", \"w\") as f:\n",
    "    f.write(\"<html><body>\")\n",
    "    f.write(df.sort_values(by=['Restriction','Age'], ascending=[True,False]).to_html(\n",
    "        escape=False,\n",
    "        columns = ['Name Link', 'Image Link','Breed','Gender','Age','Weight (lbs)','Restriction','Description', 'Location'],\n",
    "        formatters = {'Age': age_formater},\n",
    "        index=False\n",
    "    ))\n",
    "    f.write(\"</body></html>\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Output](dogs.html)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}