web-scraping-intro-final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to web scraping using Python\n",
    "## QUT DMRC - 2016\n",
    "\n",
    "###  Support for multiple pages\n",
    "\n",
    "This notebook scrapes http://www.metacritic.com/browse/albums/artist and saves the data in a dataframe.\n",
    "The script iterates through the webpage structure (structured by the first letter of the artist's name), and over multiple pages within each letter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# initialise plotting in the notebook\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import bs4\n",
    "import requests\n",
    "import pandas as pd\n",
    "from os.path import isfile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialise global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# this is the base_url\n",
    "base_url = \"http://www.metacritic.com/browse/albums/artist\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# the bot pretends to be a standard Mozilla browser\n",
    "hdrs = {\"User-Agent\": \"Mozilla/5.0\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# columns labels\n",
    "colnames = [\"artistname\", \"albumname\", \"release_date\", \"mc_score\", \"user_score\", \"url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The pages are sorted in alphabetical order based on the artist's name.\n",
    "# Artists with strange first letters are put in a page without a letter (the first one)\n",
    "letters = [\"\",\"/a\",\"/b\",\"/c\",\"/d\",\"/e\",\"/f\",\"/g\",\"/h\",\"/i\",\"/j\",\"/k\",\"/l\",\"/m\",\"/n\",\"/o\",\"/p\",\"/q\",\"/r\",\"/s\",\"/t\",\"/u\",\"/v\",\"/w\",\"/x\",\"/y\",\"/z\"]\n",
    "\n",
    "# if you want to limit the number of pages to scrape, you simply shorten this list - e.g.\n",
    "letters = [\"/a\",\"/b\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### Function definitions\n",
    "\n",
    "Add a new function to find the number of pages of reviews available for the current letter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# processes a beautiful_soup data structure and returns a the page count\n",
    "def get_page_count(thesoup):\n",
    "    # try to find all div tags of class \"pages\"\n",
    "    page_divs = soup.find_all(\"div\",class_=[\"pages\"])\n",
    "    page_count = 1\n",
    "    \n",
    "    # if there is a div of class \"pages\", then\n",
    "    if len(page_divs)>0:\n",
    "        \n",
    "        # try to find all a-tags\n",
    "        a_tags = page_divs[0].find_all(\"a\")\n",
    "        \n",
    "        # if there were a-tag(s) to be found, then pick the last one in order to get the max page number\n",
    "        if len(a_tags)>0:\n",
    "            page_count = int(a_tags[-1].get_text())\n",
    "\n",
    "    return page_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```get_itemlist``` function is unchanged from the previous page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# processes a beautiful_soup data structure and returns new album_reviews in a dataframe\n",
    "def get_itemlist(thesoup):\n",
    "    \n",
    "    #try to find all div-tags of class \"product_wrap\"\n",
    "    lotsofitems = thesoup.find_all(\"div\",class_=[\"product_wrap\"])\n",
    "    \n",
    "    thelist = []\n",
    "    for an_item in lotsofitems: \n",
    "        theitem = []\n",
    "        \n",
    "        # artistname\n",
    "        temptemp = an_item.find(\"li\",class_=\"product_artist\")\n",
    "        theitem += [temptemp.find(\"span\",class_=[\"data\"]).get_text()]\n",
    "\n",
    "        thetitle = an_item.find(\"div\",class_=\"product_title\")\n",
    "\n",
    "        # albumname\n",
    "        temptemp = thetitle.get_text()\n",
    "        temptemp = temptemp.split()\n",
    "        theitem += [\" \".join(temptemp)]\n",
    "        \n",
    "        # release_date\n",
    "        temptemp = an_item.find(\"li\",class_=\"release_date\")\n",
    "        theitem += [temptemp.find(\"span\",class_=[\"data\"]).get_text()]\n",
    "        \n",
    "        # mc_score\n",
    "        theitem += [an_item.find(\"div\",class_=\"metascore_w\").get_text()]\n",
    "\n",
    "        # user_score\n",
    "        temptemp = an_item.find(\"li\",class_=\"product_avguserscore\")\n",
    "        theitem += [temptemp.find(\"span\",class_=[\"data\"]).get_text()]\n",
    "        \n",
    "        # url\n",
    "        theitem += [\"http://www.metacritic.com\"+thetitle.a.attrs[\"href\"]]\n",
    "\n",
    "        # not all albums have both expert reviews and user reviews. Those albums\n",
    "        # that has data missing, use \"tbd\" instead. We only want to add items\n",
    "        # that have both user_score and mc_score\n",
    "        if not \"tbd\" in theitem:\n",
    "            thelist = thelist + [theitem]\n",
    "    return pd.DataFrame(thelist,columns=colnames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reset the dataframe\n",
    "\n",
    "# if there is a file...\n",
    "if isfile(\"reviews.pkl\"):\n",
    "    # ...load album_reviews from that file\n",
    "    album_reviews = pd.read_pickle(\"reviews.pkl\")\n",
    "else:\n",
    "    # otherwise, set up an empty dataframe\n",
    "    album_reviews = pd.DataFrame(columns=colnames)\n",
    "\n",
    "# show the number of reviews in the dataframe\n",
    "print(len(album_reviews))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# iterate over the list of letters\n",
    "for lett in letters:\n",
    "    \n",
    "    # initialise page_count and counter for this letter\n",
    "    page_count = 1\n",
    "    i = 0\n",
    "    \n",
    "    while i < page_count:\n",
    "        \n",
    "        # 1.build the url\n",
    "        thepage = base_url+lett+\"?page=\"+str(i)\n",
    "        \n",
    "        # 2.call the url\n",
    "        stuff = requests.get(thepage, headers=hdrs)\n",
    "        \n",
    "        # 3.transform to soup using html.parser parser\n",
    "        soup = bs4.BeautifulSoup(stuff.text, \"html.parser\")\n",
    "        \n",
    "        # 4.extract the new reviews from this page\n",
    "        new_reviews = get_itemlist(soup)\n",
    "        \n",
    "        # 5.add the new reviews to the dataframe\n",
    "        album_reviews = album_reviews.append(new_reviews)\n",
    "        \n",
    "        # 6.print something to show how the process progresses\n",
    "        print(\"URL:\",thepage,flush=True)\n",
    "        \n",
    "        \n",
    "        # if this is the first page for this letter, then extract the page count\n",
    "        if i == 0:\n",
    "            page_count = get_page_count(soup)\n",
    "        \n",
    "        # increase the counter\n",
    "        i += 1\n",
    "        \n",
    "    # *** Tidy up the data and save to disk after each letter has been scraped ***\n",
    "    # make sure the review scores are numerical (float) types\n",
    "    album_reviews[\"mc_score\"] = album_reviews[\"mc_score\"].map(float)\n",
    "    album_reviews[\"user_score\"] = album_reviews[\"user_score\"].map(float)\n",
    "        \n",
    "    # remove duplicates in case the same page has been scraped more than once\n",
    "    album_reviews = album_reviews.drop_duplicates()\n",
    "        \n",
    "    # save the reviews to a csv file\n",
    "    album_reviews.to_csv(\"reviews.csv\")\n",
    "        \n",
    "    # save the reviews to a pkl file\n",
    "    album_reviews.to_pickle(\"reviews.pkl\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check the result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# how many reviews are there in the dataframe?\n",
    "len(album_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# have a look at the first five items\n",
    "album_reviews[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "album_reviews[\"user_score_inv\"] = album_reviews[\"user_score\"].map(lambda x:1/x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "album_reviews[\"user_score_log\"] = album_reviews[\"user_score\"].map(log)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# histograms\n",
    "pp = album_reviews.hist(figsize = (12,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# scatter diagram\n",
    "pp = album_reviews.plot(kind=\"scatter\",x=\"user_score\",y=\"mc_score\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Statistical analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# simple correlations\n",
    "album_reviews.corr()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}