02_data_exploration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 2: Data Exploration and Visualization in Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "----------------------------------------------------------------------\n",
    "Filename : 02_data_exploration.py\n",
    "Date     : 12th Dec, 2013\n",
    "Author   : Jaidev Deshpande\n",
    "Purpose  : Introducing exploratory data analysis with Pandas.\n",
    "Libraries: Pandas, NumPy, SciPy, Matplotlib\n",
    "----------------------------------------------------------------------\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# standard library imports\n",
    "from os.path import join\n",
    "\n",
    "# system library imports\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Problem: Describing a dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# Load the data\n",
    "filename = join('data','01_heights_weights_genders.csv')\n",
    "hwg = pd.read_csv(filename, sep=',')\n",
    "heights = hwg['Height']\n",
    "weights = hwg['Weight']\n",
    "gender = hwg['Gender']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "summary_stats = ['min','max','mean','median','var','std','nunique']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def print_summary_statistics(series):\n",
    "    '''\n",
    "    Given a pandas series, this function prints it's summary statistics,\n",
    "    particularly: \n",
    "    1. minimum and maximum values,\n",
    "    2. mean\n",
    "    3. median\n",
    "    4. variance\n",
    "    5. standard deviation\n",
    "    6. number of unique values\n",
    "    '''\n",
    "    print(\"=\"*80)\n",
    "    print(\"Summary Statistics\")\n",
    "    print(\"=\"*80)\n",
    "    for stat in summary_stats:\n",
    "        attr = series.__getattribute__(stat)\n",
    "        value = attr()\n",
    "        print(stat.capitalize() + \":\" + \"\\t\" + str(value))\n",
    "    print(\"=\"*80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "print_summary_statistics(heights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# DataFrame.describe??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def plot_sumstats_visualizations(series, show_hist=False):\n",
    "    '''\n",
    "    Given a series, visualize it's summary statistics.\n",
    "    '''\n",
    "    fig, axes = plt.subplots(nrows=2,ncols=2, figsize=(20,10))\n",
    "\n",
    "    # Min & Max\n",
    "    ax = axes[0,0]\n",
    "    ax.axvline(heights.min(),color='g',linewidth=3)\n",
    "    ax.axvline(heights.max(),color='g',linewidth=3)\n",
    "    ax.set_title(\"Min & Max\")\n",
    "    ax.set_xlabel('Height in \"')\n",
    "    ax.set_ylabel(\"# People\")\n",
    "\n",
    "    # Mean\n",
    "    ax = axes[0,1]\n",
    "    ax.axvline(heights.mean(),color='r',linewidth=3)\n",
    "    ax.set_title(\"Mean\")\n",
    "    ax.set_xlabel('Height in \"')\n",
    "    ax.set_ylabel(\"# People\")\n",
    "\n",
    "    # Median\n",
    "    ax = axes[1,0]\n",
    "    ax.axvline(heights.median(),color='k',linewidth=3)\n",
    "    ax.set_title(\"Median\")\n",
    "    ax.set_xlabel('Height in \"')\n",
    "    ax.set_ylabel(\"# People\")\n",
    "\n",
    "    # Standard Deviation\n",
    "    ax = axes[1,1]\n",
    "    sd0     = heights.mean() + heights.std()\n",
    "    sd1     = heights.mean() + 2*heights.std()\n",
    "    sd_neg1 = heights.mean() - heights.std()\n",
    "    ax.axvline(heights.mean(),color='r',linewidth=3)\n",
    "    ax.axvline(sd0,color='c',linewidth=3)\n",
    "    ax.axvline(sd1,color='c',linewidth=3)\n",
    "    ax.axvline(sd_neg1,color='c',linewidth=3)\n",
    "    ax.set_title(\"Standard Deviation\")\n",
    "    ax.set_xlabel('Height in \"')\n",
    "    ax.set_ylabel(\"# People\")\n",
    "    \n",
    "    if show_hist:\n",
    "        series.hist(ax=axes[0,0],fc='steelblue')\n",
    "        series.hist(ax=axes[0,1],fc='steelblue')\n",
    "        series.hist(ax=axes[1,0],fc='steelblue')\n",
    "        series.hist(ax=axes[1,1],fc='steelblue')\n",
    "        \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "plot_sumstats_visualizations(heights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "plot_sumstats_visualizations(heights, show_hist=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Problem: How many bins in the histogram?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def draw_density_estimates(df,cols):\n",
    "    '''\n",
    "    Draw the kernel density estimates of data in `df`. \n",
    "    `cols` is a list of column names that specifies which columns to consider.\n",
    "    '''\n",
    "    for col in cols:\n",
    "        plt.figure(figsize=(20,5))\n",
    "        df[col].plot(kind='kde',label=col)\n",
    "        plt.xlabel(col)\n",
    "        plt.ylabel('Density')\n",
    "        plt.title('Density Plot')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "draw_density_estimates(hwg, cols=['Height', 'Weight'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def draw_density_estimates_by_gender(df, col='Height'):\n",
    "    \"\"\"\n",
    "    Same as `draw_density_estimates`, except \n",
    "    this time gender information is taken into account.\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(20,5))\n",
    "    male_heights = df[df['Gender']=='Male'][col]\n",
    "    female_heights = df[df['Gender']=='Female'][col]\n",
    "    male_heights.plot(kind='kde',label='Male',legend=True)\n",
    "    female_heights.plot(kind='kde',label='Female',legend=True)\n",
    "    plt.ylabel('Density')\n",
    "    plt.xlabel(col)\n",
    "    plt.title('Density Plot')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "draw_density_estimates_by_gender(hwg)\n",
    "draw_density_estimates_by_gender(hwg,'Weight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def draw_scatter_plot(df,cols=('Height','Weight')):\n",
    "    fig = plt.figure(figsize=(20,10))\n",
    "    axes = fig.add_subplot(111)\n",
    "    pd.tools.plotting.scatter_plot(df, cols[0], cols[1],c='steelblue',ax=axes)\n",
    "    plt.grid()\n",
    "    plt.xlabel(cols[0])\n",
    "    plt.ylabel(cols[1])\n",
    "    plt.title('Scatter Plot')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "draw_scatter_plot(hwg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def draw_binary_color_scatter_plot(df, x, y, colorby):\n",
    "    \"\"\"\n",
    "    Draw a scatter plot of data in `df`, with `x` and `y` specifying the columns,\n",
    "    and `colorby` specifying which column to use for mapping to colors.\n",
    "    `colorby` should be binary data.\n",
    "    \"\"\"\n",
    "    a = df[df[colorby]==True]\n",
    "    b = df[df[colorby]==False]\n",
    "    fig = plt.figure(figsize=(20,10))\n",
    "    axes = fig.add_subplot(111)\n",
    "    pd.tools.plotting.scatter_plot(a, x, y, ax=axes, color='0',label='Male')\n",
    "    pd.tools.plotting.scatter_plot(b, x, y, ax=axes, color='0.5', label='Female')\n",
    "    plt.legend()\n",
    "    plt.grid()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "male_inds = [gender == 'Male' for gender in hwg['Gender']]\n",
    "hwg['colorby'] = male_inds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "draw_binary_color_scatter_plot(hwg, 'Height','Weight','colorby')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}