PySAL - Spatial Autocorrelation - Global Morans I - Start with this script.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Spatial Autocorrelation with PySAL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Global Moran's I exercise\n",
    "\n",
    "Using this Jupyter notebook, you will code the same process that we used in GeoDa to run the Univariate Global Moran's I to measure overall level of Spatial Autocorrelation.\n",
    "\n",
    "Adapted from https://pysal.readthedocs.org/en/latest/users/tutorials/autocorrelation.html#moran-s-i\n",
    "Information also available on pg 35 of the PySAL_Documentation.pdf included with the SDS Bootcamp materials.\n",
    "\n",
    "Note: to run a cell, click in the cell, then go to top menu and select Cell > Run Cells.\n",
    "You can also click in the cell, and hold down Ctrl and Enter together. \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Task #1: Explore and Visualize Dataset\n",
    "\n",
    "#### First, import the needed Python components - all scripting in Python begins with import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# PySAL and Numpy the only ones needed to actually run the spatial autocorrelation analysis\n",
    "import pysal\n",
    "import numpy as np\n",
    "\n",
    "#Folium is used to create some map visualizations\n",
    "import folium\n",
    "\n",
    "# These others are to handle, query, and plot data\n",
    "import os\n",
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "import json\n",
    "import simpledbf\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# This message below will print after the commands above are successfully completed\n",
    "print ('All requested Python libraries imported successfully')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Explore the data table using pandas \n",
    "\n",
    "# Use simpledbf to read the attribute table (dbf file)\n",
    "dbf = simpledbf.Dbf5('/home/ubuntu/Documents/Counties/cnty_Lyme_disease.dbf')\n",
    "\n",
    "# Convert dbf file to a pandas data frame\n",
    "df = dbf.to_dataframe()\n",
    "\n",
    "# Create a list of the columns and print the column names \n",
    "columns = list(df)\n",
    "print(columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Extract the values for 2005 \n",
    "extracted = df.loc[:,[\"County\",\"2005\"]]\n",
    "\n",
    "# Print the extracted values for 2005\n",
    "extracted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Order the counties by highest to lowest count for 2005 \n",
    "# http://pandas.pydata.org/pandas-docs/version/0.18.1/generated/pandas.DataFrame.sort.html\n",
    "\n",
    "# ascending is the parameter used to define the sorting order\n",
    "# a value of 1 for ascending means true and orders in ascending order\n",
    "# a value of 0 for ascending means false and orders in descending order instead\n",
    "result = extracted.sort_values(by=[\"2005\", \"County\"], ascending=[0, 1])\n",
    "\n",
    "# Print the sorted values for 2005\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# We can also query values for specific counties from our extracted dataframe for 2005\n",
    "print (extracted[extracted.County == 'Alameda'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# To use Folium for map visualizations, we convert the shapefile to GeoJSON\n",
    "\n",
    "# We can use GeoPandas for this conversion\n",
    "# First, read in the shapefile to geopandas\n",
    "shapefile = gpd.read_file('/home/ubuntu/Documents/Counties/viz/cnty_Lyme_disease_WGS84.shp').set_index('NAME_PCASE')\n",
    "\n",
    "# Next, save the file out to GeoJSON\n",
    "with open('/home/ubuntu/Documents/Counties/viz/cnty_Lyme_disease_WGS84.geojson', 'w') as f:\n",
    "    f.write(shapefile.to_json())\n",
    "\n",
    "# This message below will print after the commands above are successfully completed\n",
    "print ('Successfully converted shapefile to geojson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Explore the data visually - create a basic visualization map of California counties\n",
    "\n",
    "# Read in the GeoJSON file created in previous step\n",
    "counties = '/home/ubuntu/Documents/Counties/viz/cnty_Lyme_disease_WGS84.geojson'\n",
    "geo_json_data = json.load(open(counties))\n",
    "\n",
    "# Create an empty Folium map and fill it with the GeoJSON data\n",
    "m = folium.Map([37, -122], zoom_start=6)\n",
    "folium.GeoJson(geo_json_data).add_to(m)\n",
    "\n",
    "# Save the map to html\n",
    "m.save('/home/ubuntu/Documents/Counties/viz/basic_countymap.html')\n",
    "\n",
    "# Print the map to screen\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Explore the data visually - create a choropleth map of the 2005 counts of lyme disease \n",
    "\n",
    "# Use simpledbf to read the attribute table (dbf file)\n",
    "dbf = simpledbf.Dbf5('/home/ubuntu/Documents/Counties/cnty_Lyme_disease.dbf')\n",
    "\n",
    "# Convert dbf file to a pandas data frame\n",
    "df = dbf.to_dataframe()\n",
    "\n",
    "# Extract the values for 2005 \n",
    "extracted = df.loc[:,[\"County\",\"2005\"]]\n",
    "\n",
    "# Create an empty Folium map\n",
    "m = folium.Map([37, -122], zoom_start=6)\n",
    "\n",
    "# Join the extracted 2005 values to the map\n",
    "m.choropleth(\n",
    "    geo_str=open(counties).read(),\n",
    "    data=extracted,\n",
    "    columns=['County', '2005'], \n",
    "    key_on='properties.County',\n",
    "    fill_color='YlGn',\n",
    ")\n",
    "\n",
    "# Save the map as html\n",
    "m.save(os.path.join(\"/home/ubuntu/Documents/Counties/viz/2005_countymap.html\"))\n",
    "\n",
    "# Print the map to screen\n",
    "m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Task #2: read data into PySAL and create new variables for analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### To calculate Moran's I, we need to give it a list of neighbors. We can do this by reading in a spatial weights matrix.\n",
    "Remember that PySAL likes the GAL file format, which we created in the GeoDa exercise. \n",
    "\n",
    "This file can be converted to an ArcGIS Spatial Weights Matrix (SWM) file, and vice versa.\n",
    "PySAL can read the GAL file as follows: counties = pysal.open('path/to/file/called/file.gal').read()\n",
    "\n",
    "#### Alternatively, and much easier, PySAL can read the neighbors directly from a shapefile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Instead of reading in the .gal file we created in GeoDa, we will ask PySAL to create one from the shapefile.\n",
    "counties = pysal.queen_from_shapefile('/home/ubuntu/Documents/Counties/cnty_Lyme_disease.shp')\n",
    "\n",
    "# This message below will print after the command above is successfully completed\n",
    "print ('New weights file successfully created')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### The queen_from_shapefile function has defined neighbors using the the queen weights criteria, which defines a location's neighbors as those areas with at least one shared corner\n",
    "\n",
    "#### Other options include a rook weights matrix, in which neighbors need to share an entire border (i.e. a line of two connected vertices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# counties has some atributes associated with it\n",
    "# For example, n which is equal to number of features in the data\n",
    "print (counties.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Use the help function to find out what other attributes you can call\n",
    "# You can clear/remove this output from your view when you are done\n",
    "# Go to Cell > Current Outputs > Clear\n",
    "help(counties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Remember it is important to make sure that you have no \"islands\" in your dataset\n",
    "# Which attribute of counties can you use to list islands?\n",
    "counties.islands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Next, let's read in the dbf that contains data for the counties\n",
    "# http://www.pysal.org/users/tutorials/fileio.html\n",
    "\n",
    "table = pysal.open('/home/ubuntu/Documents/Counties/cnty_Lyme_disease.dbf')\n",
    "\n",
    "# This message below will print after the command above is successfully completed\n",
    "print ('DBF file successfully imported')\n",
    "\n",
    "# Return the column headers from dbf\n",
    "table.header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Next, specify which column contain the variable of interest\n",
    "# Notice that we are using the array function from numpy, which we named np during our import\n",
    "# This array will contain the data from the column called 2005\n",
    "\n",
    "lymecases = np.array(table.by_col['2005'])\n",
    "\n",
    "# This message below will print after the command above is successfully completed\n",
    "print ('A variable called lymecases successfully created')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Task #3: complete a single run of Global Moran's I to measure overall level of Spatial Autocorrelation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Using the functions examples below, update the parameters to run Global Moran's I on the year 2005 data\n",
    "\n",
    "# In the online tutorial, the function reads as follows: mi = pysal.Moran(y,w)\n",
    "# In their example, y = array containing homicide rates and w = spatial weights variable for the neighbors\n",
    "\n",
    "# Another example could be something like: mi = pysal.Moran(crimeindex, blockgroups)\n",
    "# In this example, crimeindex is the array containing a crime index and blockgroups is the spatial weights variable \n",
    "\n",
    "# Hint: mi = pysal.Moran(y,w)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Again, examine the help to learn more about the outputs from your function\n",
    "#help(mi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Which attribute of mi can you use to see the actual observed value of spatial autocorrelation in the dataset?\n",
    "# Hint: mi.attributename\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Which attribute of mi can you use to see the expected value of spatial autocorrelation? \n",
    "# Hint: mi.attributename\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Which attribute of mi can you use to see the statistical significance of difference between I and EI?\n",
    "# Remember the goal is p < 0.05\n",
    "# Hint: mi.attributename\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Task #4: check whether null hypothesis is rejected after a single run of Global Moran's I"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Use your cheat sheet and the example below to write an If/Else statement to print a message about the p value\n",
    "# If the p value is less than 0.05, then you can reject the null hypothesis that the lyme disease cases are randomly distributed\n",
    "\n",
    "# useful online resources: \n",
    "# https://docs.python.org/2/tutorial/controlflow.html\n",
    "# http://learnpythonthehardway.org/book/ex30.html\n",
    "\n",
    "# Hint: If/Else Statement From the Presentation Slides\n",
    "# if x < 2:\n",
    "    # print ('Some Message.')\n",
    "# else:\n",
    "    # print ('A different message.')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Task #5: add permutations (multiple runs) of the random landscape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create a new instance of the pysal.Moran function that includes a parameter value for permutations\n",
    "# Name this new instance: mir\n",
    "\n",
    "# For more info: https://pysal.readthedocs.org/en/latest/users/tutorials/autocorrelation.html#moran-s-i\n",
    "# Hint: Examine the function mir and what parameters are allowed\n",
    "# Hint: help(mir)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Which attribute of mir can you use to see the actual observed value of spatial autocorrelation in the dataset?\n",
    "# Hint: help(mir)\n",
    "# Hint: mir.attributename\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Which attribute of mir can you use to see the expected value of spatial autocorrelation based on the randomized versions of the data (simulations)? \n",
    "# Hint: mir.attributename\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Which attribute of mir can you use to see the statistical significance of difference between observed I and simulated I values?\n",
    "# Remember the goal is p < 0.05\n",
    "# Hint: mir.attributename\n",
    "# Hint: the output variable with the p value has a slightly different name than before\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Does the pseudo p value support that there is statistical difference between the observed I and simulated I values?\n",
    "\n",
    "# Maybe a visualization can help - plot the distribution of random permutations for Moran's I, similar to the one in GeoDa\n",
    "# Compare the observed I of the dataset to the distribution of I for the random permutations\n",
    "\n",
    "# Create an empty list to store values after each run \n",
    "permutation = []\n",
    "\n",
    "# Calculate the Moran's I for 1000 random distributions of the dataset and store them in a list called permutation\n",
    "for i in range(0, 1000):\n",
    "    new_permute = np.random.choice(lymecases,size = len(lymecases), replace = False)\n",
    "    permuted_moran = pysal.Moran(new_permute,counties)\n",
    "    permutation.append(permuted_moran.I)\n",
    "\n",
    "# Plot the histogram    \n",
    "plt.hist(permutation)\n",
    "\n",
    "# Add a verrtical line to display the actual value Moran's I of the dataset\n",
    "plt.axvline(x= mi.I, color='y', linestyle='--', linewidth = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Task #6:  check whether null hypothesis is rejected after multiple runs of Global Moran's I"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Again, Use your cheat sheet and the example below to write an If/Else statement to print a message about the p value\n",
    "# If the p value is less than 0.05, then you can reject the null hypothesis that the lyme disease cases are randomly distributed\n",
    "\n",
    "# Hint: the output variable with the p value has a slightly different name than before\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Bonus Exercises for Global Moran's I exercise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Bonus #1: examine the \"z transformation\" values for added robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Which attributes of mir show you the z transformation values?\n",
    "# For more info: https://pysal.readthedocs.org/en/latest/users/tutorials/autocorrelation.html#moran-s-i\n",
    "# Hint: help(mir) and look for output variables with \"z\" in the name\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Bonus #2:  check whether null hypothesis is rejected based on \"z transformation\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Again, use your cheat sheet and the example below to write an If/Else statement to print a message \n",
    "# this time, print a message about the z value\n",
    "\n",
    "# based on a 95% confidence level, z-value between -1.96 and +1.96 does not reject null hypothesis\n",
    "# this would mean that the data are randomly distributed through space\n",
    "\n",
    "# Hint: the output variable with the p value has a slightly different name than before\n",
    "# Hint: you will need to use an \"or\" in the if/else statement\n",
    "# useful online resources: \n",
    "# https://docs.python.org/2/tutorial/controlflow.html\n",
    "# http://learnpythonthehardway.org/book/ex30.html\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Advanced Bonus Exercises for Global Moran's I exercise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Advanced Bonus #1: use a loop to run Global Moran's I for other years of data (2006 to 2014)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Note: these advanced bonus exercises may be challenging for new Python programmers \n",
    "# Hint: check out Lists as an iterable on: https://wiki.python.org/moin/ForLoop\n",
    "# Hint: create a list of the years to begin\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Note: these advanced bonus exercises are challenging for new Python programmers \n",
    "# Hint: check out Lists as an iterable on: https://wiki.python.org/moin/ForLoop \n",
    "# Hint: now use a loop to execute mir for each of the years in your list\n",
    "# Hint: use 'print' to display your output and be sure to output mir.p_z_sim and mir.z_sim\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Advanced Bonus #2: expand your loop to capture results from all years into pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Note: these advanced bonus exercises are challenging for new Python programmers \n",
    "# Hint: check out Lists as an iterable on: https://wiki.python.org/moin/ForLoop \n",
    "# Hint: check out Append on http://pandas.pydata.org/pandas-docs/stable/10min.html \n",
    "# Hint: create some empty lists and expand your loop to include a few appends to these empty lists\n",
    "# Hint: use 'print' to display your output and be sure to output mir.p_z_sim and mir.z_sim\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Advanced Bonus #3: query your pandas dataframe for years with significant Global Moran's I"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Note: these advanced bonus exercises are challenging for new Python programmers \n",
    "# Hint: check out Boolean Indexing on http://pandas.pydata.org/pandas-docs/stable/10min.html \n",
    "# Hint: query for p values < 0.05 in the pandas dataframe\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Final thoughts: Notice that none of our z values are negative which would indicate dispersion in the data.  Most years do not have a significant p value of the difference between the observed I and the expected value of I, thus indicating that the lyme disease cases appear to be randomly distributed.  Similarly, most years also do not have significant positive z values (which would indicate a global clustered pattern).\n",
    "\n",
    "#### For the years that you do reject the null hypothesis (random distribution), what does this mean, as there are not many of them?\n",
    "\n",
    "#### Does all this mean that there is no spatial autocorrelation in lyme disease cases in California?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## End of Global Moran's I exercise\n",
    "\n",
    "\n",
    "### Other options to continue your exploration:\n",
    "#### Move on to the next exercise for the Local Indicators of Spatial Autocorrelation exercise to explore local hot and cold spots (PySAL - Spatial Autocorrelation - Local Morans I - Start with this script.ipynb)\n",
    "\n",
    "#### You can continue to explore pandas; can you limit the last dataframe again to only significant z values? (refer to Bonus #2) http://pandas.pydata.org/pandas-docs/stable/10min.html\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Hint: check out Boolean Indexing on http://pandas.pydata.org/pandas-docs/stable/10min.html\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Or just for fun: explore some more advanced visualizations below "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Explore the data visually - create a choropleth map of the total disease count from 2005-2014 (sum)\n",
    "\n",
    "# Read in the GeoJSON file created earlier\n",
    "counties = '/home/ubuntu/Documents/Counties/viz/cnty_Lyme_disease_WGS84.geojson'\n",
    "geo_json_data = json.load(open(counties))\n",
    "\n",
    "# Use simpledbf to read the attribute table (dbf file)\n",
    "dbf = simpledbf.Dbf5('/home/ubuntu/Documents/Counties/cnty_Lyme_disease.dbf')\n",
    "\n",
    "# Convert dbf file to a pandas data frame\n",
    "df = dbf.to_dataframe()\n",
    "\n",
    "# Compute the sum of number of diseases from 2005 to 2014\n",
    "df[\"sum\"] = df.loc[:,\"2005\":\"2014\"].sum(axis = 1)\n",
    "\n",
    "# Extract the columns needed for the choropleth\n",
    "extracted = df.loc[:,[\"County\",\"sum\"]]\n",
    "\n",
    "# Create an empty Folium map\n",
    "m = folium.Map([37, -122], zoom_start=6)\n",
    "\n",
    "# Join the extracted values to the map\n",
    "m.choropleth(\n",
    "    geo_str=open(counties).read(),\n",
    "    data=extracted,\n",
    "    columns=['County', 'sum'],\n",
    "    key_on='properties.County',\n",
    "    fill_color='YlGn',\n",
    ")\n",
    "\n",
    "# Save the map as html\n",
    "m.save(os.path.join(\"/home/ubuntu/Documents/Counties/viz/sum_countymap.html\"))\n",
    "\n",
    "# Print the map to screen\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Explore the data visually - create a choropleth map of the range of disease counts from 2005-2014 (max-min)\n",
    "\n",
    "# Read in the GeoJSON file created earlier\n",
    "counties = '/home/ubuntu/Documents/Counties/viz/cnty_Lyme_disease_WGS84.geojson'\n",
    "geo_json_data = json.load(open(counties))\n",
    "\n",
    "# Use simpledbf to read the attribute table (dbf file)\n",
    "dbf = simpledbf.Dbf5('/home/ubuntu/Documents/Counties/cnty_Lyme_disease.dbf')\n",
    "\n",
    "# Convert dbf file to a pandas data frame\n",
    "df = dbf.to_dataframe()\n",
    "\n",
    "# First, identify the maximum value for each county across the time period\n",
    "max_value =[max(df.loc[:,\"2005\":\"2014\"].ix[i]) for i in range(0,len(df[\"2005\"]))]\n",
    "\n",
    "# Next, identify the minimum value for each county across the time period\n",
    "min_value = [min(df.loc[:,\"2005\":\"2014\"].ix[i]) for i in range(0,len(df[\"2005\"]))]\n",
    "\n",
    "# Calculate the range of the values and add back to data frame as new column called range\n",
    "df[\"range\"] = np.array(max_value) - np.array(min_value)\n",
    "\n",
    "# Extract the columns needed for the choropleth\n",
    "extracted = df.loc[:,[\"County\",\"range\"]]\n",
    "\n",
    "# Create an empty Folium map\n",
    "m = folium.Map([37, -122], zoom_start=6)\n",
    "\n",
    "# Join the extracted values to the map\n",
    "m.choropleth(\n",
    "    geo_str=open(counties).read(),\n",
    "    data=extracted,\n",
    "    columns=['County', 'range'],\n",
    "    key_on='properties.County',\n",
    "    fill_color='YlOrRd',\n",
    ")\n",
    "\n",
    "# Save the map as html\n",
    "m.save(os.path.join(\"/home/ubuntu/Documents/Counties/viz/range_countymap.html\"))\n",
    "\n",
    "# Print the map to screen\n",
    "m"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}