weather_scrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import math\n",
    "import json\n",
    "import pickle\n",
    "import time\n",
    "\n",
    "# spatial libs\n",
    "import googlemaps\n",
    "from geopandas import GeoDataFrame\n",
    "from shapely.geometry import Point\n",
    "from shapely.ops import nearest_points\n",
    "\n",
    "# scraping libs\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# all US cities exceeding 50K population in 2015 census\n",
    "cities = pd.read_csv('us_cities_2015.csv')\n",
    "# state name - abbreviation mapping\n",
    "abbrev = pd.read_csv('state_abbrev.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### weather stations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# scrape the NOAA website for a list of weather stations for which \n",
    "# there is weather data for 2015, 2016, and 2017\n",
    "# there are two different ID types for each station: USAF and WBAN\n",
    "# I'll primarily use USAF, but carry along WBAN too\n",
    "def get_usaf_wban(link):\n",
    "    href = link.get('href')\n",
    "    usaf = int(href[:6])\n",
    "    wban = int(href[7:12])\n",
    "    return (usaf, wban)\n",
    "\n",
    "available_usaf_dict = {}\n",
    "\n",
    "for year in ['2015', '2016', '2017']:\n",
    "    \n",
    "    # use beautifulsoup to parse the HTML\n",
    "    query_string = 'https://www1.ncdc.noaa.gov/pub/data/noaa/isd-lite/{}/'.format(year)\n",
    "    html = urllib.urlopen(query_string).read()\n",
    "    soup = BeautifulSoup(html, 'lxml')\n",
    "    \n",
    "    # isolate the USAF identifier from each embedded URL\n",
    "    # first five aren't consistent/useful\n",
    "    all_links = soup.find_all('a')[5:]\n",
    "    available_usaf_yr = map(get_usaf_wban, all_links)\n",
    "    available_usaf_dict[year] = available_usaf_yr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ensure that I only deal with weather stations that have data in all three years\n",
    "available_usaf = list(set(available_usaf_dict['2015'])&\\\n",
    "                      set(available_usaf_dict['2016'])&\\\n",
    "                      set(available_usaf_dict['2017']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# all weather stations available from ncdc/noaa, with geographic locations\n",
    "stations = pd.read_csv('https://www1.ncdc.noaa.gov/pub/data/noaa/isd-history.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cleaning weather stations df; US only, exclude stations with missing information\n",
    "stations = stations[stations.CTRY=='US']\n",
    "stations.dropna(axis=0, how='any', inplace=True,\\\n",
    "                subset=['USAF', 'WBAN', 'STATION NAME', 'STATE', 'LAT', 'LON'])\n",
    "stations.drop(['ICAO', 'ELEV(M)', 'BEGIN', 'END'], axis=1, inplace=True)\n",
    "\n",
    "# 9's represent missing value in this dataset\n",
    "stations = stations[stations.USAF != 999999]\n",
    "# only stations for which there is data!\n",
    "mask = stations.apply(lambda row: (row['USAF'], row['WBAN']) in available_usaf, axis=1)\n",
    "stations = stations.loc[mask, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USAF</th>\n",
       "      <th>WBAN</th>\n",
       "      <th>STATION NAME</th>\n",
       "      <th>CTRY</th>\n",
       "      <th>STATE</th>\n",
       "      <th>LAT</th>\n",
       "      <th>LON</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14466</th>\n",
       "      <td>690150</td>\n",
       "      <td>93121</td>\n",
       "      <td>TWENTY NINE PALMS</td>\n",
       "      <td>US</td>\n",
       "      <td>CA</td>\n",
       "      <td>34.300</td>\n",
       "      <td>-116.167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14474</th>\n",
       "      <td>690190</td>\n",
       "      <td>13910</td>\n",
       "      <td>ABILENE DYESS AFB</td>\n",
       "      <td>US</td>\n",
       "      <td>TX</td>\n",
       "      <td>32.433</td>\n",
       "      <td>-99.850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15106</th>\n",
       "      <td>699604</td>\n",
       "      <td>3145</td>\n",
       "      <td>YUMA MCAS</td>\n",
       "      <td>US</td>\n",
       "      <td>AZ</td>\n",
       "      <td>32.650</td>\n",
       "      <td>-114.617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15107</th>\n",
       "      <td>700001</td>\n",
       "      <td>26492</td>\n",
       "      <td>PORTAGE GLACIER</td>\n",
       "      <td>US</td>\n",
       "      <td>AK</td>\n",
       "      <td>60.785</td>\n",
       "      <td>-148.839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15109</th>\n",
       "      <td>700197</td>\n",
       "      <td>26558</td>\n",
       "      <td>SELAWIK</td>\n",
       "      <td>US</td>\n",
       "      <td>AK</td>\n",
       "      <td>66.600</td>\n",
       "      <td>-159.986</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         USAF   WBAN       STATION NAME CTRY STATE     LAT      LON\n",
       "14466  690150  93121  TWENTY NINE PALMS   US    CA  34.300 -116.167\n",
       "14474  690190  13910  ABILENE DYESS AFB   US    TX  32.433  -99.850\n",
       "15106  699604   3145          YUMA MCAS   US    AZ  32.650 -114.617\n",
       "15107  700001  26492    PORTAGE GLACIER   US    AK  60.785 -148.839\n",
       "15109  700197  26558            SELAWIK   US    AK  66.600 -159.986"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stations.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cities "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_abbrev(state):\n",
    "    \"\"\"\n",
    "    maps a full US state name to its abbreviation\n",
    "    \"\"\"\n",
    "    try:\n",
    "        return abbrev[abbrev.state == state]['abbrev'].values[0]\n",
    "    except:\n",
    "        return 'nah'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cleaning the cities df\n",
    "cities['state'] = map(lambda place: place.split(', ')[1], cities.loc[:, 'city'])\n",
    "cities['state_abb'] = map(lambda state: get_abbrev(state), cities.loc[:, 'state'])\n",
    "cities['city'] = map(lambda place: place.split(', ')[0], cities.loc[:, 'city'])\n",
    "cities['population'] = map(lambda p: int(p.replace(',', '')), cities.loc[:, 'pop'])\n",
    "cities.drop(['pop'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create a googlemaps API client\n",
    "with open('secrets.json') as f:    \n",
    "    creds = json.load(f)\n",
    "    google_client = googlemaps.Client(key=creds[\"googlemaps\"][\"key\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get a latlon from google API for each city\n",
    "def get_point(row):\n",
    "    # query google API with city and state string; response is geolocation\n",
    "    city_string = row['city'] + ', ' + row['state_abb']\n",
    "    geo_response = google_client.geocode(city_string)\n",
    "    loc = geo_response[0][\"geometry\"][\"location\"]\n",
    "    \n",
    "    # create and return a shapely point from the response\n",
    "    return Point(loc['lat'], loc['lng'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cities['location'] = cities.apply(get_point, axis=1)\n",
    "stations['location'] = stations.apply(lambda row: Point(row.LAT, row.LON), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "crs = {'init': 'epsg:4326'}\n",
    "geo_cities = GeoDataFrame(cities, crs=crs, geometry='location')\n",
    "geo_stations = GeoDataFrame(stations, crs=crs, geometry='location')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USAF</th>\n",
       "      <th>WBAN</th>\n",
       "      <th>STATION NAME</th>\n",
       "      <th>CTRY</th>\n",
       "      <th>STATE</th>\n",
       "      <th>LAT</th>\n",
       "      <th>LON</th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14466</th>\n",
       "      <td>690150</td>\n",
       "      <td>93121</td>\n",
       "      <td>TWENTY NINE PALMS</td>\n",
       "      <td>US</td>\n",
       "      <td>CA</td>\n",
       "      <td>34.300</td>\n",
       "      <td>-116.167</td>\n",
       "      <td>POINT (34.3 -116.167)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14474</th>\n",
       "      <td>690190</td>\n",
       "      <td>13910</td>\n",
       "      <td>ABILENE DYESS AFB</td>\n",
       "      <td>US</td>\n",
       "      <td>TX</td>\n",
       "      <td>32.433</td>\n",
       "      <td>-99.850</td>\n",
       "      <td>POINT (32.433 -99.84999999999999)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15106</th>\n",
       "      <td>699604</td>\n",
       "      <td>3145</td>\n",
       "      <td>YUMA MCAS</td>\n",
       "      <td>US</td>\n",
       "      <td>AZ</td>\n",
       "      <td>32.650</td>\n",
       "      <td>-114.617</td>\n",
       "      <td>POINT (32.65 -114.617)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15107</th>\n",
       "      <td>700001</td>\n",
       "      <td>26492</td>\n",
       "      <td>PORTAGE GLACIER</td>\n",
       "      <td>US</td>\n",
       "      <td>AK</td>\n",
       "      <td>60.785</td>\n",
       "      <td>-148.839</td>\n",
       "      <td>POINT (60.785 -148.839)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15109</th>\n",
       "      <td>700197</td>\n",
       "      <td>26558</td>\n",
       "      <td>SELAWIK</td>\n",
       "      <td>US</td>\n",
       "      <td>AK</td>\n",
       "      <td>66.600</td>\n",
       "      <td>-159.986</td>\n",
       "      <td>POINT (66.59999999999999 -159.986)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         USAF   WBAN       STATION NAME CTRY STATE     LAT      LON  \\\n",
       "14466  690150  93121  TWENTY NINE PALMS   US    CA  34.300 -116.167   \n",
       "14474  690190  13910  ABILENE DYESS AFB   US    TX  32.433  -99.850   \n",
       "15106  699604   3145          YUMA MCAS   US    AZ  32.650 -114.617   \n",
       "15107  700001  26492    PORTAGE GLACIER   US    AK  60.785 -148.839   \n",
       "15109  700197  26558            SELAWIK   US    AK  66.600 -159.986   \n",
       "\n",
       "                                 location  \n",
       "14466               POINT (34.3 -116.167)  \n",
       "14474   POINT (32.433 -99.84999999999999)  \n",
       "15106              POINT (32.65 -114.617)  \n",
       "15107             POINT (60.785 -148.839)  \n",
       "15109  POINT (66.59999999999999 -159.986)  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "geo_stations.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>state_abb</th>\n",
       "      <th>population</th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>New York</td>\n",
       "      <td>New York</td>\n",
       "      <td>NY</td>\n",
       "      <td>8550405</td>\n",
       "      <td>POINT (40.7127837 -74.0059413)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>CA</td>\n",
       "      <td>3971883</td>\n",
       "      <td>POINT (34.0522342 -118.2436849)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chicago</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>IL</td>\n",
       "      <td>2720546</td>\n",
       "      <td>POINT (41.8781136 -87.6297982)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Houston</td>\n",
       "      <td>Texas</td>\n",
       "      <td>TX</td>\n",
       "      <td>2296224</td>\n",
       "      <td>POINT (29.7604267 -95.3698028)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>PA</td>\n",
       "      <td>1567442</td>\n",
       "      <td>POINT (39.9525839 -75.1652215)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           city         state state_abb  population  \\\n",
       "0      New York      New York        NY     8550405   \n",
       "1   Los Angeles    California        CA     3971883   \n",
       "2       Chicago      Illinois        IL     2720546   \n",
       "3       Houston         Texas        TX     2296224   \n",
       "4  Philadelphia  Pennsylvania        PA     1567442   \n",
       "\n",
       "                          location  \n",
       "0   POINT (40.7127837 -74.0059413)  \n",
       "1  POINT (34.0522342 -118.2436849)  \n",
       "2   POINT (41.8781136 -87.6297982)  \n",
       "3   POINT (29.7604267 -95.3698028)  \n",
       "4   POINT (39.9525839 -75.1652215)  "
      ]
     },
     "execution_count": 404,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "geo_cities.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### joining weather stations to cities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# locate the closest weather station (with full weather data) to each city\n",
    "all_stations = geo_stations.location.unary_union\n",
    "for i, row in geo_cities.iterrows():\n",
    "    nearest_station = nearest_points(row['location'], all_stations)[1]\n",
    "    station_ids = geo_stations[geo_stations.location == nearest_station][['USAF', 'WBAN']]\n",
    "    # append USAF and WBAN identifier to cities data\n",
    "    geo_cities.loc[i, 'USAF'] = station_ids.iloc[0, 0]\n",
    "    geo_cities.loc[i, 'WBAN'] = station_ids.iloc[0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "geo_cities['USAF'] = geo_cities.loc[:, 'USAF'].astype(int)\n",
    "geo_cities['WBAN'] = geo_cities.loc[:, 'WBAN'].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write and Read geostation , geocities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('cities.pickle', 'wb') as handle:\n",
    "    pickle.dump(geo_cities, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open('stations.pickle', 'wb') as handle:\n",
    "    pickle.dump(geo_stations, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('cities.pickle', 'rb') as handle:\n",
    "    geo_cities = pickle.load(handle)\n",
    "\n",
    "with open('stations.pickle', 'rb') as handle:\n",
    "    geo_stations = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>state_abb</th>\n",
       "      <th>population</th>\n",
       "      <th>location</th>\n",
       "      <th>USAF</th>\n",
       "      <th>WBAN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>New York</td>\n",
       "      <td>New York</td>\n",
       "      <td>NY</td>\n",
       "      <td>8550405</td>\n",
       "      <td>POINT (40.7127837 -74.0059413)</td>\n",
       "      <td>997271</td>\n",
       "      <td>99999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>California</td>\n",
       "      <td>CA</td>\n",
       "      <td>3971883</td>\n",
       "      <td>POINT (34.0522342 -118.2436849)</td>\n",
       "      <td>722874</td>\n",
       "      <td>93134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chicago</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>IL</td>\n",
       "      <td>2720546</td>\n",
       "      <td>POINT (41.8781136 -87.6297982)</td>\n",
       "      <td>998499</td>\n",
       "      <td>99999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Houston</td>\n",
       "      <td>Texas</td>\n",
       "      <td>TX</td>\n",
       "      <td>2296224</td>\n",
       "      <td>POINT (29.7604267 -95.3698028)</td>\n",
       "      <td>720594</td>\n",
       "      <td>188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>PA</td>\n",
       "      <td>1567442</td>\n",
       "      <td>POINT (39.9525839 -75.1652215)</td>\n",
       "      <td>724080</td>\n",
       "      <td>13739</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           city         state state_abb  population  \\\n",
       "0      New York      New York        NY     8550405   \n",
       "1   Los Angeles    California        CA     3971883   \n",
       "2       Chicago      Illinois        IL     2720546   \n",
       "3       Houston         Texas        TX     2296224   \n",
       "4  Philadelphia  Pennsylvania        PA     1567442   \n",
       "\n",
       "                          location    USAF   WBAN  \n",
       "0   POINT (40.7127837 -74.0059413)  997271  99999  \n",
       "1  POINT (34.0522342 -118.2436849)  722874  93134  \n",
       "2   POINT (41.8781136 -87.6297982)  998499  99999  \n",
       "3   POINT (29.7604267 -95.3698028)  720594    188  \n",
       "4   POINT (39.9525839 -75.1652215)  724080  13739  "
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "geo_cities.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "weather_dict = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pulling down weather time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 :  New York, NY\n",
      "1 :  Los Angeles, CA\n",
      "2 :  Chicago, IL\n",
      "3 :  Houston, TX\n",
      "4 :  Philadelphia, PA\n",
      "5 :  Phoenix, AZ\n",
      "6 :  San Antonio, TX\n",
      "7 :  San Diego, CA\n",
      "8 :  Dallas, TX\n",
      "9 :  San Jose, CA\n",
      "10 :  Austin, TX\n",
      "11 :  Jacksonville, FL\n",
      "12 :  San Francisco, CA\n",
      "13 :  Indianapolis, IN\n",
      "14 :  Columbus, OH\n",
      "15 :  Fort Worth, TX\n",
      "16 :  Charlotte, NC\n",
      "17 :  Seattle, WA\n",
      "18 :  Denver, CO\n",
      "19 :  El Paso, TX\n",
      "20 :  Detroit, MI\n",
      "21 :  Washington, DC\n",
      "22 :  Boston, MA\n",
      "23 :  Memphis, TN\n",
      "24 :  Nashville, TN\n",
      "25 :  Portland, OR\n",
      "26 :  Oklahoma City, OK\n",
      "27 :  Las Vegas, NV\n",
      "28 :  Baltimore, MD\n",
      "29 :  Louisville, KY\n",
      "30 :  Milwaukee, WI\n",
      "31 :  Albuquerque, NM\n",
      "32 :  Tucson, AZ\n",
      "33 :  Fresno, CA\n",
      "34 :  Sacramento, CA\n",
      "35 :  Kansas City, MO\n",
      "36 :  Long Beach, CA\n",
      "37 :  Mesa, AZ\n",
      "38 :  Atlanta, GA\n",
      "39 :  Colorado Springs, CO\n",
      "40 :  Virginia Beach, VA\n",
      "41 :  Raleigh, NC\n",
      "42 :  Omaha, NE\n",
      "43 :  Miami, FL\n",
      "44 :  Oakland, CA\n",
      "45 :  Minneapolis, MN\n",
      "46 :  Tulsa, OK\n",
      "47 :  Wichita, KS\n",
      "48 :  New Orleans, LA\n",
      "49 :  Arlington, TX\n",
      "50 :  Cleveland, OH\n",
      "51 :  Bakersfield, CA\n",
      "52 :  Tampa, FL\n",
      "53 :  Aurora, CO\n",
      "54 :  Urban Honolulu, HI\n",
      "55 :  Anaheim, CA\n",
      "56 :  Santa Ana, CA\n",
      "57 :  Corpus Christi, TX\n",
      "58 :  Riverside, CA\n",
      "59 :  St. Louis, MO\n",
      "60 :  Lexington, KY\n",
      "61 :  Stockton, CA\n",
      "62 :  Pittsburgh, PA\n",
      "63 :  St. Paul, MN\n",
      "64 :  Anchorage, AK\n",
      "65 :  Cincinnati, OH\n",
      "66 :  Henderson, NV\n",
      "67 :  Greensboro, NC\n",
      "68 :  Plano, TX\n",
      "69 :  Newark, NJ\n",
      "70 :  Toledo, OH\n",
      "71 :  Lincoln, NE\n",
      "72 :  Orlando, FL\n",
      "73 :  Chula Vista, CA\n",
      "74 :  Jersey City, NJ\n",
      "75 :  Chandler, AZ\n",
      "76 :  Fort Wayne, IN\n",
      "77 :  Buffalo, NY\n",
      "78 :  Durham, NC\n",
      "79 :  St. Petersburg, FL\n",
      "80 :  Irvine, CA\n",
      "81 :  Laredo, TX\n",
      "82 :  Lubbock, TX\n",
      "83 :  Madison, WI\n",
      "84 :  Gilbert, AZ\n",
      "85 :  Norfolk, VA\n",
      "86 :  Reno, NV\n",
      "87 :  Winston-Salem, NC\n",
      "88 :  Glendale, AZ\n",
      "89 :  Hialeah, FL\n",
      "90 :  Garland, TX\n",
      "91 :  Scottsdale, AZ\n",
      "92 :  Irving, TX\n",
      "93 :  Chesapeake, VA\n",
      "94 :  North Las Vegas, NV\n",
      "95 :  Fremont, CA\n",
      "96 :  Baton Rouge, LA\n",
      "97 :  Richmond, VA\n",
      "98 :  Boise City, ID\n",
      "99 :  San Bernardino, CA\n",
      "100 :  Spokane, WA\n",
      "101 :  Birmingham, AL\n",
      "102 :  Modesto, CA\n",
      "103 :  Des Moines, IA\n",
      "104 :  Rochester, NY\n",
      "105 :  Tacoma, WA\n",
      "106 :  Fontana, CA\n",
      "107 :  Oxnard, CA\n",
      "108 :  Moreno Valley, CA\n",
      "109 :  Fayetteville, NC\n",
      "110 :  Huntington Beach, CA\n",
      "111 :  Yonkers, NY\n",
      "112 :  Glendale, CA\n",
      "113 :  Aurora, IL\n",
      "114 :  Montgomery, AL\n",
      "115 :  Columbus, GA\n",
      "116 :  Amarillo, TX\n",
      "117 :  Little Rock, AR\n",
      "118 :  Akron, OH\n",
      "119 :  Shreveport, LA\n",
      "120 :  Augusta-Richmond County, GA\n",
      "121 :  Grand Rapids, MI\n",
      "122 :  Mobile, AL\n",
      "123 :  Salt Lake City, UT\n",
      "124 :  Huntsville, AL\n",
      "125 :  Tallahassee, FL\n",
      "126 :  Grand Prairie, TX\n",
      "127 :  Overland Park, KS\n",
      "128 :  Knoxville, TN\n",
      "129 :  Worcester, MA\n",
      "130 :  Brownsville, TX\n",
      "131 :  Newport News, VA\n",
      "132 :  Santa Clarita, CA\n",
      "133 :  Port St. Lucie, FL\n",
      "134 :  Providence, RI\n",
      "135 :  Fort Lauderdale, FL\n",
      "136 :  Chattanooga, TN\n",
      "137 :  Tempe, AZ\n",
      "138 :  Oceanside, CA\n",
      "139 :  Garden Grove, CA\n",
      "140 :  Rancho Cucamonga, CA\n",
      "141 :  Cape Coral, FL\n",
      "142 :  Santa Rosa, CA\n",
      "143 :  Vancouver, WA\n",
      "144 :  Sioux Falls, SD\n",
      "145 :  Peoria, AZ\n",
      "146 :  Ontario, CA\n",
      "147 :  Jackson, MS\n",
      "148 :  Elk Grove, CA\n",
      "149 :  Springfield, MO\n",
      "150 :  Pembroke Pines, FL\n",
      "151 :  Salem, OR\n",
      "152 :  Corona, CA\n",
      "153 :  Eugene, OR\n",
      "154 :  McKinney, TX\n",
      "155 :  Fort Collins, CO\n",
      "156 :  Lancaster, CA\n",
      "157 :  Cary, NC\n",
      "158 :  Palmdale, CA\n",
      "159 :  Hayward, CA\n",
      "160 :  Salinas, CA\n",
      "161 :  Frisco, TX\n",
      "162 :  Springfield, MA\n",
      "163 :  Pasadena, TX\n",
      "164 :  Macon, GA\n",
      "165 :  Alexandria, VA\n",
      "166 :  Pomona, CA\n",
      "167 :  Lakewood, CO\n",
      "168 :  Sunnyvale, CA\n",
      "169 :  Escondido, CA\n",
      "170 :  Kansas City, KS\n",
      "171 :  Hollywood, FL\n",
      "172 :  Clarksville, TN\n",
      "173 :  Torrance, CA\n",
      "174 :  Rockford, IL\n",
      "175 :  Joliet, IL\n",
      "176 :  Paterson, NJ\n",
      "177 :  Bridgeport, CT\n",
      "178 :  Naperville, IL\n",
      "179 :  Savannah, GA\n",
      "180 :  Mesquite, TX\n",
      "181 :  Syracuse, NY\n",
      "182 :  Pasadena, CA\n",
      "183 :  Orange, CA\n",
      "184 :  Fullerton, CA\n",
      "185 :  Killeen, TX\n",
      "186 :  Dayton, OH\n",
      "187 :  McAllen, TX\n",
      "188 :  Bellevue, WA\n",
      "189 :  Miramar, FL\n",
      "190 :  Hampton, VA\n",
      "191 :  West Valley City, UT\n",
      "192 :  Warren, MI\n",
      "193 :  Olathe, KS\n",
      "194 :  Columbia, SC\n",
      "195 :  Thornton, CO\n",
      "196 :  Carrollton, TX\n",
      "197 :  Midland, TX\n",
      "198 :  Charleston, SC\n",
      "199 :  Waco, TX\n",
      "200 :  Sterling Heights, MI\n",
      "201 :  Denton, TX\n",
      "202 :  Cedar Rapids, IA\n",
      "203 :  New Haven, CT\n",
      "204 :  Roseville, CA\n",
      "205 :  Gainesville, FL\n",
      "206 :  Visalia, CA\n",
      "207 :  Coral Springs, FL\n",
      "208 :  Thousand Oaks, CA\n",
      "209 :  Elizabeth, NJ\n",
      "210 :  Stamford, CT\n",
      "211 :  Concord, CA\n",
      "212 :  Surprise, AZ\n",
      "213 :  Lafayette, LA\n",
      "214 :  Topeka, KS\n",
      "215 :  Kent, WA\n",
      "216 :  Simi Valley, CA\n",
      "217 :  Santa Clara, CA\n",
      "218 :  Murfreesboro, TN\n",
      "219 :  Hartford, CT\n",
      "220 :  Athens-Clarke County, GA\n",
      "221 :  Victorville, CA\n",
      "222 :  Abilene, TX\n",
      "223 :  Vallejo, CA\n",
      "224 :  Berkeley, CA\n",
      "225 :  Norman, OK\n",
      "226 :  Allentown, PA\n",
      "227 :  Evansville, IN\n",
      "228 :  Columbia, MO\n",
      "229 :  Odessa, TX\n",
      "230 :  Fargo, ND\n",
      "231 :  Beaumont, TX\n",
      "232 :  Independence, MO\n",
      "233 :  Ann Arbor, MI\n",
      "234 :  El Monte, CA\n",
      "235 :  Springfield, IL\n",
      "236 :  Round Rock, TX\n",
      "237 :  Wilmington, NC\n",
      "238 :  Arvada, CO\n",
      "239 :  Provo, UT\n",
      "240 :  Peoria, IL\n",
      "241 :  Lansing, MI\n",
      "242 :  Downey, CA\n",
      "243 :  Carlsbad, CA\n",
      "244 :  Costa Mesa, CA\n",
      "245 :  Miami Gardens, FL\n",
      "246 :  Westminster, CO\n",
      "247 :  Clearwater, FL\n",
      "248 :  Fairfield, CA\n",
      "249 :  Rochester, MN\n",
      "250 :  Elgin, IL\n",
      "251 :  Temecula, CA\n",
      "252 :  West Jordan, UT\n",
      "253 :  Inglewood, CA\n",
      "254 :  Richardson, TX\n",
      "255 :  Lowell, MA\n",
      "256 :  Gresham, OR\n",
      "257 :  Antioch, CA\n",
      "258 :  Cambridge, MA\n",
      "259 :  High Point, NC\n",
      "260 :  Billings, MT\n",
      "261 :  Manchester, NH\n",
      "262 :  Murrieta, CA\n",
      "263 :  Centennial, CO\n",
      "264 :  Richmond, CA\n",
      "265 :  San Buenaventura, CA\n",
      "266 :  Pueblo, CO\n",
      "267 :  Pearland, TX\n",
      "268 :  Waterbury, CT\n",
      "269 :  West Covina, CA\n",
      "270 :  North Charleston, SC\n",
      "271 :  Everett, WA\n",
      "272 :  College Station, TX\n",
      "273 :  Palm Bay, FL\n",
      "274 :  Pompano Beach, FL\n",
      "275 :  Boulder, CO\n",
      "276 :  Norwalk, CA\n",
      "277 :  West Palm Beach, FL\n",
      "278 :  Broken Arrow, OK\n",
      "279 :  Daly City, CA\n",
      "280 :  Sandy Springs, GA\n",
      "281 :  Burbank, CA\n",
      "282 :  Green Bay, WI\n",
      "283 :  Santa Maria, CA\n",
      "284 :  Wichita Falls, TX\n",
      "285 :  Lakeland, FL\n",
      "286 :  Clovis, CA\n",
      "287 :  Lewisville, TX\n",
      "288 :  Tyler, TX\n",
      "289 :  El Cajon, CA\n",
      "290 :  San Mateo, CA\n",
      "291 :  Rialto, CA\n",
      "292 :  Davenport, IA\n",
      "293 :  Hillsboro, OR\n",
      "294 :  Las Cruces, NM\n",
      "295 :  South Bend, IN\n",
      "296 :  Vista, CA\n",
      "297 :  Greeley, CO\n",
      "298 :  Davie, FL\n",
      "299 :  San Angelo, TX\n",
      "300 :  Jurupa Valley, CA\n",
      "301 :  Renton, WA\n",
      "302 :  Roanoke, VA\n",
      "303 :  Kenosha, WI\n",
      "304 :  Erie, PA\n",
      "305 :  Albany, NY\n",
      "306 :  Compton, CA\n",
      "307 :  Tuscaloosa, AL\n",
      "308 :  League City, TX\n",
      "309 :  Flint, MI\n",
      "310 :  Allen, TX\n",
      "311 :  Mission Viejo, CA\n",
      "312 :  Vacaville, CA\n",
      "313 :  Lawton, OK\n",
      "314 :  Beaverton, OR\n",
      "315 :  South Gate, CA\n",
      "316 :  Portsmouth, VA\n",
      "317 :  Sparks, NV\n",
      "318 :  Brockton, MA\n",
      "319 :  Dearborn, MI\n",
      "320 :  Federal Way, WA\n",
      "321 :  Lee's Summit, MO\n",
      "322 :  New Bedford, MA\n",
      "323 :  Spokane Valley, WA\n",
      "324 :  Livonia, MI\n",
      "325 :  Roswell, GA\n",
      "326 :  Orem, UT\n",
      "327 :  Rio Rancho, NM\n",
      "328 :  Yuma, AZ\n",
      "329 :  Lawrence, KS\n",
      "330 :  Yakima, WA\n",
      "331 :  Quincy, MA\n",
      "332 :  Sandy, UT\n",
      "333 :  Hesperia, CA\n",
      "334 :  Carson, CA\n",
      "335 :  Boca Raton, FL\n",
      "336 :  Santa Monica, CA\n",
      "337 :  San Marcos, CA\n",
      "338 :  Sunrise, FL\n",
      "339 :  Plantation, FL\n",
      "340 :  Lynn, MA\n",
      "341 :  Miami Beach, FL\n",
      "342 :  Westminster, CA\n",
      "343 :  Longmont, CO\n",
      "344 :  Santa Barbara, CA\n",
      "345 :  Redding, CA\n",
      "346 :  Meridian, ID\n",
      "347 :  San Leandro, CA\n",
      "348 :  Greenville, NC\n",
      "349 :  Chico, CA\n",
      "350 :  Edmond, OK\n",
      "351 :  Nampa, ID\n",
      "352 :  Newton, MA\n",
      "353 :  Fall River, MA\n",
      "354 :  Carmel, IN\n",
      "355 :  Fishers, IN\n",
      "356 :  Asheville, NC\n",
      "357 :  Norwalk, CT\n",
      "358 :  Waukegan, IL\n",
      "359 :  Deltona, FL\n",
      "360 :  Hawthorne, CA\n",
      "361 :  Fort Smith, AR\n",
      "362 :  Suffolk, VA\n",
      "363 :  Sugar Land, TX\n",
      "364 :  Livermore, CA\n",
      "365 :  Nashua, NH\n",
      "366 :  Reading, PA\n",
      "367 :  Concord, NC\n",
      "368 :  Indio, CA\n",
      "369 :  Whittier, CA\n",
      "370 :  Kirkland, WA\n",
      "371 :  Menifee, CA\n",
      "372 :  Newport Beach, CA\n",
      "373 :  Tracy, CA\n",
      "374 :  Citrus Heights, CA\n",
      "375 :  Bend, OR\n",
      "376 :  Bloomington, MN\n",
      "377 :  Clifton, NJ\n",
      "378 :  Duluth, MN\n",
      "379 :  Champaign, IL\n",
      "380 :  Chino, CA\n",
      "381 :  Alhambra, CA\n",
      "382 :  Ogden, UT\n",
      "383 :  Redwood City, CA\n",
      "384 :  Bellingham, WA\n",
      "385 :  O'Fallon, MO\n",
      "386 :  Hoover, AL\n",
      "387 :  Danbury, CT\n",
      "388 :  Edinburg, TX\n",
      "389 :  Trenton, NJ\n",
      "390 :  Santa Fe, NM\n",
      "391 :  Bloomington, IN\n",
      "392 :  Cicero, IL\n",
      "393 :  Hemet, CA\n",
      "394 :  Johns Creek, GA\n",
      "395 :  Mission, TX\n",
      "396 :  Troy, MI\n",
      "397 :  Buena Park, CA\n",
      "398 :  Palm Coast, FL\n",
      "399 :  Fayetteville, AR\n",
      "400 :  Sioux City, IA\n",
      "401 :  Lake Forest, CA\n",
      "402 :  Merced, CA\n",
      "403 :  Longview, TX\n",
      "404 :  Bryan, TX\n",
      "405 :  Westland, MI\n",
      "406 :  Warwick, RI\n",
      "407 :  Lakewood, CA\n",
      "408 :  Farmington Hills, MI\n",
      "409 :  Mount Pleasant, SC\n",
      "410 :  Cranston, RI\n",
      "411 :  Largo, FL\n",
      "412 :  Avondale, AZ\n",
      "413 :  Tustin, CA\n",
      "414 :  Mountain View, CA\n",
      "415 :  Napa, CA\n",
      "416 :  Somerville, MA\n",
      "417 :  Lawrence, MA\n",
      "418 :  St. George, UT\n",
      "419 :  Melbourne, FL\n",
      "420 :  Parma, OH\n",
      "421 :  New Rochelle, NY\n",
      "422 :  Lynchburg, VA\n",
      "423 :  Medford, OR\n",
      "424 :  Deerfield Beach, FL\n",
      "425 :  Pleasanton, CA\n",
      "426 :  Brooklyn Park, MN\n",
      "427 :  Goodyear, AZ\n",
      "428 :  Kennewick, WA\n",
      "429 :  Alameda, CA\n",
      "430 :  Bellflower, CA\n",
      "431 :  Chino Hills, CA\n",
      "432 :  Bloomington, IL\n",
      "433 :  Springdale, AR\n",
      "434 :  Racine, WI\n",
      "435 :  Hammond, IN\n",
      "436 :  Milpitas, CA\n",
      "437 :  Gary, IN\n",
      "438 :  Scranton, PA\n",
      "439 :  Baldwin Park, CA\n",
      "440 :  Auburn, WA\n",
      "441 :  St. Joseph, MO\n",
      "442 :  Pharr, TX\n",
      "443 :  Upland, CA\n",
      "444 :  Folsom, CA\n",
      "445 :  Baytown, TX\n",
      "446 :  San Ramon, CA\n",
      "447 :  Camden, NJ\n",
      "448 :  Lake Charles, LA\n",
      "449 :  Kalamazoo, MI\n",
      "450 :  Arlington Heights, IL\n",
      "451 :  Plymouth, MN\n",
      "452 :  Evanston, IL\n",
      "453 :  Manteca, CA\n",
      "454 :  Wyoming, MI\n",
      "455 :  Loveland, CO\n",
      "456 :  Perris, CA\n",
      "457 :  Bethlehem, PA\n",
      "458 :  Albany, GA\n",
      "459 :  Schaumburg, IL\n",
      "460 :  Gastonia, NC\n",
      "461 :  Union City, CA\n",
      "462 :  Bolingbrook, IL\n",
      "463 :  Iowa City, IA\n",
      "464 :  Layton, UT\n",
      "465 :  Appleton, WI\n",
      "466 :  Missouri City, TX\n",
      "467 :  Fort Myers, FL\n",
      "468 :  Boynton Beach, FL\n",
      "469 :  Jonesboro, AR\n",
      "470 :  Rapid City, SD\n",
      "471 :  Warner Robins, GA\n",
      "472 :  Rochester Hills, MI\n",
      "473 :  Decatur, IL\n",
      "474 :  Southfield, MI\n",
      "475 :  New Britain, CT\n",
      "476 :  Franklin, TN\n",
      "477 :  Turlock, CA\n",
      "478 :  Temple, TX\n",
      "479 :  Apple Valley, CA\n",
      "480 :  Lynwood, CA\n",
      "481 :  Waukesha, WI\n",
      "482 :  Wilmington, DE\n",
      "483 :  Canton, OH\n",
      "484 :  Gulfport, MS\n",
      "485 :  Pawtucket, RI\n",
      "486 :  Lauderhill, FL\n",
      "487 :  Rock Hill, SC\n",
      "488 :  Flower Mound, TX\n",
      "489 :  Bismarck, ND\n",
      "490 :  Lafayette, IN\n",
      "491 :  Passaic, NJ\n",
      "492 :  Redlands, CA\n",
      "493 :  Missoula, MT\n",
      "494 :  Rancho Cordova, CA\n",
      "495 :  New Braunfels, TX\n",
      "496 :  Flagstaff, AZ\n",
      "497 :  Muncie, IN\n",
      "498 :  Weston, FL\n",
      "499 :  Frederick, MD\n",
      "500 :  Pasco, WA\n",
      "501 :  Pittsburg, CA\n",
      "502 :  Palatine, IL\n",
      "503 :  North Richland Hills, TX\n",
      "504 :  Union City, NJ\n",
      "505 :  Kissimmee, FL\n",
      "506 :  Walnut Creek, CA\n",
      "507 :  St. Charles, MO\n",
      "508 :  Mount Vernon, NY\n",
      "509 :  Conroe, TX\n",
      "510 :  Dothan, AL\n",
      "511 :  Waterloo, IA\n",
      "512 :  Maple Grove, MN\n",
      "513 :  Redondo Beach, CA\n",
      "514 :  Bossier City, LA\n",
      "515 :  Yorba Linda, CA\n",
      "516 :  Woodbury, MN\n",
      "517 :  Eau Claire, WI\n",
      "518 :  Davis, CA\n",
      "519 :  Camarillo, CA\n",
      "520 :  Victoria, TX\n",
      "521 :  Gaithersburg, MD\n",
      "522 :  Jacksonville, NC\n",
      "523 :  South San Francisco, CA\n",
      "524 :  St. Cloud, MN\n",
      "525 :  Kenner, LA\n",
      "526 :  Rockville, MD\n",
      "527 :  Jackson, TN\n",
      "528 :  Yuba City, CA\n",
      "529 :  Portland, ME\n",
      "530 :  Palo Alto, CA\n",
      "531 :  Marysville, WA\n",
      "532 :  South Jordan, UT\n",
      "533 :  Oshkosh, WI\n",
      "534 :  North Little Rock, AR\n",
      "535 :  Homestead, FL\n",
      "536 :  Bayonne, NJ\n",
      "537 :  Eagan, MN\n",
      "538 :  Delray Beach, FL\n",
      "539 :  Johnson City, TN\n",
      "540 :  Cedar Park, TX\n",
      "541 :  Laguna Niguel, CA\n",
      "542 :  Harlingen, TX\n",
      "543 :  San Clemente, CA\n",
      "544 :  Schenectady, NY\n",
      "545 :  Broomfield, CO\n",
      "546 :  Ames, IA\n",
      "547 :  Shawnee, KS\n",
      "548 :  Conway, AR\n",
      "549 :  East Orange, NJ\n",
      "550 :  Skokie, IL\n",
      "551 :  Daytona Beach, FL\n",
      "552 :  Tamarac, FL\n",
      "553 :  Youngstown, OH\n",
      "554 :  Lodi, CA\n",
      "555 :  Greenville, SC\n",
      "556 :  Mansfield, TX\n",
      "557 :  Santa Cruz, CA\n",
      "558 :  Pico Rivera, CA\n",
      "559 :  Madera, CA\n",
      "560 :  Janesville, WI\n",
      "561 :  West Des Moines, IA\n",
      "562 :  Montebello, CA\n",
      "563 :  Georgetown, TX\n",
      "564 :  Alpharetta, GA\n",
      "565 :  Lorain, OH\n",
      "566 :  Bowling Green, KY\n",
      "567 :  Eden Prairie, MN\n",
      "568 :  Waltham, MA\n",
      "569 :  Cheyenne, WY\n",
      "570 :  Rogers, AR\n",
      "571 :  Encinitas, CA\n",
      "572 :  Haverhill, MA\n",
      "573 :  Jupiter, FL\n",
      "574 :  Council Bluffs, IA\n",
      "575 :  Wellington, FL\n",
      "576 :  North Miami, FL\n",
      "577 :  Hamilton, OH\n",
      "578 :  North Port, FL\n",
      "579 :  Tulare, CA\n",
      "580 :  Coon Rapids, MN\n",
      "581 :  Buckeye, AZ\n",
      "582 :  La Habra, CA\n",
      "583 :  Blaine, MN\n",
      "584 :  Auburn, AL\n",
      "585 :  Lake Elsinore, CA\n",
      "586 :  Taylor, MI\n",
      "587 :  Burnsville, MN\n",
      "588 :  Monterey Park, CA\n",
      "589 :  Rocklin, CA\n",
      "590 :  Utica, NY\n",
      "591 :  Malden, MA\n",
      "592 :  National City, CA\n",
      "593 :  Springfield, OR\n",
      "594 :  Terre Haute, IN\n",
      "595 :  Vineland, NJ\n",
      "596 :  San Marcos, TX\n",
      "597 :  Lakeville, MN\n",
      "598 :  West Allis, WI\n",
      "599 :  Redmond, WA\n",
      "600 :  Cupertino, CA\n",
      "601 :  Taylorsville, UT\n",
      "602 :  Bristol, CT\n",
      "603 :  Moore, OK\n",
      "604 :  Gardena, CA\n",
      "605 :  Petaluma, CA\n",
      "606 :  Grand Junction, CO\n",
      "607 :  Casper, WY\n",
      "608 :  Rowlett, TX\n",
      "609 :  La Mesa, CA\n",
      "610 :  Meriden, CT\n",
      "611 :  Pontiac, MI\n",
      "612 :  St. Clair Shores, MI\n",
      "613 :  Port Orange, FL\n",
      "614 :  Lakewood, WA\n",
      "615 :  Springfield, OH\n",
      "616 :  Great Falls, MT\n",
      "617 :  Chapel Hill, NC\n",
      "618 :  Huntington Park, CA\n",
      "619 :  Lancaster, PA\n",
      "620 :  Coconut Creek, FL\n",
      "621 :  Idaho Falls, ID\n",
      "622 :  San Rafael, CA\n",
      "623 :  Noblesville, IN\n",
      "624 :  Marietta, GA\n",
      "625 :  Owensboro, KY\n",
      "626 :  Eastvale, CA\n",
      "627 :  Royal Oak, MI\n",
      "628 :  Brentwood, CA\n",
      "629 :  Dubuque, IA\n",
      "630 :  Novi, MI\n",
      "631 :  Des Plaines, IL\n",
      "632 :  Orland Park, IL\n",
      "633 :  Bartlett, TN\n",
      "634 :  Woodland, CA\n",
      "635 :  Lehi, UT\n",
      "636 :  White Plains, NY\n",
      "637 :  Arcadia, CA\n",
      "638 :  Ocala, FL\n",
      "639 :  Sanford, FL\n",
      "640 :  Bowie, MD\n",
      "641 :  Kokomo, IN\n",
      "642 :  Santee, CA\n",
      "643 :  Dublin, CA\n",
      "644 :  Medford, MA\n",
      "645 :  Midwest City, OK\n",
      "646 :  Margate, FL\n",
      "647 :  Tinley Park, IL\n",
      "648 :  Pflugerville, TX\n",
      "649 :  New Brunswick, NJ\n",
      "650 :  Grand Forks, ND\n",
      "651 :  Fountain Valley, CA\n",
      "652 :  St. Peters, MO\n",
      "653 :  Diamond Bar, CA\n",
      "654 :  Taunton, MA\n",
      "655 :  Oak Lawn, IL\n",
      "656 :  Ankeny, IA\n",
      "657 :  Chicopee, MA\n",
      "658 :  Berwyn, IL\n",
      "659 :  Manhattan, KS\n",
      "660 :  Smyrna, GA\n",
      "661 :  Dearborn Heights, MI\n",
      "662 :  Porterville, CA\n",
      "663 :  Doral, FL\n",
      "664 :  Hendersonville, TN\n",
      "665 :  Weymouth Town, MA\n",
      "666 :  Rocky Mount, NC\n",
      "667 :  Corvallis, OR\n",
      "668 :  Valdosta, GA\n",
      "669 :  Hanford, CA\n",
      "670 :  Castle Rock, CO\n",
      "671 :  Greenwood, IN\n",
      "672 :  Hempstead, NY\n",
      "673 :  Novato, CA\n",
      "674 :  Kettering, OH\n",
      "675 :  Bellevue, NE\n",
      "676 :  Shoreline, WA\n",
      "677 :  Decatur, AL\n",
      "678 :  Paramount, CA\n",
      "679 :  Port Arthur, TX\n",
      "680 :  Anderson, IN\n",
      "681 :  Sarasota, FL\n",
      "682 :  West Haven, CT\n",
      "683 :  Rosemead, CA\n",
      "684 :  Highland, CA\n",
      "685 :  Mount Prospect, IL\n",
      "686 :  Colton, CA\n",
      "687 :  Carson City, NV\n",
      "688 :  Pocatello, ID\n",
      "689 :  Bradenton, FL\n",
      "690 :  Normal, IL\n",
      "691 :  Richland, WA\n",
      "692 :  Euless, TX\n",
      "693 :  Blue Springs, MO\n",
      "694 :  Cathedral City, CA\n",
      "695 :  Elyria, OH\n",
      "696 :  Wheaton, IL\n",
      "697 :  Commerce City, CO\n",
      "698 :  Hoboken, NJ\n",
      "699 :  Watsonville, CA\n",
      "700 :  Lake Havasu City, AZ\n",
      "701 :  Revere, MA\n",
      "702 :  West New York, NJ\n",
      "703 :  Yucaipa, CA\n",
      "704 :  Gilroy, CA\n",
      "705 :  Pensacola, FL\n",
      "706 :  Kingsport, TN\n",
      "707 :  Palm Beach Gardens, FL\n",
      "708 :  Delano, CA\n",
      "709 :  West Sacramento, CA\n",
      "710 :  Huntersville, NC\n",
      "711 :  Perth Amboy, NJ\n",
      "712 :  Southaven, MS\n",
      "713 :  Harrisonburg, VA\n",
      "714 :  Peabody, MA\n",
      "715 :  Placentia, CA\n",
      "716 :  Lenexa, KS\n",
      "717 :  DeSoto, TX\n",
      "718 :  Burlington, NC\n",
      "719 :  Elkhart, IN\n",
      "720 :  La Crosse, WI\n",
      "721 :  Oak Park, IL\n",
      "722 :  Florissant, MO\n",
      "723 :  Sammamish, WA\n",
      "724 :  Albany, OR\n",
      "725 :  Hoffman Estates, IL\n",
      "726 :  Milford, CT\n",
      "727 :  Glendora, CA\n",
      "728 :  Brookhaven, GA\n",
      "729 :  Palm Desert, CA\n",
      "730 :  Joplin, MO\n",
      "731 :  Enid, OK\n",
      "732 :  Bonita Springs, FL\n",
      "733 :  Caldwell, ID\n",
      "734 :  Minnetonka, MN\n",
      "735 :  Pinellas Park, FL\n",
      "736 :  Battle Creek, MI\n",
      "737 :  Casa Grande, AZ\n",
      "738 :  Grand Island, NE\n",
      "739 :  Grapevine, TX\n",
      "740 :  Kentwood, MI\n",
      "741 :  Tigard, OR\n",
      "742 :  Apple Valley, MN\n",
      "743 :  Plainfield, NJ\n",
      "744 :  Leesburg, VA\n",
      "745 :  Coral Gables, FL\n",
      "746 :  Lakewood, OH\n",
      "747 :  Burien, WA\n",
      "748 :  Logan, UT\n",
      "749 :  Olympia, WA\n",
      "750 :  Aliso Viejo, CA\n",
      "751 :  Galveston, TX\n",
      "752 :  Poway, CA\n",
      "753 :  Edina, MN\n"
     ]
    }
   ],
   "source": [
    "# \n",
    "for i, row in geo_cities.iterrows():\n",
    "    place_string = '{}, {}'.format(row['city'], row['state_abb'])\n",
    "    print i, ': ', place_string\n",
    "    usaf = str(row['USAF'])\n",
    "    wban = str(row['WBAN'])\n",
    "    # some WBAN ID's are less than 5 chars; append with leading zeroes\n",
    "    wban_gap = 5 - len(wban)\n",
    "    wban = (wban_gap * '0') + wban\n",
    "    \n",
    "    all_weather = pd.DataFrame(columns=['year', 'month', 'day', 'hour', 'temp'])\n",
    "    for year in ['2015', '2016', '2017']:\n",
    "        # build a URL for each station and year\n",
    "        query_string = 'https://www1.ncdc.noaa.gov/pub/data/noaa/isd-lite/{}/{}-{}-{}.gz'\\\n",
    "            .format(year, usaf, wban, year)\n",
    "        # hit the NOAA API for weather data\n",
    "        weather_series = pd.read_csv(query_string, compression='gzip',\\\n",
    "                                     delim_whitespace=True, header=None,\\\n",
    "                                    names=['year', 'month', 'day', 'hour', 'temp',\\\n",
    "                                        'x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7'])\n",
    "        weather_series.drop(['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7'], axis=1, inplace=True)\n",
    "        # add year's worth of data to all_weather, and return\n",
    "        all_weather = all_weather.append(weather_series, ignore_index=True)\n",
    "        weather_dict[place_string] = all_weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('weather_series.pickle', 'wb') as handle:\n",
    "    pickle.dump(weather_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>temp</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20469</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>217.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>10077.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>51.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20470</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>10088.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20471</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10092.0</td>\n",
       "      <td>270.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20472</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10095.0</td>\n",
       "      <td>270.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20473</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>194.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>10096.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "      <td>-9999.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         year  month  day  hour   temp    x1       x2     x3    x4      x5  \\\n",
       "20469  2017.0    5.0  3.0   1.0  217.0  17.0  10077.0  280.0  51.0 -9999.0   \n",
       "20470  2017.0    5.0  3.0   2.0  211.0  22.0  10088.0  280.0  46.0 -9999.0   \n",
       "20471  2017.0    5.0  3.0   3.0  206.0   6.0  10092.0  270.0  57.0 -9999.0   \n",
       "20472  2017.0    5.0  3.0   4.0  200.0   6.0  10095.0  270.0  77.0 -9999.0   \n",
       "20473  2017.0    5.0  3.0   5.0  194.0  11.0  10096.0  280.0  36.0 -9999.0   \n",
       "\n",
       "           x6      x7  \n",
       "20469 -9999.0 -9999.0  \n",
       "20470 -9999.0 -9999.0  \n",
       "20471 -9999.0 -9999.0  \n",
       "20472 -9999.0 -9999.0  \n",
       "20473 -9999.0 -9999.0  "
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weather_dict['Philadelphia'].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}