ethnicolr/examples/ethnicolr_app_contrib20xx-fl_reg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Application: 2000/2010 Political Campaign Contributions by Race\n",
    "\n",
    "Using ethnicolr, we look to answer three basic questions:\n",
    "<ol>\n",
    "<li>What proportion of contributions were made by blacks, whites, Hispanics, and Asians? \n",
    "<li>What proportion of unique contributors were blacks, whites, Hispanics, and Asians?\n",
    "<li>What proportion of total donations were given by blacks, whites, Hispanics, and Asians?\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'cycle', u'transaction_id', u'transaction_type', u'amount', u'date',\n",
       "       u'bonica_cid', u'contributor_name', u'contributor_lname',\n",
       "       u'contributor_fname', u'contributor_mname', u'contributor_suffix',\n",
       "       u'contributor_title', u'contributor_ffname', u'contributor_type',\n",
       "       u'contributor_gender', u'contributor_address', u'contributor_city',\n",
       "       u'contributor_state', u'contributor_zipcode', u'contributor_occupation',\n",
       "       u'contributor_employer', u'contributor_category',\n",
       "       u'contributor_category_order', u'is_corp', u'organization_name',\n",
       "       u'parent_organization_name', u'recipient_name', u'bonica_rid',\n",
       "       u'recipient_party', u'recipient_type', u'recipient_state',\n",
       "       u'recipient_category', u'recipient_category_order',\n",
       "       u'recipient_district', u'seat', u'election_type',\n",
       "       u'contributor_cfscore', u'candidate_cfscore', u'latitude', u'longitude',\n",
       "       u'gis_confidence', u'contributor_district_90s',\n",
       "       u'contributor_district_00s', u'contributor_district_10s',\n",
       "       u'lname_frequency', u'efec_memo', u'efec_memo2',\n",
       "       u'efec_transaction_id_orig', u'efec_org_orig', u'efec_comid_orig',\n",
       "       u'efec_form_type'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('/opt/names/fec_contrib/contribDB_2000.csv', nrows=100)\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/venv/local/lib/python2.7/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from ethnicolr import pred_fl_reg_name, pred_fl_reg_ln"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load and Subset on Individual Contributors**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/venv/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (8,7) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>amount</th>\n",
       "      <th>contributor_name</th>\n",
       "      <th>contributor_lname</th>\n",
       "      <th>contributor_fname</th>\n",
       "      <th>contributor_type</th>\n",
       "      <th>race</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>102912</th>\n",
       "      <td>180.0</td>\n",
       "      <td>JOHNSON, KENYIE PAUL</td>\n",
       "      <td>johnson</td>\n",
       "      <td>kenyie</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_black</td>\n",
       "      <td>0.000481</td>\n",
       "      <td>0.002378</td>\n",
       "      <td>0.894507</td>\n",
       "      <td>0.102633</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103990</th>\n",
       "      <td>743.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001272</td>\n",
       "      <td>0.005285</td>\n",
       "      <td>0.001623</td>\n",
       "      <td>0.991820</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105298</th>\n",
       "      <td>180.0</td>\n",
       "      <td>MCCOY, TIMOTHY D</td>\n",
       "      <td>mccoy</td>\n",
       "      <td>timothy</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001921</td>\n",
       "      <td>0.005301</td>\n",
       "      <td>0.147964</td>\n",
       "      <td>0.844815</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105344</th>\n",
       "      <td>188.0</td>\n",
       "      <td>WILLIAMS, VICTOR K</td>\n",
       "      <td>williams</td>\n",
       "      <td>victor</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.004435</td>\n",
       "      <td>0.026017</td>\n",
       "      <td>0.449958</td>\n",
       "      <td>0.519590</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105577</th>\n",
       "      <td>211.0</td>\n",
       "      <td>ELDER, CHESTER H</td>\n",
       "      <td>elder</td>\n",
       "      <td>chester</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.003519</td>\n",
       "      <td>0.011062</td>\n",
       "      <td>0.074804</td>\n",
       "      <td>0.910614</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105659</th>\n",
       "      <td>13000.0</td>\n",
       "      <td>MACARTHUR, GREG</td>\n",
       "      <td>macarthur</td>\n",
       "      <td>greg</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.006452</td>\n",
       "      <td>0.017204</td>\n",
       "      <td>0.054990</td>\n",
       "      <td>0.921354</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105665</th>\n",
       "      <td>13972.0</td>\n",
       "      <td>ABELE, CHRIS</td>\n",
       "      <td>abele</td>\n",
       "      <td>chris</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.004977</td>\n",
       "      <td>0.019195</td>\n",
       "      <td>0.092955</td>\n",
       "      <td>0.882874</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105829</th>\n",
       "      <td>15000.0</td>\n",
       "      <td>PRICE, SOL</td>\n",
       "      <td>price</td>\n",
       "      <td>sol</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.034847</td>\n",
       "      <td>0.024663</td>\n",
       "      <td>0.114083</td>\n",
       "      <td>0.826407</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106029</th>\n",
       "      <td>13600.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001272</td>\n",
       "      <td>0.005285</td>\n",
       "      <td>0.001623</td>\n",
       "      <td>0.991820</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106150</th>\n",
       "      <td>22146.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001272</td>\n",
       "      <td>0.005285</td>\n",
       "      <td>0.001623</td>\n",
       "      <td>0.991820</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106690</th>\n",
       "      <td>813.0</td>\n",
       "      <td>SHAPP, RICHARD A</td>\n",
       "      <td>shapp</td>\n",
       "      <td>richard</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.002375</td>\n",
       "      <td>0.006348</td>\n",
       "      <td>0.005313</td>\n",
       "      <td>0.985964</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107082</th>\n",
       "      <td>949.0</td>\n",
       "      <td>WILLIAMS, VICTOR K</td>\n",
       "      <td>williams</td>\n",
       "      <td>victor</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.004435</td>\n",
       "      <td>0.026017</td>\n",
       "      <td>0.449958</td>\n",
       "      <td>0.519590</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109127</th>\n",
       "      <td>1717.0</td>\n",
       "      <td>HELFRICK, EDWARD W</td>\n",
       "      <td>helfrick</td>\n",
       "      <td>edward</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001051</td>\n",
       "      <td>0.006796</td>\n",
       "      <td>0.014138</td>\n",
       "      <td>0.978015</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109266</th>\n",
       "      <td>8.0</td>\n",
       "      <td>MCCOY, TIMOTHY D</td>\n",
       "      <td>mccoy</td>\n",
       "      <td>timothy</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001921</td>\n",
       "      <td>0.005301</td>\n",
       "      <td>0.147964</td>\n",
       "      <td>0.844815</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109541</th>\n",
       "      <td>24150.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001272</td>\n",
       "      <td>0.005285</td>\n",
       "      <td>0.001623</td>\n",
       "      <td>0.991820</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109586</th>\n",
       "      <td>265.0</td>\n",
       "      <td>WILLIAMS, VICTOR K</td>\n",
       "      <td>williams</td>\n",
       "      <td>victor</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.004435</td>\n",
       "      <td>0.026017</td>\n",
       "      <td>0.449958</td>\n",
       "      <td>0.519590</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109603</th>\n",
       "      <td>25750.0</td>\n",
       "      <td>SUTHERLAND, DWIGHT D JR</td>\n",
       "      <td>sutherland</td>\n",
       "      <td>dwight</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_black</td>\n",
       "      <td>0.006460</td>\n",
       "      <td>0.014527</td>\n",
       "      <td>0.587264</td>\n",
       "      <td>0.391748</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109664</th>\n",
       "      <td>45795.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001272</td>\n",
       "      <td>0.005285</td>\n",
       "      <td>0.001623</td>\n",
       "      <td>0.991820</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109730</th>\n",
       "      <td>28884.0</td>\n",
       "      <td>KIRSCH, STEVEN T</td>\n",
       "      <td>kirsch</td>\n",
       "      <td>steven</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.001272</td>\n",
       "      <td>0.005285</td>\n",
       "      <td>0.001623</td>\n",
       "      <td>0.991820</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109731</th>\n",
       "      <td>30000.0</td>\n",
       "      <td>WOODS, FRED R</td>\n",
       "      <td>woods</td>\n",
       "      <td>fred</td>\n",
       "      <td>I</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>0.000525</td>\n",
       "      <td>0.004697</td>\n",
       "      <td>0.318002</td>\n",
       "      <td>0.676776</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         amount         contributor_name contributor_lname contributor_fname  \\\n",
       "102912    180.0     JOHNSON, KENYIE PAUL           johnson            kenyie   \n",
       "103990    743.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "105298    180.0         MCCOY, TIMOTHY D             mccoy           timothy   \n",
       "105344    188.0       WILLIAMS, VICTOR K          williams            victor   \n",
       "105577    211.0         ELDER, CHESTER H             elder           chester   \n",
       "105659  13000.0          MACARTHUR, GREG         macarthur              greg   \n",
       "105665  13972.0             ABELE, CHRIS             abele             chris   \n",
       "105829  15000.0               PRICE, SOL             price               sol   \n",
       "106029  13600.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "106150  22146.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "106690    813.0         SHAPP, RICHARD A             shapp           richard   \n",
       "107082    949.0       WILLIAMS, VICTOR K          williams            victor   \n",
       "109127   1717.0       HELFRICK, EDWARD W          helfrick            edward   \n",
       "109266      8.0         MCCOY, TIMOTHY D             mccoy           timothy   \n",
       "109541  24150.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "109586    265.0       WILLIAMS, VICTOR K          williams            victor   \n",
       "109603  25750.0  SUTHERLAND, DWIGHT D JR        sutherland            dwight   \n",
       "109664  45795.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "109730  28884.0         KIRSCH, STEVEN T            kirsch            steven   \n",
       "109731  30000.0            WOODS, FRED R             woods              fred   \n",
       "\n",
       "       contributor_type      race     asian  hispanic  nh_black  nh_white  \\\n",
       "102912                I  nh_black  0.000481  0.002378  0.894507  0.102633   \n",
       "103990                I  nh_white  0.001272  0.005285  0.001623  0.991820   \n",
       "105298                I  nh_white  0.001921  0.005301  0.147964  0.844815   \n",
       "105344                I  nh_white  0.004435  0.026017  0.449958  0.519590   \n",
       "105577                I  nh_white  0.003519  0.011062  0.074804  0.910614   \n",
       "105659                I  nh_white  0.006452  0.017204  0.054990  0.921354   \n",
       "105665                I  nh_white  0.004977  0.019195  0.092955  0.882874   \n",
       "105829                I  nh_white  0.034847  0.024663  0.114083  0.826407   \n",
       "106029                I  nh_white  0.001272  0.005285  0.001623  0.991820   \n",
       "106150                I  nh_white  0.001272  0.005285  0.001623  0.991820   \n",
       "106690                I  nh_white  0.002375  0.006348  0.005313  0.985964   \n",
       "107082                I  nh_white  0.004435  0.026017  0.449958  0.519590   \n",
       "109127                I  nh_white  0.001051  0.006796  0.014138  0.978015   \n",
       "109266                I  nh_white  0.001921  0.005301  0.147964  0.844815   \n",
       "109541                I  nh_white  0.001272  0.005285  0.001623  0.991820   \n",
       "109586                I  nh_white  0.004435  0.026017  0.449958  0.519590   \n",
       "109603                I  nh_black  0.006460  0.014527  0.587264  0.391748   \n",
       "109664                I  nh_white  0.001272  0.005285  0.001623  0.991820   \n",
       "109730                I  nh_white  0.001272  0.005285  0.001623  0.991820   \n",
       "109731                I  nh_white  0.000525  0.004697  0.318002  0.676776   \n",
       "\n",
       "        year  \n",
       "102912  2000  \n",
       "103990  2000  \n",
       "105298  2000  \n",
       "105344  2000  \n",
       "105577  2000  \n",
       "105659  2000  \n",
       "105665  2000  \n",
       "105829  2000  \n",
       "106029  2000  \n",
       "106150  2000  \n",
       "106690  2000  \n",
       "107082  2000  \n",
       "109127  2000  \n",
       "109266  2000  \n",
       "109541  2000  \n",
       "109586  2000  \n",
       "109603  2000  \n",
       "109664  2000  \n",
       "109730  2000  \n",
       "109731  2000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('/opt/names/fec_contrib/contribDB_2000.csv', usecols=['amount', 'contributor_type', 'contributor_lname', 'contributor_fname', 'contributor_name'])\n",
    "sdf = df[df.contributor_type=='I'].copy()\n",
    "sdf.fillna('', inplace=True)\n",
    "rdf2000 = pred_fl_reg_name(sdf, 'contributor_lname', 'contributor_fname')\n",
    "rdf2000['year'] = 2000\n",
    "\n",
    "df = pd.read_csv('/opt/names/fec_contrib/contribDB_2010.csv.zip', usecols=['amount', 'contributor_type', 'contributor_lname', 'contributor_fname', 'contributor_name'])\n",
    "sdf = df[df.contributor_type=='I'].copy()\n",
    "sdf.fillna('', inplace=True)\n",
    "rdf2010 = pred_fl_reg_name(sdf, 'contributor_lname', 'contributor_fname')\n",
    "rdf2010['year'] = 2010\n",
    "\n",
    "rdf = pd.concat([rdf2000, rdf2010])\n",
    "rdf.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  What proportion of contributons were by blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" colspan=4>contributor_lname</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level1\" >race</th> \n",
       "        <th class=\"col_heading level1 col0\" >asian</th> \n",
       "        <th class=\"col_heading level1 col1\" >hispanic</th> \n",
       "        <th class=\"col_heading level1 col2\" >nh_black</th> \n",
       "        <th class=\"col_heading level1 col3\" >nh_white</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >1.25%</td> \n",
       "        <td id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >1.89%</td> \n",
       "        <td id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >3.00%</td> \n",
       "        <td id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >93.86%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >1.61%</td> \n",
       "        <td id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >2.77%</td> \n",
       "        <td id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >2.93%</td> \n",
       "        <td id=\"T_6607befa_4c7a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >92.68%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb2c116df90>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adf = rdf.groupby(['year', 'race']).agg({'contributor_lname': 'count'})\n",
    "adf.unstack().apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What proportion of the donors were blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" colspan=4>contributor_name</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level1\" >race</th> \n",
       "        <th class=\"col_heading level1 col0\" >asian</th> \n",
       "        <th class=\"col_heading level1 col1\" >hispanic</th> \n",
       "        <th class=\"col_heading level1 col2\" >nh_black</th> \n",
       "        <th class=\"col_heading level1 col3\" >nh_white</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >1.59%</td> \n",
       "        <td id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >2.31%</td> \n",
       "        <td id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >3.46%</td> \n",
       "        <td id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >92.63%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >2.40%</td> \n",
       "        <td id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >3.44%</td> \n",
       "        <td id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >3.93%</td> \n",
       "        <td id=\"T_d442d9d2_4c7e_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >90.23%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb2049a3e50>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "udf = rdf.drop_duplicates(subset=['contributor_name']).copy()\n",
    "gdf = udf.groupby(['year', 'race']).agg({'contributor_name': 'count'})\n",
    "gdf.unstack().apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What proportion of the total donation was given by blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" colspan=4>amount</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level1\" >race</th> \n",
       "        <th class=\"col_heading level1 col0\" >asian</th> \n",
       "        <th class=\"col_heading level1 col1\" >hispanic</th> \n",
       "        <th class=\"col_heading level1 col2\" >nh_black</th> \n",
       "        <th class=\"col_heading level1 col3\" >nh_white</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >1.38%</td> \n",
       "        <td id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >1.71%</td> \n",
       "        <td id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >2.54%</td> \n",
       "        <td id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >94.37%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >1.89%</td> \n",
       "        <td id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >2.00%</td> \n",
       "        <td id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >2.14%</td> \n",
       "        <td id=\"T_71ba76de_4c7a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >93.97%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb2c127f590>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bdf = rdf.groupby(['year', 'race']).agg({'amount': 'sum'})\n",
    "bdf.unstack().apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### What if we estimated by using probabilities for race rather than labels?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What proportion of contributons were by blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >black_count</th> \n",
       "        <th class=\"col_heading level0 col1\" >hispanic_count</th> \n",
       "        <th class=\"col_heading level0 col2\" >white_count</th> \n",
       "        <th class=\"col_heading level0 col3\" >asian_count</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >9.40%</td> \n",
       "        <td id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >3.38%</td> \n",
       "        <td id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >85.34%</td> \n",
       "        <td id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >1.89%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >8.99%</td> \n",
       "        <td id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >4.11%</td> \n",
       "        <td id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >84.74%</td> \n",
       "        <td id=\"T_add09fc8_4c7e_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >2.16%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb2b7b60150>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdf['white_count'] = rdf.nh_white\n",
    "rdf['black_count'] = rdf.nh_black\n",
    "rdf['asian_count'] = rdf.asian\n",
    "rdf['hispanic_count'] = rdf.hispanic\n",
    "gdf = rdf.groupby(['year']).agg({'white_count': 'sum', 'black_count': 'sum', 'asian_count': 'sum', 'hispanic_count': 'sum'})\n",
    "gdf.apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What proportion of the donors were blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >black_count</th> \n",
       "        <th class=\"col_heading level0 col1\" >hispanic_count</th> \n",
       "        <th class=\"col_heading level0 col2\" >white_count</th> \n",
       "        <th class=\"col_heading level0 col3\" >asian_count</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >9.58%</td> \n",
       "        <td id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >3.90%</td> \n",
       "        <td id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >84.23%</td> \n",
       "        <td id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >2.29%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >9.45%</td> \n",
       "        <td id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >5.04%</td> \n",
       "        <td id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >82.45%</td> \n",
       "        <td id=\"T_d99a9d3e_4c7e_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >3.05%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb2b7bc9f90>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "udf['white_count'] = udf.nh_white\n",
    "udf['black_count'] = udf.nh_black\n",
    "udf['asian_count'] = udf.asian\n",
    "udf['hispanic_count'] = udf.hispanic\n",
    "gdf = udf.groupby(['year']).agg({'white_count': 'sum', 'black_count': 'sum', 'asian_count': 'sum', 'hispanic_count': 'sum'})\n",
    "gdf.apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What proportion of the total donation was given by blacks, whites, Hispanics, and Asians?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >api_amount</th> \n",
       "        <th class=\"col_heading level0 col1\" >white_amount</th> \n",
       "        <th class=\"col_heading level0 col2\" >black_amount</th> \n",
       "        <th class=\"col_heading level0 col3\" >hispanic_amount</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >3.60</td> \n",
       "        <td id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >154.43</td> \n",
       "        <td id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >16.07</td> \n",
       "        <td id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >5.82</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >10.32</td> \n",
       "        <td id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >390.74</td> \n",
       "        <td id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >35.77</td> \n",
       "        <td id=\"T_7695cfdc_4c7a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >14.96</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb2b7b57510>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdf['white_amount'] = rdf.amount * rdf.nh_white\n",
    "rdf['black_amount'] = rdf.amount * rdf.nh_black\n",
    "rdf['api_amount'] = rdf.amount * rdf.asian\n",
    "rdf['hispanic_amount'] = rdf.amount * rdf.hispanic\n",
    "gdf = rdf.groupby(['year']).agg({'white_amount': 'sum', 'black_amount': 'sum', 'api_amount': 'sum', 'hispanic_amount': 'sum'}) / 10e6\n",
    "gdf.style.format(\"{:0.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >api_amount</th> \n",
       "        <th class=\"col_heading level0 col1\" >white_amount</th> \n",
       "        <th class=\"col_heading level0 col2\" >black_amount</th> \n",
       "        <th class=\"col_heading level0 col3\" >hispanic_amount</th> \n",
       "    </tr>    <tr> \n",
       "        <th class=\"index_name level0\" >year</th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "        <th class=\"blank\" ></th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175level0_row0\" class=\"row_heading level0 row0\" >2000</th> \n",
       "        <td id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175row0_col0\" class=\"data row0 col0\" >2.00%</td> \n",
       "        <td id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175row0_col1\" class=\"data row0 col1\" >85.84%</td> \n",
       "        <td id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175row0_col2\" class=\"data row0 col2\" >8.93%</td> \n",
       "        <td id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175row0_col3\" class=\"data row0 col3\" >3.23%</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175level0_row1\" class=\"row_heading level0 row1\" >2010</th> \n",
       "        <td id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175row1_col0\" class=\"data row1 col0\" >2.28%</td> \n",
       "        <td id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175row1_col1\" class=\"data row1 col1\" >86.49%</td> \n",
       "        <td id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175row1_col2\" class=\"data row1 col2\" >7.92%</td> \n",
       "        <td id=\"T_7695cfdd_4c7a_11e8_8cdc_fa7917587175row1_col3\" class=\"data row1 col3\" >3.31%</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb2b7bc9f10>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.apply(lambda r: r / r.sum(), axis=1).style.format(\"{:.2%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}