anes_2016_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Welcome to your Quantitative Social Sciences Analysis Toolkit!\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. load survey data into the notebook \n",
    "(run this first & run this everytime you close and reopen the notebook)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd # load a specialized piece of software that will help us with the analysis\n",
    "data = pd.read_csv('data/anes_pilot_2016.csv',low_memory=False) # read in the table of data\n",
    "data = data[data.V161002 != -1] # remove any responent that wasn't interviewed face-to-face"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. display data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1170</th>\n",
       "      <th>1171</th>\n",
       "      <th>1172</th>\n",
       "      <th>1173</th>\n",
       "      <th>1174</th>\n",
       "      <th>1175</th>\n",
       "      <th>1176</th>\n",
       "      <th>1177</th>\n",
       "      <th>1178</th>\n",
       "      <th>1179</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>1170</td>\n",
       "      <td>1171</td>\n",
       "      <td>1172</td>\n",
       "      <td>1173</td>\n",
       "      <td>1174</td>\n",
       "      <td>1175</td>\n",
       "      <td>1176</td>\n",
       "      <td>1177</td>\n",
       "      <td>1178</td>\n",
       "      <td>1179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>version</th>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>...</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "      <td>ANES2016TimeSeries_20190904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V160001</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>1171</td>\n",
       "      <td>1172</td>\n",
       "      <td>1173</td>\n",
       "      <td>1174</td>\n",
       "      <td>1175</td>\n",
       "      <td>1176</td>\n",
       "      <td>1177</td>\n",
       "      <td>1178</td>\n",
       "      <td>1179</td>\n",
       "      <td>1180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V160001_orig</th>\n",
       "      <td>300001</td>\n",
       "      <td>300002</td>\n",
       "      <td>300003</td>\n",
       "      <td>300004</td>\n",
       "      <td>300006</td>\n",
       "      <td>300007</td>\n",
       "      <td>300008</td>\n",
       "      <td>300012</td>\n",
       "      <td>300018</td>\n",
       "      <td>300020</td>\n",
       "      <td>...</td>\n",
       "      <td>302866</td>\n",
       "      <td>302867</td>\n",
       "      <td>302871</td>\n",
       "      <td>302872</td>\n",
       "      <td>302873</td>\n",
       "      <td>302874</td>\n",
       "      <td>302876</td>\n",
       "      <td>302877</td>\n",
       "      <td>302878</td>\n",
       "      <td>302880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V160101</th>\n",
       "      <td>0.827</td>\n",
       "      <td>1.0806</td>\n",
       "      <td>0.3878</td>\n",
       "      <td>0.3596</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.7062</td>\n",
       "      <td>3.9604</td>\n",
       "      <td>0.962</td>\n",
       "      <td>0.9761</td>\n",
       "      <td>0.6182</td>\n",
       "      <td>...</td>\n",
       "      <td>1.4991</td>\n",
       "      <td>0.7705</td>\n",
       "      <td>0.4732</td>\n",
       "      <td>0.6149</td>\n",
       "      <td>1.3355</td>\n",
       "      <td>1.4078</td>\n",
       "      <td>1.0449</td>\n",
       "      <td>1.323</td>\n",
       "      <td>0.552</td>\n",
       "      <td>1.4148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V168524</th>\n",
       "      <td>81</td>\n",
       "      <td>82</td>\n",
       "      <td>82</td>\n",
       "      <td>82</td>\n",
       "      <td>82</td>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "      <td>82</td>\n",
       "      <td>82</td>\n",
       "      <td>...</td>\n",
       "      <td>82</td>\n",
       "      <td>82</td>\n",
       "      <td>81</td>\n",
       "      <td>82</td>\n",
       "      <td>81</td>\n",
       "      <td>82</td>\n",
       "      <td>-1</td>\n",
       "      <td>21</td>\n",
       "      <td>82</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V168525</th>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>...</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>Roy Asberry Cooper</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V168526</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V168527</th>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>...</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>Patrick L. 'Pat' McCrory</td>\n",
       "      <td>-1. Inap</td>\n",
       "      <td>-1. Inap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>V168528</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1843 rows × 1180 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     0                            1     \\\n",
       "Unnamed: 0                              0                            1   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904   \n",
       "V160001                                 1                            2   \n",
       "V160001_orig                       300001                       300002   \n",
       "V160101                             0.827                       1.0806   \n",
       "...                                   ...                          ...   \n",
       "V168524                                81                           82   \n",
       "V168525               -1. Inap                     -1. Inap              \n",
       "V168526                                -1                           -1   \n",
       "V168527       -1. Inap                     -1. Inap                      \n",
       "V168528                                -1                           -1   \n",
       "\n",
       "                                     2                            3     \\\n",
       "Unnamed: 0                              2                            3   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904   \n",
       "V160001                                 3                            4   \n",
       "V160001_orig                       300003                       300004   \n",
       "V160101                            0.3878                       0.3596   \n",
       "...                                   ...                          ...   \n",
       "V168524                                82                           82   \n",
       "V168525               -1. Inap                     -1. Inap              \n",
       "V168526                                -1                           -1   \n",
       "V168527       -1. Inap                     -1. Inap                      \n",
       "V168528                                -1                           -1   \n",
       "\n",
       "                                     4                            5     \\\n",
       "Unnamed: 0                              4                            5   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904   \n",
       "V160001                                 5                            6   \n",
       "V160001_orig                       300006                       300007   \n",
       "V160101                             0.647                       0.7062   \n",
       "...                                   ...                          ...   \n",
       "V168524                                82                           81   \n",
       "V168525               -1. Inap                     -1. Inap              \n",
       "V168526                                -1                           -1   \n",
       "V168527       -1. Inap                     -1. Inap                      \n",
       "V168528                                -1                           -1   \n",
       "\n",
       "                                     6                            7     \\\n",
       "Unnamed: 0                              6                            7   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904   \n",
       "V160001                                 7                            8   \n",
       "V160001_orig                       300008                       300012   \n",
       "V160101                            3.9604                        0.962   \n",
       "...                                   ...                          ...   \n",
       "V168524                                81                           81   \n",
       "V168525               -1. Inap                     -1. Inap              \n",
       "V168526                                -1                           -1   \n",
       "V168527       -1. Inap                     -1. Inap                      \n",
       "V168528                                -1                           -1   \n",
       "\n",
       "                                     8                            9     ...  \\\n",
       "Unnamed: 0                              8                            9  ...   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904  ...   \n",
       "V160001                                 9                           10  ...   \n",
       "V160001_orig                       300018                       300020  ...   \n",
       "V160101                            0.9761                       0.6182  ...   \n",
       "...                                   ...                          ...  ...   \n",
       "V168524                                82                           82  ...   \n",
       "V168525               -1. Inap                     -1. Inap             ...   \n",
       "V168526                                -1                           -1  ...   \n",
       "V168527       -1. Inap                     -1. Inap                     ...   \n",
       "V168528                                -1                           -1  ...   \n",
       "\n",
       "                                     1170                         1171  \\\n",
       "Unnamed: 0                           1170                         1171   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904   \n",
       "V160001                              1171                         1172   \n",
       "V160001_orig                       302866                       302867   \n",
       "V160101                            1.4991                       0.7705   \n",
       "...                                   ...                          ...   \n",
       "V168524                                82                           82   \n",
       "V168525               -1. Inap                     -1. Inap              \n",
       "V168526                                -1                           -1   \n",
       "V168527       -1. Inap                     -1. Inap                      \n",
       "V168528                                -1                           -1   \n",
       "\n",
       "                                     1172                         1173  \\\n",
       "Unnamed: 0                           1172                         1173   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904   \n",
       "V160001                              1173                         1174   \n",
       "V160001_orig                       302871                       302872   \n",
       "V160101                            0.4732                       0.6149   \n",
       "...                                   ...                          ...   \n",
       "V168524                                81                           82   \n",
       "V168525               -1. Inap                     -1. Inap              \n",
       "V168526                                -1                           -1   \n",
       "V168527       -1. Inap                     -1. Inap                      \n",
       "V168528                                -1                           -1   \n",
       "\n",
       "                                     1174                         1175  \\\n",
       "Unnamed: 0                           1174                         1175   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904   \n",
       "V160001                              1175                         1176   \n",
       "V160001_orig                       302873                       302874   \n",
       "V160101                            1.3355                       1.4078   \n",
       "...                                   ...                          ...   \n",
       "V168524                                81                           82   \n",
       "V168525               -1. Inap                     -1. Inap              \n",
       "V168526                                -1                           -1   \n",
       "V168527       -1. Inap                     -1. Inap                      \n",
       "V168528                                -1                           -1   \n",
       "\n",
       "                                     1176                         1177  \\\n",
       "Unnamed: 0                           1176                         1177   \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904   \n",
       "V160001                              1177                         1178   \n",
       "V160001_orig                       302876                       302877   \n",
       "V160101                            1.0449                        1.323   \n",
       "...                                   ...                          ...   \n",
       "V168524                                -1                           21   \n",
       "V168525               -1. Inap                     Roy Asberry Cooper    \n",
       "V168526                                -1                            1   \n",
       "V168527       -1. Inap                     Patrick L. 'Pat' McCrory      \n",
       "V168528                                -1                            1   \n",
       "\n",
       "                                     1178                         1179  \n",
       "Unnamed: 0                           1178                         1179  \n",
       "version       ANES2016TimeSeries_20190904  ANES2016TimeSeries_20190904  \n",
       "V160001                              1179                         1180  \n",
       "V160001_orig                       302878                       302880  \n",
       "V160101                             0.552                       1.4148  \n",
       "...                                   ...                          ...  \n",
       "V168524                                82                           81  \n",
       "V168525               -1. Inap                     -1. Inap             \n",
       "V168526                                -1                           -1  \n",
       "V168527       -1. Inap                     -1. Inap                     \n",
       "V168528                                -1                           -1  \n",
       "\n",
       "[1843 rows x 1180 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.T # display a snapshot of raw data -- the first column here shows your variables, \n",
    "       # the other colums are responses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. get category counts for a categorical variable\n",
    "\n",
    "\n",
    "V161002 = gender  \n",
    "V161027 = for whom did respondent vote?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4fb81d2faed5443782fcb5dd86366ee9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='variable', index=18, options=('Unnamed: 0', 'V160001', 'V160001_or…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from prettytable import PrettyTable\n",
    "from ipywidgets import interact\n",
    "from IPython.core.display import display, HTML\n",
    "\n",
    "@interact(variable=data.columns.sort_values())\n",
    "def categorical_table(variable='V161002'):\n",
    "    x = PrettyTable()\n",
    "    x.field_names = [variable, 'Count']\n",
    "    for i, row in data.groupby(variable).size().reset_index().iterrows():\n",
    "        x.add_row((row[variable], row[0]))\n",
    "    display(HTML(x.get_html_string()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. get average and spread for a continuous variable\n",
    "\n",
    "V161267 = age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f4a9a2f672a4d1faedbb6dd2c48fbe4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='variable', options=('Unnamed: 0', 'V160001', 'V160001_orig', 'V160…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from prettytable import PrettyTable\n",
    "from ipywidgets import interact, widgets\n",
    "import numpy as np\n",
    "from IPython.core.display import display, HTML\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "def cast(v):\n",
    "    try:\n",
    "        return float(v)\n",
    "    except:\n",
    "        return np.nan\n",
    "\n",
    "\n",
    "variable_select = widgets.Dropdown(options=data.columns.sort_values())\n",
    "\n",
    "drop_select = widgets.SelectMultiple(options=[])\n",
    "\n",
    "def update_drop_select(*args):\n",
    "    drop_select.options=np.sort(data[variable_select.value].unique())\n",
    "\n",
    "variable_select.observe(update_drop_select, 'value')\n",
    "\n",
    "def printer(variable, drop_vals, drop_na, zoom=widgets.IntSlider(min=10,max=100,step=5,value=10)):\n",
    "    df = data.copy()\n",
    "    df[variable] = df[variable].apply(cast)\n",
    "    \n",
    "    df= df[[v not in drop_vals for v in df[variable]]]\n",
    "\n",
    "    if drop_na:\n",
    "        df = df[df[variable] > 0]\n",
    "    \n",
    "    if len(drop_vals):\n",
    "        print('dropped values: {}'.format(drop_vals))\n",
    "        \n",
    "    x = PrettyTable()\n",
    "    x.field_names = [variable, 'mean', 'standard deviation']\n",
    "    mu = np.mean(df[variable])\n",
    "    sigma = np.std(df[variable])\n",
    "    \n",
    "    result = (variable, mu, sigma)\n",
    "    x.add_row(result)\n",
    "    \n",
    "    display(HTML(x.get_html_string()))\n",
    "    plt.figure(figsize=(10,5))\n",
    "    plt.hist(df[variable], bins=zoom)\n",
    "    ax = plt.gca()\n",
    "    ymin, ymax = ax.get_ylim()\n",
    "    \n",
    "    for val in range(-3,3):\n",
    "        x = val*sigma+mu \n",
    "        col='black'\n",
    "        \n",
    "        if val==0:\n",
    "            ax.vlines(x,ymin,ymax, alpha=1, color='red')\n",
    "        else:\n",
    "            ax.vlines(x,ymin,ymax/np.abs(val)/(zoom/10), alpha=1/np.abs(val), color=col)\n",
    "            \n",
    "interact(printer, drop_vals=drop_select, variable=variable_select, drop_na=True);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Compare two categorical variables (or ordinal)\n",
    "\n",
    "V165602 = something  \n",
    "V161002 = gender  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0725134b18c84448ad8a20467cba2fad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='dependent_variable', options=('Unnamed: 0', 'V160001', 'V160001_or…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from ipywidgets import interact\n",
    "import scipy.stats as scs\n",
    "from scipy.stats import chi2_contingency\n",
    "\n",
    "\n",
    "dependent_variable_select = widgets.Dropdown(options=data.columns.sort_values())\n",
    "independent_variable_select = widgets.Dropdown(options=data.columns.sort_values())\n",
    "\n",
    "dependent_drop_select = widgets.SelectMultiple(options=[])\n",
    "independent_drop_select = widgets.SelectMultiple(options=[])\n",
    "\n",
    "def update_dependent_drop_select(*args):\n",
    "    dependent_drop_select.options=np.sort(data[dependent_variable_select.value].unique())\n",
    "\n",
    "\n",
    "def update_independent_drop_select(*args):\n",
    "    independent_drop_select.options=np.sort(data[independent_variable_select.value].unique())\n",
    "\n",
    "    \n",
    "dependent_variable_select.observe(update_dependent_drop_select, 'value')\n",
    "independent_variable_select.observe(update_independent_drop_select, 'value')\n",
    "\n",
    "def categorical_table(dependent_variable, independent_variable, dep_drop_vals, indep_drop_vals,drop_na=True):\n",
    "    df = data.copy()\n",
    "    \n",
    "    if drop_na:\n",
    "        for variable in [independent_variable, dependent_variable]:\n",
    "            try:\n",
    "                df[variable] = df[variable].astype(float)\n",
    "                df = df[df[variable]>-1]\n",
    "            except:\n",
    "                \n",
    "                pass\n",
    "            \n",
    "    df = df[[v not in dep_drop_vals for v in df[dependent_variable]]]\n",
    "    df = df[[v not in indep_drop_vals for v in df[independent_variable]]]\n",
    "            \n",
    "        \n",
    "    if len(dep_drop_vals):\n",
    "        print('dropped dependent values: {}'.format(dep_drop_vals))\n",
    "        \n",
    "    if len(indep_drop_vals):\n",
    "        print('dropped independent values: {}'.format(indep_drop_vals))\n",
    "        \n",
    "    cross_tab = pd.crosstab(df[dependent_variable], df[independent_variable])\n",
    "    stats =  chi2_contingency(cross_tab)\n",
    "    print(\"chi-sq = {}, p-val = {}\".format(round(stats[0],5), round(stats[1], 5)))\n",
    "    return cross_tab\n",
    "\n",
    "interact(categorical_table, dependent_variable=dependent_variable_select ,\n",
    "         independent_variable=independent_variable_select, dep_drop_vals=dependent_drop_select, \n",
    "         indep_drop_vals=independent_drop_select, drop_na=True);\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Compare a categorical with a numeric/ordinal\n",
    "\n",
    "V165602 = something  \n",
    "V161361x = something   \n",
    "V161002 = gender  \n",
    "V161267 = age  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33ffd20daae4445591cd4bb8b975a792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='categorical_variable', options=('Unnamed: 0', 'V160001', 'V160001_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from ipywidgets import interact\n",
    "import scipy.stats as scs\n",
    "import statsmodels.api as sm\n",
    "from statsmodels.formula.api import ols\n",
    "\n",
    "\n",
    "categorical_variable_select = widgets.Dropdown(options=data.columns.sort_values())\n",
    "numeric_variable_select = widgets.Dropdown(options=data.columns.sort_values())\n",
    "\n",
    "categorical_drop_select = widgets.SelectMultiple(options=[])\n",
    "numeric_drop_select = widgets.SelectMultiple(options=[])\n",
    "\n",
    "def update_categorical_drop_select(*args):\n",
    "    categorical_drop_select.options=np.sort(data[categorical_variable_select.value].unique())\n",
    "\n",
    "def update_numeric_drop_select(*args):\n",
    "    numeric_drop_select.options=np.sort(data[numeric_variable_select.value].unique())\n",
    "\n",
    "    \n",
    "categorical_variable_select.observe(update_categorical_drop_select, 'value')\n",
    "numeric_variable_select.observe(update_numeric_drop_select, 'value')\n",
    "\n",
    "\n",
    "def categorical_table(categorical_variable, numeric_variable,cat_drop_vals, num_drop_vals, drop_na=True):\n",
    "    df = data.copy()\n",
    "    \n",
    "    if drop_na:\n",
    "        for variable in [categorical_variable, numeric_variable]:\n",
    "            try:\n",
    "                df[variable] = df[variable].astype(float)\n",
    "                df = df[df[variable]>-1]\n",
    "            except:\n",
    "                pass\n",
    "        \n",
    "    if len(df[categorical_variable].unique())>15:\n",
    "        print(\"PLEASE CHOOSE A CATEGORICAL VARIABLE\")\n",
    "        return\n",
    "    \n",
    "    try:\n",
    "        df[numeric_variable].astype(float)\n",
    "    except:\n",
    "        print(\"PLEASE CHOOSE A NUMERIC VARIABLE\")\n",
    "        return\n",
    "\n",
    "    \n",
    "    df = df[[v not in cat_drop_vals for v in df[categorical_variable]]]\n",
    "    df = df[[v not in num_drop_vals for v in df[numeric_variable]]]\n",
    "            \n",
    "        \n",
    "    if len(cat_drop_vals):\n",
    "        print('dropped dependent values: {}'.format(cat_drop_vals))\n",
    "        \n",
    "    if len(num_drop_vals):\n",
    "        print('dropped independent values: {}'.format(num_drop_vals))\n",
    "        \n",
    "    \n",
    "    \n",
    "    plt.figure(figsize=(10,5))\n",
    "    ax=plt.gca()\n",
    "    for c in np.sort(df[categorical_variable].unique()):\n",
    "        dat = df[df[categorical_variable]==c]\n",
    "        ax.hist(dat[numeric_variable], alpha=.5, bins='doane')\n",
    "    ax.legend(np.sort(df[categorical_variable].unique()))\n",
    "    \n",
    "    X = df[numeric_variable]\n",
    "    X = sm.add_constant(X)\n",
    "    \n",
    "    res = ols(\"{} ~ C({})\".format(numeric_variable, categorical_variable), df).fit()\n",
    "    pw = res.t_test_pairwise(\"C({})\".format(categorical_variable))\n",
    "    return pw.result_frame[['t','P>|t|','pvalue-hs','reject-hs']]\n",
    "    \n",
    "    \n",
    "    \n",
    "interact(categorical_table, categorical_variable=categorical_variable_select,\n",
    "         numeric_variable=numeric_variable_select, cat_drop_vals=categorical_drop_select, \n",
    "         num_drop_vals=numeric_drop_select, drop_na=True)\n",
    "pass\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}