ethnicolr/models/ethnicolr_keras_lstm_census2010_ln.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 01:19:21.478834: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2021-12-21 01:19:21.478877: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n",
      "2.5.2\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import tensorflow as tf\n",
    "print(keras.__version__)\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>rank</th>\n",
       "      <th>count</th>\n",
       "      <th>prop100k</th>\n",
       "      <th>cum_prop100k</th>\n",
       "      <th>pctwhite</th>\n",
       "      <th>pctblack</th>\n",
       "      <th>pctapi</th>\n",
       "      <th>pctaian</th>\n",
       "      <th>pct2prace</th>\n",
       "      <th>pcthispanic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SMITH</td>\n",
       "      <td>1</td>\n",
       "      <td>2442977</td>\n",
       "      <td>828.19</td>\n",
       "      <td>828.19</td>\n",
       "      <td>70.9</td>\n",
       "      <td>23.11</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.89</td>\n",
       "      <td>2.19</td>\n",
       "      <td>2.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>JOHNSON</td>\n",
       "      <td>2</td>\n",
       "      <td>1932812</td>\n",
       "      <td>655.24</td>\n",
       "      <td>1483.42</td>\n",
       "      <td>58.97</td>\n",
       "      <td>34.63</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.94</td>\n",
       "      <td>2.56</td>\n",
       "      <td>2.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WILLIAMS</td>\n",
       "      <td>3</td>\n",
       "      <td>1625252</td>\n",
       "      <td>550.97</td>\n",
       "      <td>2034.39</td>\n",
       "      <td>45.75</td>\n",
       "      <td>47.68</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.82</td>\n",
       "      <td>2.81</td>\n",
       "      <td>2.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BROWN</td>\n",
       "      <td>4</td>\n",
       "      <td>1437026</td>\n",
       "      <td>487.16</td>\n",
       "      <td>2521.56</td>\n",
       "      <td>57.95</td>\n",
       "      <td>35.6</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.87</td>\n",
       "      <td>2.55</td>\n",
       "      <td>2.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>JONES</td>\n",
       "      <td>5</td>\n",
       "      <td>1425470</td>\n",
       "      <td>483.24</td>\n",
       "      <td>3004.80</td>\n",
       "      <td>55.19</td>\n",
       "      <td>38.48</td>\n",
       "      <td>0.44</td>\n",
       "      <td>1</td>\n",
       "      <td>2.61</td>\n",
       "      <td>2.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162249</th>\n",
       "      <td>DIETZMANN</td>\n",
       "      <td>160975</td>\n",
       "      <td>100</td>\n",
       "      <td>0.03</td>\n",
       "      <td>90062.93</td>\n",
       "      <td>96</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162250</th>\n",
       "      <td>DOKAS</td>\n",
       "      <td>160975</td>\n",
       "      <td>100</td>\n",
       "      <td>0.03</td>\n",
       "      <td>90062.96</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162251</th>\n",
       "      <td>DONLEA</td>\n",
       "      <td>160975</td>\n",
       "      <td>100</td>\n",
       "      <td>0.03</td>\n",
       "      <td>90062.99</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162252</th>\n",
       "      <td>DORIOTT</td>\n",
       "      <td>160975</td>\n",
       "      <td>100</td>\n",
       "      <td>0.03</td>\n",
       "      <td>90063.03</td>\n",
       "      <td>89</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162253</th>\n",
       "      <td>ALL OTHER NAMES</td>\n",
       "      <td>0</td>\n",
       "      <td>29312001</td>\n",
       "      <td>9936.97</td>\n",
       "      <td>9936.97</td>\n",
       "      <td>66.65</td>\n",
       "      <td>8.53</td>\n",
       "      <td>7.97</td>\n",
       "      <td>0.86</td>\n",
       "      <td>2.32</td>\n",
       "      <td>13.67</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>162253 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   name    rank     count  prop100k  cum_prop100k pctwhite  \\\n",
       "0                 SMITH       1   2442977    828.19        828.19     70.9   \n",
       "1               JOHNSON       2   1932812    655.24       1483.42    58.97   \n",
       "2              WILLIAMS       3   1625252    550.97       2034.39    45.75   \n",
       "3                 BROWN       4   1437026    487.16       2521.56    57.95   \n",
       "4                 JONES       5   1425470    483.24       3004.80    55.19   \n",
       "...                 ...     ...       ...       ...           ...      ...   \n",
       "162249        DIETZMANN  160975       100      0.03      90062.93       96   \n",
       "162250            DOKAS  160975       100      0.03      90062.96       94   \n",
       "162251           DONLEA  160975       100      0.03      90062.99       94   \n",
       "162252          DORIOTT  160975       100      0.03      90063.03       89   \n",
       "162253  ALL OTHER NAMES       0  29312001   9936.97       9936.97    66.65   \n",
       "\n",
       "       pctblack pctapi pctaian pct2prace pcthispanic  \n",
       "0         23.11    0.5    0.89      2.19         2.4  \n",
       "1         34.63   0.54    0.94      2.56        2.36  \n",
       "2         47.68   0.46    0.82      2.81        2.49  \n",
       "3          35.6   0.51    0.87      2.55        2.52  \n",
       "4         38.48   0.44       1      2.61        2.29  \n",
       "...         ...    ...     ...       ...         ...  \n",
       "162249        0      0       0         0           0  \n",
       "162250        0      0       0         0           0  \n",
       "162251        0      0       0         0           6  \n",
       "162252        0      0       0         5           0  \n",
       "162253     8.53   7.97    0.86      2.32       13.67  \n",
       "\n",
       "[162253 rows x 11 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "\n",
    "NGRAMS = 2\n",
    "EPOCHS = 15\n",
    "#YEAR = '2000'\n",
    "YEAR = '2010'\n",
    "\n",
    "df = pd.read_csv('../data/census/census_%s.csv' % YEAR)\n",
    "df.dropna(subset=['name'], inplace=True)\n",
    "df.replace('(S)', 0, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resampling with weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sdf = df.sample(1000000, weights=df['count'], replace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assign race by pertcentage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>rank</th>\n",
       "      <th>count</th>\n",
       "      <th>prop100k</th>\n",
       "      <th>cum_prop100k</th>\n",
       "      <th>pctwhite</th>\n",
       "      <th>pctblack</th>\n",
       "      <th>pctapi</th>\n",
       "      <th>pctaian</th>\n",
       "      <th>pct2prace</th>\n",
       "      <th>pcthispanic</th>\n",
       "      <th>race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WILLIAMS</td>\n",
       "      <td>3</td>\n",
       "      <td>1625252</td>\n",
       "      <td>550.97</td>\n",
       "      <td>2034.39</td>\n",
       "      <td>45.75</td>\n",
       "      <td>47.68</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.82</td>\n",
       "      <td>2.81</td>\n",
       "      <td>2.49</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>TAYLOR</td>\n",
       "      <td>17</td>\n",
       "      <td>751209</td>\n",
       "      <td>254.67</td>\n",
       "      <td>6886.93</td>\n",
       "      <td>65.38</td>\n",
       "      <td>28.42</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.76</td>\n",
       "      <td>2.41</td>\n",
       "      <td>2.46</td>\n",
       "      <td>black</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WILLIAMS</td>\n",
       "      <td>3</td>\n",
       "      <td>1625252</td>\n",
       "      <td>550.97</td>\n",
       "      <td>2034.39</td>\n",
       "      <td>45.75</td>\n",
       "      <td>47.68</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.82</td>\n",
       "      <td>2.81</td>\n",
       "      <td>2.49</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42423</th>\n",
       "      <td>KASA</td>\n",
       "      <td>42378</td>\n",
       "      <td>512</td>\n",
       "      <td>0.17</td>\n",
       "      <td>81468.90</td>\n",
       "      <td>74.02</td>\n",
       "      <td>8.4</td>\n",
       "      <td>10.35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.47</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8516</th>\n",
       "      <td>LEPAGE</td>\n",
       "      <td>8517</td>\n",
       "      <td>3865</td>\n",
       "      <td>1.31</td>\n",
       "      <td>66581.46</td>\n",
       "      <td>94.1</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.26</td>\n",
       "      <td>1.97</td>\n",
       "      <td>2.64</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>SMALL</td>\n",
       "      <td>861</td>\n",
       "      <td>40212</td>\n",
       "      <td>13.63</td>\n",
       "      <td>39207.27</td>\n",
       "      <td>62.66</td>\n",
       "      <td>30.81</td>\n",
       "      <td>0.58</td>\n",
       "      <td>1.41</td>\n",
       "      <td>1.93</td>\n",
       "      <td>2.61</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>WALKER</td>\n",
       "      <td>31</td>\n",
       "      <td>523129</td>\n",
       "      <td>177.34</td>\n",
       "      <td>9862.30</td>\n",
       "      <td>58.66</td>\n",
       "      <td>35.05</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.87</td>\n",
       "      <td>2.55</td>\n",
       "      <td>2.41</td>\n",
       "      <td>black</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>ALVAREZ</td>\n",
       "      <td>92</td>\n",
       "      <td>233983</td>\n",
       "      <td>79.32</td>\n",
       "      <td>16697.86</td>\n",
       "      <td>5.18</td>\n",
       "      <td>0.6</td>\n",
       "      <td>1.16</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.23</td>\n",
       "      <td>92.45</td>\n",
       "      <td>hispanic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9396</th>\n",
       "      <td>KEMPTON</td>\n",
       "      <td>9397</td>\n",
       "      <td>3460</td>\n",
       "      <td>1.17</td>\n",
       "      <td>67672.28</td>\n",
       "      <td>92.54</td>\n",
       "      <td>1.79</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.46</td>\n",
       "      <td>1.97</td>\n",
       "      <td>2.69</td>\n",
       "      <td>hispanic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>JOHNSON</td>\n",
       "      <td>2</td>\n",
       "      <td>1932812</td>\n",
       "      <td>655.24</td>\n",
       "      <td>1483.42</td>\n",
       "      <td>58.97</td>\n",
       "      <td>34.63</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.94</td>\n",
       "      <td>2.56</td>\n",
       "      <td>2.36</td>\n",
       "      <td>black</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000000 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           name   rank    count  prop100k  cum_prop100k pctwhite pctblack  \\\n",
       "2      WILLIAMS      3  1625252    550.97       2034.39    45.75    47.68   \n",
       "16       TAYLOR     17   751209    254.67       6886.93    65.38    28.42   \n",
       "2      WILLIAMS      3  1625252    550.97       2034.39    45.75    47.68   \n",
       "42423      KASA  42378      512      0.17      81468.90    74.02      8.4   \n",
       "8516     LEPAGE   8517     3865      1.31      66581.46     94.1     0.44   \n",
       "...         ...    ...      ...       ...           ...      ...      ...   \n",
       "860       SMALL    861    40212     13.63      39207.27    62.66    30.81   \n",
       "30       WALKER     31   523129    177.34       9862.30    58.66    35.05   \n",
       "91      ALVAREZ     92   233983     79.32      16697.86     5.18      0.6   \n",
       "9396    KEMPTON   9397     3460      1.17      67672.28    92.54     1.79   \n",
       "1       JOHNSON      2  1932812    655.24       1483.42    58.97    34.63   \n",
       "\n",
       "      pctapi pctaian pct2prace pcthispanic      race  \n",
       "2       0.46    0.82      2.81        2.49     white  \n",
       "16      0.56    0.76      2.41        2.46     black  \n",
       "2       0.46    0.82      2.81        2.49     white  \n",
       "42423  10.35       0         0        5.47     white  \n",
       "8516     0.6    0.26      1.97        2.64     white  \n",
       "...      ...     ...       ...         ...       ...  \n",
       "860     0.58    1.41      1.93        2.61     white  \n",
       "30      0.45    0.87      2.55        2.41     black  \n",
       "91      1.16    0.38      0.23       92.45  hispanic  \n",
       "9396    0.55    0.46      1.97        2.69  hispanic  \n",
       "1       0.54    0.94      2.56        2.36     black  \n",
       "\n",
       "[1000000 rows x 12 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from numpy.random import choice\n",
    "\n",
    "races = ['white', 'black', 'api', 'hispanic']\n",
    "\n",
    "def to_race(c):\n",
    "    w = np.array(c).astype(float)\n",
    "    if w.sum() == 0:\n",
    "        return 'white'\n",
    "    probs = w/w.sum()\n",
    "    return choice(races, p=probs)\n",
    "\n",
    "sdf['race'] = sdf[['pctwhite', 'pctblack', 'pctapi', 'pcthispanic']].apply(lambda c: to_race(c), axis=1)\n",
    "sdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check the correctness of race assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>rank</th>\n",
       "      <th>count</th>\n",
       "      <th>prop100k</th>\n",
       "      <th>cum_prop100k</th>\n",
       "      <th>pctwhite</th>\n",
       "      <th>pctblack</th>\n",
       "      <th>pctapi</th>\n",
       "      <th>pctaian</th>\n",
       "      <th>pct2prace</th>\n",
       "      <th>pcthispanic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SMITH</td>\n",
       "      <td>1</td>\n",
       "      <td>2442977</td>\n",
       "      <td>828.19</td>\n",
       "      <td>828.19</td>\n",
       "      <td>70.9</td>\n",
       "      <td>23.11</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.89</td>\n",
       "      <td>2.19</td>\n",
       "      <td>2.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    name  rank    count  prop100k  cum_prop100k pctwhite pctblack pctapi  \\\n",
       "0  SMITH     1  2442977    828.19        828.19     70.9    23.11    0.5   \n",
       "\n",
       "  pctaian pct2prace pcthispanic  \n",
       "0    0.89      2.19         2.4  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.name == 'SMITH']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>api</th>\n",
       "      <td>0.464946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>black</th>\n",
       "      <td>24.054815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>2.691790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>white</th>\n",
       "      <td>72.788450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name\n",
       "race               \n",
       "api        0.464946\n",
       "black     24.054815\n",
       "hispanic   2.691790\n",
       "white     72.788450"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xdf = sdf[sdf.name=='SMITH'].groupby(['race']).agg({'name': 'count'})\n",
    "xdf * 100 / xdf.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>api</th>\n",
       "      <td>49841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>black</th>\n",
       "      <td>123962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>165010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>white</th>\n",
       "      <td>661187</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name_last\n",
       "race               \n",
       "api           49841\n",
       "black        123962\n",
       "hispanic     165010\n",
       "white        661187"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Additional features\n",
    "sdf['name_last'] = sdf.name.str.title()\n",
    "sdf.groupby('race').agg({'name_last': 'count'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing the input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "976"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# only last name in Census data\n",
    "sdf['name_last_name_first'] = sdf['name_last']\n",
    "\n",
    "# build n-gram list\n",
    "vect = CountVectorizer(analyzer='char', max_df=0.3, min_df=3, ngram_range=(NGRAMS, NGRAMS), lowercase=False) \n",
    "#vect = CountVectorizer(analyzer='char', ngram_range=(2, 2), lowercase=False) \n",
    "a = vect.fit_transform(sdf.name_last_name_first)\n",
    "vocab = vect.vocabulary_\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import operator\n",
    "sorted_vocab = sorted(vocab.items(), key=operator.itemgetter(1))\n",
    "cols = list(map(operator.itemgetter(0), sorted_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>N</th>\n",
       "      <th>O</th>\n",
       "      <th>Aa</th>\n",
       "      <th>Ab</th>\n",
       "      <th>Ac</th>\n",
       "      <th>Ad</th>\n",
       "      <th>Ae</th>\n",
       "      <th>Af</th>\n",
       "      <th>Ag</th>\n",
       "      <th>Ah</th>\n",
       "      <th>...</th>\n",
       "      <th>zp</th>\n",
       "      <th>zq</th>\n",
       "      <th>zr</th>\n",
       "      <th>zs</th>\n",
       "      <th>zt</th>\n",
       "      <th>zu</th>\n",
       "      <th>zv</th>\n",
       "      <th>zw</th>\n",
       "      <th>zy</th>\n",
       "      <th>zz</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999995</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999996</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999997</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999998</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999999</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000000 rows × 976 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         N   O  Aa  Ab  Ac  Ad  Ae  Af  Ag  Ah  ...  zp  zq  zr  zs  zt  zu  \\\n",
       "0        0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "1        0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "2        0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "3        0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "4        0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "...     ..  ..  ..  ..  ..  ..  ..  ..  ..  ..  ...  ..  ..  ..  ..  ..  ..   \n",
       "999995   0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "999996   0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "999997   0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "999998   0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "999999   0   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   \n",
       "\n",
       "        zv  zw  zy  zz  \n",
       "0        0   0   0   0  \n",
       "1        0   0   0   0  \n",
       "2        0   0   0   0  \n",
       "3        0   0   0   0  \n",
       "4        0   0   0   0  \n",
       "...     ..  ..  ..  ..  \n",
       "999995   0   0   0   0  \n",
       "999996   0   0   0   0  \n",
       "999997   0   0   0   0  \n",
       "999998   0   0   0   0  \n",
       "999999   0   0   0   0  \n",
       "\n",
       "[1000000 rows x 976 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_df = pd.DataFrame(a.todense(), columns=cols)\n",
    "count_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count       976.000000\n",
       "mean       6322.976434\n",
       "std       18148.435773\n",
       "min           3.000000\n",
       "25%          75.750000\n",
       "50%         845.500000\n",
       "75%        4990.750000\n",
       "max      254005.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_df.sum().sort_values(ascending=False).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "er    254005\n",
       "ll    174191\n",
       "es    143601\n",
       "am    131473\n",
       "he    123684\n",
       "       ...  \n",
       "Iu         3\n",
       "tq         3\n",
       "Ej         3\n",
       "vj         3\n",
       "mv         3\n",
       "Length: 976, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_rows', 20)\n",
    "count_df.sum().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_words = 977\n"
     ]
    }
   ],
   "source": [
    "# sort n-gram by freq (highest -> lowest)\n",
    "words = []\n",
    "for b in vocab:\n",
    "    c = vocab[b]\n",
    "    #print(b, c, a[:, c].sum())\n",
    "    words.append((a[:, c].sum(), b))\n",
    "    #break\n",
    "words = sorted(words, reverse=True)\n",
    "words_list = ['UNK']\n",
    "words_list.extend([w[1] for w in words])\n",
    "num_words = len(words_list)\n",
    "print(\"num_words = %d\" % num_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max feature len = 14, Avg. feature len = 6\n"
     ]
    }
   ],
   "source": [
    "def find_ngrams(text, n):\n",
    "    a = zip(*[text[i:] for i in range(n)])\n",
    "    wi = []\n",
    "    for i in a:\n",
    "        w = ''.join(i)\n",
    "        try:\n",
    "            idx = words_list.index(w)\n",
    "        except:\n",
    "            idx = 0\n",
    "        wi.append(idx)\n",
    "    return wi\n",
    "\n",
    "# build X from index of n-gram sequence\n",
    "X = np.array(sdf.name_last_name_first.apply(lambda c: find_ngrams(c, NGRAMS)))\n",
    "\n",
    "# check max/avg feature\n",
    "X_len = []\n",
    "for x in X:\n",
    "    X_len.append(len(x))\n",
    "\n",
    "max_feature_len = max(X_len)\n",
    "avg_feature_len = int(np.mean(X_len))\n",
    "\n",
    "print(\"Max feature len = %d, Avg. feature len = %d\" % (max_feature_len, avg_feature_len))\n",
    "y = np.array(sdf.race.astype('category').cat.codes)\n",
    "\n",
    "# Split train and test dataset\n",
    "X_train,  X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=21, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a LSTM model\n",
    "\n",
    "ref: http://machinelearningmastery.com/sequence-classification-lstm-recurrent-neural-networks-python-keras/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800000 train sequences\n",
      "200000 test sequences\n",
      "Pad sequences (samples x time)\n",
      "X_train shape: (800000, 20)\n",
      "X_test shape: (200000, 20)\n",
      "4 classes\n",
      "Convert class vector to binary class matrix (for use with categorical_crossentropy)\n",
      "y_train shape: (800000, 4)\n",
      "y_test shape: (200000, 4)\n"
     ]
    }
   ],
   "source": [
    "'''The dataset is actually too small for LSTM to be of any advantage\n",
    "compared to simpler, much faster methods such as TF-IDF + LogReg.\n",
    "Notes:\n",
    "\n",
    "- RNNs are tricky. Choice of batch size is important,\n",
    "choice of loss and optimizer is critical, etc.\n",
    "Some configurations won't converge.\n",
    "\n",
    "- LSTM loss decrease patterns during training can be quite different\n",
    "from what you see with CNNs/MLPs/etc.\n",
    "'''\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, Dropout, Activation\n",
    "from keras.layers import LSTM\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D\n",
    "from keras.models import load_model\n",
    "\n",
    "max_features = num_words # 20000\n",
    "feature_len = 20 # avg_feature_len # cut texts after this number of words (among top max_features most common words)\n",
    "batch_size = 32\n",
    "\n",
    "print(len(X_train), 'train sequences')\n",
    "print(len(X_test), 'test sequences')\n",
    "\n",
    "print('Pad sequences (samples x time)')\n",
    "X_train = sequence.pad_sequences(X_train, maxlen=feature_len)\n",
    "X_test = sequence.pad_sequences(X_test, maxlen=feature_len)\n",
    "print('X_train shape:', X_train.shape)\n",
    "print('X_test shape:', X_test.shape)\n",
    "\n",
    "num_classes = np.max(y_train) + 1\n",
    "print(num_classes, 'classes')\n",
    "\n",
    "print('Convert class vector to binary class matrix '\n",
    "      '(for use with categorical_crossentropy)')\n",
    "y_train = tf.keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, num_classes)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print('y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 20, 32)            31264     \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 128)               82432     \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 4)                 516       \n",
      "=================================================================\n",
      "Total params: 114,212\n",
      "Trainable params: 114,212\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 01:25:35.989612: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2021-12-21 01:25:35.989652: W tensorflow/stream_executor/cuda/cuda_driver.cc:326] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2021-12-21 01:25:35.989669: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (jupyter-server-2): /proc/driver/nvidia/version does not exist\n",
      "2021-12-21 01:25:35.989888: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "print('Build model...')\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(num_words, 32, input_length=feature_len))\n",
    "model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "# try using different optimizers and different optimizer configs\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train...\n",
      "Epoch 1/15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 01:25:36.307378: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:176] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2021-12-21 01:25:36.307923: I tensorflow/core/platform/profile_utils/cpu_utils.cc:114] CPU Frequency: 2250000000 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22500/22500 [==============================] - 445s 19ms/step - loss: 0.6870 - accuracy: 0.7653 - val_loss: 0.6236 - val_accuracy: 0.7864\n",
      "Epoch 2/15\n",
      "22500/22500 [==============================] - 415s 18ms/step - loss: 0.6177 - accuracy: 0.7884 - val_loss: 0.6052 - val_accuracy: 0.7917\n",
      "Epoch 3/15\n",
      "22500/22500 [==============================] - 413s 18ms/step - loss: 0.6026 - accuracy: 0.7931 - val_loss: 0.5953 - val_accuracy: 0.7946\n",
      "Epoch 4/15\n",
      "22500/22500 [==============================] - 412s 18ms/step - loss: 0.5930 - accuracy: 0.7959 - val_loss: 0.5899 - val_accuracy: 0.7968\n",
      "Epoch 5/15\n",
      "22500/22500 [==============================] - 412s 18ms/step - loss: 0.5862 - accuracy: 0.7980 - val_loss: 0.5860 - val_accuracy: 0.7978\n",
      "Epoch 6/15\n",
      "22500/22500 [==============================] - 412s 18ms/step - loss: 0.5829 - accuracy: 0.7984 - val_loss: 0.5822 - val_accuracy: 0.7980\n",
      "Epoch 7/15\n",
      "22500/22500 [==============================] - 411s 18ms/step - loss: 0.5836 - accuracy: 0.7980 - val_loss: 0.5809 - val_accuracy: 0.7988\n",
      "Epoch 8/15\n",
      "22500/22500 [==============================] - 413s 18ms/step - loss: 0.5803 - accuracy: 0.7991 - val_loss: 0.5799 - val_accuracy: 0.7989\n",
      "Epoch 9/15\n",
      "22500/22500 [==============================] - 413s 18ms/step - loss: 0.5775 - accuracy: 0.7998 - val_loss: 0.5771 - val_accuracy: 0.8001\n",
      "Epoch 10/15\n",
      "22500/22500 [==============================] - 414s 18ms/step - loss: 0.5774 - accuracy: 0.7996 - val_loss: 0.5782 - val_accuracy: 0.8000\n",
      "Epoch 11/15\n",
      "22500/22500 [==============================] - 412s 18ms/step - loss: 0.5770 - accuracy: 0.8002 - val_loss: 0.5774 - val_accuracy: 0.8002\n",
      "Epoch 12/15\n",
      "22500/22500 [==============================] - 412s 18ms/step - loss: 0.5775 - accuracy: 0.8000 - val_loss: 0.5757 - val_accuracy: 0.8001\n",
      "Epoch 13/15\n",
      "22500/22500 [==============================] - 410s 18ms/step - loss: 0.5766 - accuracy: 0.8000 - val_loss: 0.5753 - val_accuracy: 0.8008\n",
      "Epoch 14/15\n",
      "22500/22500 [==============================] - 410s 18ms/step - loss: 0.5770 - accuracy: 0.7994 - val_loss: 0.5747 - val_accuracy: 0.8009\n",
      "Epoch 15/15\n",
      "22500/22500 [==============================] - 410s 18ms/step - loss: 0.5737 - accuracy: 0.8012 - val_loss: 0.5751 - val_accuracy: 0.8003\n",
      "6250/6250 [==============================] - 29s 5ms/step - loss: 0.5726 - accuracy: 0.8025\n",
      "Test score: 0.5725786685943604\n",
      "Test accuracy: 0.8024849891662598\n"
     ]
    }
   ],
   "source": [
    "print('Train...')\n",
    "model.fit(X_train, y_train, batch_size=batch_size, epochs=EPOCHS,\n",
    "          validation_split=0.1, verbose=1)\n",
    "score, acc = model.evaluate(X_test, y_test,\n",
    "                            batch_size=batch_size, verbose=1)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6250/6250 - 27s\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         api       0.83      0.57      0.68      9968\n",
      "       black       0.55      0.08      0.14     24792\n",
      "    hispanic       0.88      0.77      0.82     33002\n",
      "       white       0.79      0.96      0.87    132238\n",
      "\n",
      "    accuracy                           0.80    200000\n",
      "   macro avg       0.76      0.60      0.63    200000\n",
      "weighted avg       0.78      0.80      0.76    200000\n",
      "\n",
      "[[  5708     36    773   3451]\n",
      " [   271   1946    251  22324]\n",
      " [   157    120  25321   7404]\n",
      " [   731   1441   2544 127522]]\n"
     ]
    }
   ],
   "source": [
    "p = model.predict(X_test, verbose=2) # to predict probability\n",
    "y_pred = np.argmax(p, axis=-1)\n",
    "target_names = list(sdf.race.astype('category').cat.categories)\n",
    "print(classification_report(np.argmax(y_test, axis=1), y_pred, target_names=target_names))\n",
    "print(confusion_matrix(np.argmax(y_test, axis=1), y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('./census/lstm/census%s_ln_lstm.h5' % YEAR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_df = pd.DataFrame(words_list, columns=['vocab'])\n",
    "words_df.to_csv('./census/lstm/census%s_ln_vocab.csv' % YEAR, index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/suriyant_gmail_com/.conda/envs/jupyter-env/lib/python3.8/site-packages/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
      "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000/25000 - 105s\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         api       0.85      0.58      0.69     39873\n",
      "       black       0.57      0.08      0.14     99170\n",
      "    hispanic       0.88      0.77      0.82    132008\n",
      "       white       0.79      0.97      0.87    528949\n",
      "\n",
      "    accuracy                           0.81    800000\n",
      "   macro avg       0.77      0.60      0.63    800000\n",
      "weighted avg       0.78      0.81      0.76    800000\n",
      "\n",
      "[[ 23294    135   3146  13298]\n",
      " [   890   8066    918  89296]\n",
      " [   604    390 101828  29186]\n",
      " [  2666   5496   9882 510905]]\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict_classes(X_train, verbose=2)\n",
    "target_names = list(sdf.race.astype('category').cat.categories)\n",
    "print(classification_report(np.argmax(y_train, axis=1), y_pred, target_names=target_names))\n",
    "print(confusion_matrix(np.argmax(y_train, axis=1), y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "xdf.to_csv('./census/lstm/census%s_race.csv' % YEAR, columns=[])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}