notebooks/5.2_multioutput_regressor_lastname.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c1cca555-f1c3-4f4b-b7d1-3e7e5efd3b8b",
   "metadata": {},
   "source": [
    "### Multi-Output Regressor (DL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7c9cefc6-4d54-48cf-9eac-dd98fd2f26df",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "import joblib\n",
    "\n",
    "from numpy import array, hstack, math\n",
    "from numpy.random import uniform\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.multioutput import MultiOutputRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6e262ccf-a368-4d28-99d4-5b54be3332d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 3s, sys: 4.59 s, total: 1min 8s\n",
      "Wall time: 1min 3s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "      <th>name_first</th>\n",
       "      <th>race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hessler-smith</td>\n",
       "      <td>Jason</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rogers</td>\n",
       "      <td>Renee</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bartolome</td>\n",
       "      <td>Crystal</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bailey</td>\n",
       "      <td>Donna</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>carlson</td>\n",
       "      <td>Greggory</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15455105</th>\n",
       "      <td>ballew</td>\n",
       "      <td>Christina</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15455106</th>\n",
       "      <td>watts</td>\n",
       "      <td>Mark</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15455107</th>\n",
       "      <td>mcrae</td>\n",
       "      <td>Evelyn</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15455108</th>\n",
       "      <td>ward</td>\n",
       "      <td>Stephanie</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15455109</th>\n",
       "      <td>edenfield</td>\n",
       "      <td>Marcus</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15009371 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              name_last name_first      race\n",
       "0         hessler-smith      Jason  nh_white\n",
       "1                rogers      Renee  nh_white\n",
       "2             bartolome    Crystal  nh_white\n",
       "3                bailey      Donna  nh_white\n",
       "4               carlson   Greggory  nh_white\n",
       "...                 ...        ...       ...\n",
       "15455105         ballew  Christina  nh_white\n",
       "15455106          watts       Mark  nh_white\n",
       "15455107          mcrae     Evelyn  nh_white\n",
       "15455108           ward  Stephanie  nh_white\n",
       "15455109      edenfield     Marcus  nh_white\n",
       "\n",
       "[15009371 rows x 3 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "NGRAMS = 2\n",
    "\n",
    "df = pd.read_csv(\"data/fl_reg_name_race_2022.csv.gz\", engine = \"pyarrow\")\n",
    "df.dropna(subset=['name_last'], inplace=True)\n",
    "\n",
    "# We assume unknown as missing at random\n",
    "sdf = df[df.race.isin(['unknown']) == False]\n",
    "del df\n",
    "\n",
    "# Setting consistent case for names\n",
    "sdf['name_last'] = sdf.name_last.str.strip().str.lower()\n",
    "\n",
    "sdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "36428bb7-a7a4-42ae-8418-cc3786031b14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "race\n",
       "nh_white    9446851\n",
       "hispanic    2722580\n",
       "nh_black    2086584\n",
       "other        424309\n",
       "asian        329047\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the mapping for recoding\n",
    "mapping = {'multi_racial': 'other', 'native_indian': 'other'}\n",
    "\n",
    "# Recode the 'Category' column using replace()\n",
    "sdf['race'] = sdf['race'].replace(mapping)\n",
    "sdf['race'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "dcd65d45-6369-4672-bdbb-d46370bece81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(246167, 3)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf = sdf.groupby(['name_last','race'], as_index=False)['race'].agg(['count'])\n",
    "gdf_sm = gdf[(gdf['count'] >= 5) & (gdf['name_last'].str.len() > 1)]\n",
    "gdf_sm.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c83e2591-4af2-4236-9c59-48fe0d40623f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>name_last</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "      <th>total_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a'hearn</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aabel</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aaberg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aadland</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aagaard</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202635</th>\n",
       "      <td>zysman</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202636</th>\n",
       "      <td>zytczak</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202637</th>\n",
       "      <td>zywczyk</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202638</th>\n",
       "      <td>zywica</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202639</th>\n",
       "      <td>zywicki</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>202640 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "race   name_last  asian  hispanic  nh_black  nh_white  other  total_n\n",
       "0        a'hearn    0.0       0.0       0.0       6.0    0.0      6.0\n",
       "1          aabel    0.0       0.0       0.0       7.0    0.0      7.0\n",
       "2         aaberg    0.0       0.0       0.0      13.0    0.0     13.0\n",
       "3        aadland    0.0       0.0       0.0       9.0    0.0      9.0\n",
       "4        aagaard    0.0       0.0       0.0      32.0    0.0     32.0\n",
       "...          ...    ...       ...       ...       ...    ...      ...\n",
       "202635    zysman    0.0       0.0       0.0      10.0    0.0     10.0\n",
       "202636   zytczak    0.0       0.0       0.0       5.0    0.0      5.0\n",
       "202637   zywczyk    0.0       0.0       0.0       6.0    0.0      6.0\n",
       "202638    zywica    0.0       0.0       0.0       6.0    0.0      6.0\n",
       "202639   zywicki    0.0       0.0       0.0       9.0    0.0      9.0\n",
       "\n",
       "[202640 rows x 7 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating a pivot table so that each name has a count of the # of races with that last name\n",
    "gdf_pv = gdf_sm.pivot_table(values='count', columns='race',index='name_last')\n",
    "\n",
    "# Converting NaN to zeros since that means there is no one that identifies with that race with that last name\n",
    "gdf_pv = gdf_pv.fillna(0)\n",
    "\n",
    "# Getting the totals of each last name\n",
    "gdf_pv['total_n'] = gdf_pv.sum(axis=1)\n",
    "gdf_pv.reset_index(inplace=True)\n",
    "gdf_pv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "36a14ccb-5f4f-4cac-b663-dc7a060df5af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13.7 ms, sys: 0 ns, total: 13.7 ms\n",
      "Wall time: 12.6 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>name_last</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "      <th>total_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a'hearn</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aabel</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aaberg</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aadland</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aagaard</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202635</th>\n",
       "      <td>zysman</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202636</th>\n",
       "      <td>zytczak</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202637</th>\n",
       "      <td>zywczyk</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202638</th>\n",
       "      <td>zywica</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202639</th>\n",
       "      <td>zywicki</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>202640 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "race   name_last  asian  hispanic  nh_black  nh_white  other  total_n\n",
       "0        a'hearn    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "1          aabel    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "2         aaberg    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "3        aadland    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "4        aagaard    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "...          ...    ...       ...       ...       ...    ...      ...\n",
       "202635    zysman    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "202636   zytczak    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "202637   zywczyk    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "202638    zywica    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "202639   zywicki    0.0       0.0       0.0       1.0    0.0      1.0\n",
       "\n",
       "[202640 rows x 7 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Calculate the proportion of people with a particular last name\n",
    "#  that identify with various races/ethnicity\n",
    "gdf_pv.iloc[:, 1:] = gdf_pv.iloc[:, 1:].div(gdf_pv.total_n, axis=0)\n",
    "gdf_pv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dbc56413-30d6-4923-8bff-ad230e15a31c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(202640, 7)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf_pv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3ace41ea-69b0-46be-95b2-7dd6743af6c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202635</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202636</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202637</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202638</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202639</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>202640 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "race    asian  hispanic  nh_black  nh_white  other\n",
       "0         0.0       0.0       0.0       1.0    0.0\n",
       "1         0.0       0.0       0.0       1.0    0.0\n",
       "2         0.0       0.0       0.0       1.0    0.0\n",
       "3         0.0       0.0       0.0       1.0    0.0\n",
       "4         0.0       0.0       0.0       1.0    0.0\n",
       "...       ...       ...       ...       ...    ...\n",
       "202635    0.0       0.0       0.0       1.0    0.0\n",
       "202636    0.0       0.0       0.0       1.0    0.0\n",
       "202637    0.0       0.0       0.0       1.0    0.0\n",
       "202638    0.0       0.0       0.0       1.0    0.0\n",
       "202639    0.0       0.0       0.0       1.0    0.0\n",
       "\n",
       "[202640 rows x 5 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = gdf_pv.name_last\n",
    "y = gdf_pv.iloc[:, 1:6]\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b81e93c2-ecfa-4e9c-8c8c-8e81b115aaa7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xtrain: (172244,) ytrain: (172244, 5)\n",
      "xtest: (30396,) ytest: (30396, 5)\n"
     ]
    }
   ],
   "source": [
    "xtrain, xtest, ytrain, ytest=train_test_split(X, y, test_size=0.15)\n",
    "print(\"xtrain:\", xtrain.shape, \"ytrain:\", ytrain.shape)\n",
    "print(\"xtest:\", xtest.shape, \"ytest:\", ytest.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "4f769dec-ec6b-4aeb-8b2c-1f323458576a",
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = CountVectorizer(analyzer='char', max_df=0.3, min_df=3, ngram_range=(1, NGRAMS), lowercase=False) \n",
    "\n",
    "X_combined_transformed = vect.fit_transform(X)\n",
    "\n",
    "# Transform the training data\n",
    "xtrain = X_combined_transformed[:xtrain.shape[0]]\n",
    "\n",
    "# Transform the test data\n",
    "xtest = X_combined_transformed[xtrain.shape[0]:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b43947cd-f69b-401d-a167-39aa8113ec36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [100/10000], Loss: 0.0630\n",
      "Epoch [200/10000], Loss: 0.0530\n",
      "Epoch [400/10000], Loss: 0.0471\n",
      "Epoch [700/10000], Loss: 0.0451\n",
      "Epoch [800/10000], Loss: 0.0447\n",
      "Epoch [900/10000], Loss: 0.0433\n",
      "Epoch [1000/10000], Loss: 0.0439\n",
      "Early stopping at epoch 1047\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "input_size = xtrain.toarray().shape[1]\n",
    "output_size = ytrain.shape[1]\n",
    "\n",
    "# Convert the data to PyTorch tensors\n",
    "input_data = torch.tensor(xtrain.toarray(), dtype=torch.float32)\n",
    "output_data = torch.tensor(ytrain.values, dtype=torch.float32)\n",
    "\n",
    "testx_data = torch.tensor(xtest.toarray(), dtype=torch.float32)\n",
    "testy_data = torch.tensor(ytest.values, dtype=torch.float32)\n",
    "\n",
    "# Define the neural network model\n",
    "class MultiOutputRegression(nn.Module):\n",
    "    def __init__(self, input_size, output_size):\n",
    "        super(MultiOutputRegression, self).__init__()\n",
    "        self.linear1 = nn.Linear(input_size, 128)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.linear2 = nn.Linear(128, 64)\n",
    "        self.relu2 = nn.ReLU()\n",
    "        self.linear3 = nn.Linear(64, output_size)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.linear1(x)\n",
    "        x = self.relu1(x)\n",
    "        x = self.linear2(x)\n",
    "        x = self.relu2(x)\n",
    "        x = self.linear3(x)\n",
    "        return x\n",
    "\n",
    "# Set random seed for reproducibility\n",
    "torch.manual_seed(42)\n",
    "\n",
    "# Define the model\n",
    "model = MultiOutputRegression(input_size, output_size)\n",
    "\n",
    "# Define the loss function\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "# Define the optimizer\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.01)\n",
    "\n",
    "num_epochs = 5000\n",
    "best_loss = float('inf')\n",
    "patience = 2000\n",
    "counter = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    # Forward pass\n",
    "    outputs = model(input_data)\n",
    "    loss = criterion(outputs, output_data)\n",
    "\n",
    "    # Backward and optimize\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    # Print progress\n",
    "    if (epoch+1) % 100 == 0:\n",
    "        print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.4f}')\n",
    "\n",
    "    # Calculate test loss\n",
    "    with torch.no_grad():\n",
    "        test_outputs = model(testx_data)\n",
    "        test_loss = criterion(testy_data, test_outputs)\n",
    "\n",
    "    # Check for early stopping\n",
    "    if test_loss < best_loss:\n",
    "        best_loss = test_loss\n",
    "        counter = 0\n",
    "    else:\n",
    "        counter += 1\n",
    "        if counter >= patience:\n",
    "            print(f'Early stopping at epoch {epoch+1}')\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5d3a6542-8b1a-4cf8-9462-3ee546e4220d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ytrain.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1966f58a-b00c-4283-80d5-4d0b8238dc1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = xtest.toarray()\n",
    "test_data = torch.tensor(test_data, dtype=torch.float32)\n",
    "predictions = model(test_data)\n",
    "\n",
    "# Convert predictions to numpy array\n",
    "predictions = predictions.detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "375e5a38-f93e-47de-9f5c-e8ebdb4742cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6924924332148967"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(np.argmax(ytest.values, axis = 1) == np.argmax(predictions, axis = 1)).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "ed773d41-add5-4a68-83f6-8b861a450041",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>97</td>\n",
       "      <td>0</td>\n",
       "      <td>602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>527</td>\n",
       "      <td>6</td>\n",
       "      <td>3494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>223</td>\n",
       "      <td>3</td>\n",
       "      <td>1596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3145</td>\n",
       "      <td>40</td>\n",
       "      <td>20519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0  0     1   2      3\n",
       "row_0                    \n",
       "0      0    97   0    602\n",
       "1      0   527   6   3494\n",
       "2      0   223   3   1596\n",
       "3      1  3145  40  20519\n",
       "4      0    18   0    125"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(np.argmax(ytest.values, axis = 1), np.argmax(predictions, axis = 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "7ed13ddf-2936-43eb-af7b-dffeb88e6287",
   "metadata": {},
   "outputs": [],
   "source": [
    "normed = predictions/np.sum(predictions, axis = 1).reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "014207ec-54cc-43c6-b7dc-6f19e0182860",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7798723516252138"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(np.argmax(ytest.values, axis = 1) == 3).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "54d3180e-7e9e-456f-b15e-866544bf6e81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "white normed MSE:0.3953\n"
     ]
    }
   ],
   "source": [
    "print(\"white normed MSE:%.4f\" % mean_squared_error(ytest.iloc[:,3], normed[:,3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "5ac15e37-5a3b-4565-b8ef-c65ff20f74ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "asian MSE:0.0193\n",
      "hispanic MSE:0.2830\n",
      "nh_black MSE:0.0566\n",
      "nh_white MSE:0.3943\n"
     ]
    }
   ],
   "source": [
    "print(\"asian MSE:%.4f\" % mean_squared_error(ytest.iloc[:,0], predictions[:,0]))\n",
    "print(\"hispanic MSE:%.4f\" % mean_squared_error(ytest.iloc[:,1], predictions[:,1]))\n",
    "print(\"nh_black MSE:%.4f\" % mean_squared_error(ytest.iloc[:,2], predictions[:,2]))\n",
    "print(\"nh_white MSE:%.4f\" % mean_squared_error(ytest.iloc[:,3], predictions[:,3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "eff6cffd-abbb-486b-82d3-221537e2dce4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "base asian MSE:0.0189\n",
      "base white MSE:0.1553\n"
     ]
    }
   ],
   "source": [
    "print(\"base asian MSE:%.4f\" % mean_squared_error(ytest.iloc[:,0], [np.mean(ytest.iloc[:,0]) for _ in range(30396)]))\n",
    "print(\"base white MSE:%.4f\" % mean_squared_error(ytest.iloc[:,3], [np.mean(ytest.iloc[:,3]) for _ in range(30396)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ace003a9-c392-47b6-aabd-8ff6856b0161",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model, 'v2/fl_last_name_multioutput.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaa085be-1fd8-4b5b-b6f5-6e59578eff24",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}