09 Neural Networks (Regression).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img align=\"center\" src=\"http://sydney.edu.au/images/content/about/logo-mono.jpg\">\n",
    "<h1 align=\"center\" style=\"margin-top:10px\">Statistical Learning with Python</h1>\n",
    "<h2 align=\"center\" style=\"margin-top:20px\">Lecture 9: Neural Networks (Regression)</h2>\n",
    "<br>\n",
    "\n",
    "<a href=\"#1.-Credit-Card-Data\">Credit Card Data</a> <br>\n",
    "<a href=\"#2.-Single-Layer-Perceptron\">Single Layer Perceptron</a> <br>\n",
    "<a href=\"#3.-Scikit-Learn-Wrapper\">Scikit-Learn Wrapper</a> <br>\n",
    "<a href=\"#4.-Model-Evaluation\">Model Evaluation</a> <br>\n",
    "\n",
    "This notebook relies on the following libraries and settings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot settings\n",
    "sns.set_context('notebook') # optimises figures for notebook display\n",
    "sns.set_style('ticks') # set default plot style\n",
    "crayon = ['#4E79A7','#F28E2C','#E15759','#76B7B2','#59A14F', \n",
    "          '#EDC949','#AF7AA1','#FF9DA7','#9C755F','#BAB0AB']\n",
    "sns.set_palette(crayon) # set custom color scheme\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (9, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error, r2_score,  mean_absolute_error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Credit Card Data\n",
    "\n",
    "We again use the `Credit` dataset. Here we simply repeat the steps from the previous notebook to load and process the data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Income</th>\n",
       "      <th>Limit</th>\n",
       "      <th>Cards</th>\n",
       "      <th>Age</th>\n",
       "      <th>Education</th>\n",
       "      <th>Student</th>\n",
       "      <th>Married</th>\n",
       "      <th>Balance</th>\n",
       "      <th>Male</th>\n",
       "      <th>Caucasian</th>\n",
       "      <th>Asian</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Obs</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>18.701</td>\n",
       "      <td>5524</td>\n",
       "      <td>5</td>\n",
       "      <td>64</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>966</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>14.090</td>\n",
       "      <td>4323</td>\n",
       "      <td>5</td>\n",
       "      <td>25</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>671</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>54.319</td>\n",
       "      <td>3063</td>\n",
       "      <td>3</td>\n",
       "      <td>59</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>269</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>67.937</td>\n",
       "      <td>5184</td>\n",
       "      <td>4</td>\n",
       "      <td>63</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>345</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>23.793</td>\n",
       "      <td>3821</td>\n",
       "      <td>4</td>\n",
       "      <td>56</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>868</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381</th>\n",
       "      <td>115.123</td>\n",
       "      <td>7760</td>\n",
       "      <td>3</td>\n",
       "      <td>83</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>661</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>53.566</td>\n",
       "      <td>5891</td>\n",
       "      <td>4</td>\n",
       "      <td>82</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>712</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>17.700</td>\n",
       "      <td>2860</td>\n",
       "      <td>4</td>\n",
       "      <td>63</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>89</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>28.508</td>\n",
       "      <td>3933</td>\n",
       "      <td>4</td>\n",
       "      <td>56</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>336</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>41.400</td>\n",
       "      <td>2561</td>\n",
       "      <td>2</td>\n",
       "      <td>36</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Income  Limit  Cards  Age  Education  Student  Married  Balance  Male  \\\n",
       "Obs                                                                           \n",
       "400   18.701   5524      5   64          7        0        0      966     0   \n",
       "26    14.090   4323      5   25         16        0        1      671     0   \n",
       "280   54.319   3063      3   59          8        1        0      269     0   \n",
       "261   67.937   5184      4   63         12        0        1      345     1   \n",
       "131   23.793   3821      4   56         12        1        1      868     0   \n",
       "381  115.123   7760      3   83         14        0        0      661     0   \n",
       "361   53.566   5891      4   82         10        0        0      712     0   \n",
       "21    17.700   2860      4   63         16        0        0       89     0   \n",
       "193   28.508   3933      4   56         14        0        1      336     1   \n",
       "259   41.400   2561      2   36         14        0        1        0     1   \n",
       "\n",
       "     Caucasian  Asian  \n",
       "Obs                    \n",
       "400          0      1  \n",
       "26           0      0  \n",
       "280          1      0  \n",
       "261          0      1  \n",
       "131          0      0  \n",
       "381          0      0  \n",
       "361          1      0  \n",
       "21           0      1  \n",
       "193          0      1  \n",
       "259          1      0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We will always assume that the data file is in a subdirectory called \"Data\"\n",
    "train=pd.read_hdf('Data/Credit.h5', 'train')\n",
    "test=pd.read_hdf('Data/Credit.h5', 'test')\n",
    "train.head(10) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We consider two predictors, the credit card limit and income. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Response label\n",
    "response = 'Balance'\n",
    "\n",
    "# Creates a list with the names of all variables which are not the respose\n",
    "predictors = [x for x in train.columns if x!= response]\n",
    "\n",
    "# Here we convert the data to NumPy arrays. This is not strictly necessary, but generally better.\n",
    "y_train = train[response].to_numpy() \n",
    "X_train = train[predictors].to_numpy() \n",
    "\n",
    "y_test = test[response].to_numpy()\n",
    "X_test = test[predictors].to_numpy() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test  = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Single Layer Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\marce\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\marce\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1680328af28>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "\n",
    "slp = Sequential()\n",
    "slp.add(Dense(24, input_dim=X_train.shape[1], activation='relu'))\n",
    "slp.add(Dense(1))\n",
    "slp.compile(loss='mse', optimizer='rmsprop')\n",
    "slp.fit(X_train, y_train, epochs=10000, verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Scikit-Learn Wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-480320.19213479, -394190.32693342, -470625.01228574,\n",
       "       -505443.28002781, -420750.55544441])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "\n",
    "def build_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(24, input_dim=X_train.shape[1], activation='relu'))\n",
    "    model.add(Dense(1))\n",
    "    model.compile(loss='mse', optimizer='rmsprop')\n",
    "    return model\n",
    "\n",
    "estimator = KerasRegressor(build_fn=build_model, epochs=100, verbose=0)\n",
    "cross_val_score(estimator, X_train, y_train, cv=5, scoring = 'neg_mean_squared_error')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Model Evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RMSE</th>\n",
       "      <th>R-Squared</th>\n",
       "      <th>MAE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Linear Regression</th>\n",
       "      <td>97.19</td>\n",
       "      <td>0.96</td>\n",
       "      <td>80.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Single Layer Perceptron</th>\n",
       "      <td>13.27</td>\n",
       "      <td>1.00</td>\n",
       "      <td>9.73</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          RMSE  R-Squared    MAE\n",
       "Linear Regression        97.19       0.96  80.03\n",
       "Single Layer Perceptron  13.27       1.00   9.73"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Benchmark\n",
    "ols = LinearRegression()\n",
    "ols.fit(X_train, y_train)\n",
    "\n",
    "# Initialise table\n",
    "columns=['RMSE', 'R-Squared', 'MAE']\n",
    "rows=['Linear Regression', 'Single Layer Perceptron']\n",
    "results =pd.DataFrame(0.0, columns=columns, index=rows)\n",
    "\n",
    "methods = [ols, slp] \n",
    "\n",
    "for i, method in enumerate(methods):    \n",
    "    y_pred = method.predict(X_test)\n",
    "    results.iloc[i, 0] = np.sqrt(mean_squared_error(y_test, y_pred))\n",
    "    results.iloc[i, 1] = r2_score(y_test, y_pred)\n",
    "    results.iloc[i, 2] = mean_absolute_error(y_test, y_pred) \n",
    "\n",
    "results.round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Additional Code\n",
    "\n",
    "The two cells below format the notebook for display online. Please omit them from your work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "@import url('https://fonts.googleapis.com/css?family=Source+Sans+Pro|Open+Sans:800&display=swap');\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<style>\n",
    "@import url('https://fonts.googleapis.com/css?family=Source+Sans+Pro|Open+Sans:800&display=swap');\n",
    "</style>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.CodeMirror pre {font-family: monospace; font-size: 12pt; line-height: 22.4px;}\n",
       "\n",
       "pre {font-family: monospace; font-size: 12pt; line-height: 22.4px;}\n",
       "\n",
       ".output_area pre {font-family: monospace; font-size: 12pt; line-height: 22.4px;}\n",
       "\n",
       ".rendered_html table {font-size: 10.5pt;}\n",
       "\n",
       "h1 {\n",
       "  font-family: 'Open Sans' !important;\n",
       "  color: #222;\n",
       "  font-size: 2.5em !important;\n",
       "  padding-bottom: 0px;  \n",
       "  margin-top: 14px;\n",
       "  margin-bottom: 6px;\n",
       "  font-weight: 800 !important;\n",
       "}\n",
       "\n",
       "\n",
       "h2 {\n",
       "  font-family: 'Open Sans' !important;\n",
       "  font-size: 1.6em !important;\n",
       "  font-weight: 800 !important;\n",
       "  color: #222;\n",
       "}\n",
       "\n",
       "\n",
       "h3 {\n",
       "  font-family: 'Open Sans' !important;\n",
       "  font-size: 25px !important;\n",
       "  color: #222 !important;\n",
       "  margin-bottom: 30px !important;\n",
       "  margin-top: 25px !important;\n",
       "}\n",
       "\n",
       "\n",
       "p {\n",
       "  font-family: 'Source Sans Pro', sans-serif;\n",
       "  color: #222;\n",
       "  font-size: 19px;\n",
       "  margin-bottom: 10px;\n",
       "  margin-top:0;\n",
       "  line-height: 1.4;\n",
       "}\n",
       "\n",
       "li {\n",
       "  font-family: 'Source Sans Pro', sans-serif;\n",
       "  color: #222;\n",
       "  font-size: 19px;\n",
       "  line-height: 1.4;\n",
       "}\n",
       "\n",
       "\n",
       "\n",
       "/*code {\n",
       "     padding: 2px 4px !important; \n",
       "     font-size: 90% !important;\n",
       "     color: #c7254e !important; \n",
       "     background-color: #f9f2f4 !important; \n",
       "     border-radius: 2px !important;\n",
       "     text-decoration: none !important;\n",
       "      font-size: 90% !important;\n",
       "}\n",
       "*/\n",
       "\n",
       "\n",
       "code {\n",
       "      font-size: 90% !important;\n",
       "/*      border-style: solid !important;\n",
       "      border-width: 1px !important;\n",
       "      border-color :#cccccc !important;*/\n",
       "}\n",
       "\n",
       "TT {\n",
       "      font-family: monospace;\n",
       "      font-size: 95% !important;\n",
       "}\n",
       "\n",
       "\n",
       "\n",
       "a.neutral {\n",
       "  text-decoration: none !important;\n",
       "  border-style: dotted !important;\n",
       "  border-width: 1px !important;\n",
       "  border-color :#cccccc !important;\n",
       "\n",
       "}\n",
       "\n",
       "</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.core.display import HTML\n",
    "style = open('jstyle.css', \"r\").read()\n",
    "HTML('<style>'+ style +'</style>')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}