Neural Network for Tabular Data with Categorical Variables.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making Prediction for Tabular Data with Categorical Variable using FastAI\n",
    "\n",
    "Publication at arXiv:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.tabular import *  # Quick accesss to tabular functionality\n",
    "from fastai.utils.mem import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE = '/home/odin/Data Science/Temp/PTP/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eq = pd.read_csv(BASE+'df_EQ_dataset.csv', encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eq.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eq['billPaid'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eq['billPaid'].value_counts()/len(df_eq)*100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split Dataset into Train and Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_eq.iloc[0 : int(len(df_eq)*0.8)]\n",
    "df_test = df_eq.iloc[int(len(df_eq)*0.8):]\n",
    "del df_eq\n",
    "len(df_train), len(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_train)/(len(df_train) + len(df_test)), len(df_test)/(len(df_train) + len(df_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a function for under sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def undersample(df=None):\n",
    "    df_0 = df[df.billPaid==0]\n",
    "    df_1 = df[df.billPaid==1]\n",
    "    if len(df_0) < len(df_1):\n",
    "        df_1 = df_1.sample(len(df_0), replace=False)\n",
    "    elif len(df_0) > len(df_1):\n",
    "        df_0 = df_0.sample(len(df_1), replace=False)\n",
    "    df = pd.concat([df_0, df_1])\n",
    "    del df_0, df_1\n",
    "    df = df.sample(frac=1).reset_index(drop=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_train = undersample(df=df_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a function for oversampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def oversample(df=None):\n",
    "    df_0 = df[df.billPaid==0]\n",
    "    df_1 = df[df.billPaid==1]\n",
    "    if len(df_0) > len(df_1):\n",
    "        df_1 = df_1.sample(len(df_0), replace=True)\n",
    "    elif len(df_0) < len(df_1):\n",
    "        df_0 = df_0.sample(len(df_1), replace=True)\n",
    "    df = pd.concat([df_0, df_1])\n",
    "    del df_0, df_1\n",
    "    df = df.sample(frac=1).reset_index(drop=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = oversample(df=df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['billPaid'].value_counts()/len(df_train)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['billPaid'].value_counts()/len(df_test)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decide categorical and continuous variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep_var = 'billPaid'\n",
    "cat_names = ['billOrder', 'billType', 'dueDayW', 'dueMonth', 'billRoute', 'hasMailAddress', 'segment', \n",
    "             'dueDayM', 'dueMonth', 'dueDayW', 'billDayM', 'billMonth', 'billRoute', 'numAccountHolders',\n",
    "             'hasMailAddress', 'city', 'postcode', 'incomeGroup', 'wealthGroup', 'segment', 'RA_CODE_2016',\n",
    "            'AverageHhdSize', 'MB_CODE_2016', 'SA1_7DIGITCODE_2016', 'medianPersonPerBedroom']\n",
    "cont_names = ['dueYear', 'billDuration', 'minDOB', 'medianHhdIncWkly', 'medianMortgageWkly', 'medianRentWkly',\n",
    "             'setupYear']\n",
    "procs = [FillMissing, Categorify, Normalize]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check which variables are excluded\n",
    "set(df_train.columns) - set(cat_names) - set(cont_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if any variable has been added mistakenly to both categorical and continuous variables\n",
    "set(cat_names).intersection(set(cont_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#get the GPU ID of the GPU with max free ram\n",
    "gpu_with_max_free_mem()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Optimise memory space of dataframe\n",
    "reduce_mem_usage(df_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test1 = df_test.copy(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Optimise memory space of dataframe\n",
    "reduce_mem_usage(df_test);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creat data loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creat data loader for test data\n",
    "#Test data from len(df)*0.8 index to len(df)\n",
    "test_data = TabularList.from_df(df_test, cat_names=cat_names, cont_names=cont_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creat data loader for train data \n",
    "# 10% of the training data is used for validation\n",
    "data = (TabularList.from_df(df_train, cat_names=cat_names, cont_names=cont_names, procs=procs)\n",
    "                           .split_by_idx(list(range(int(len(df_train)*0.9),len(df_train))))\n",
    "                           .label_from_df(cols=dep_var)\n",
    "                           .add_test(test_data)\n",
    "                           .databunch())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data.train_ds)/len(df_train)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data.valid_ds)/len(df_train)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data.test_ds)/(len(df_train)+len(df_test))*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.show_batch(rows=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "learn = tabular_learner(data, layers=[512,1024, 256,64], metrics=accuracy)\n",
    "learn.fit(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "learn.fit(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "#learn.fit(5, 1e-2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df_test1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "row = df_test.sample().iloc[0]\n",
    "learn.predict(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df_test.reset_index(drop=True)\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = df_test[dep_var]\n",
    "y_test = list(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "limit = len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_names = list(set(cat_names).union(set(cont_names)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "predicted = learn.predict(df_test[all_names].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#limit = 1000\n",
    "predicted = [learn.predict(df_test.iloc[idx]) for idx in df_test.index[:limit]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted = [p[0].obj for p in predicted]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Measure Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import cohen_kappa_score\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "import numpy as np\n",
    "\n",
    "predicted = np.array(predicted)\n",
    "actual = np.array(y_test[:limit])\n",
    "\n",
    "tp = np.count_nonzero(predicted * actual)\n",
    "tn = np.count_nonzero((predicted - 1) * (actual - 1))\n",
    "fp = np.count_nonzero(predicted * (actual - 1))\n",
    "fn = np.count_nonzero((predicted - 1) * actual)\n",
    "\n",
    "print('True Positive\\t' + str(tp))\n",
    "print('True Negative\\t' + str(tn))\n",
    "print('False Positive\\t' + str(fp))\n",
    "print('False Negative\\t' + str(fn))\n",
    "\n",
    "accuracy = (tp + tn) / (tp + fp + fn + tn)\n",
    "precision = tp / (tp + fp)\n",
    "recall = tp / (tp + fn)\n",
    "fmeasure = (2 * precision * recall) / (precision + recall)\n",
    "cohen_kappa_score = cohen_kappa_score(predicted, actual)\n",
    "false_positive_rate, true_positive_rate, thresholds = roc_curve(actual, predicted)\n",
    "auc_val = auc(false_positive_rate, true_positive_rate)\n",
    "roc_auc_val = roc_auc_score(actual, predicted)\n",
    "\n",
    "print('Accuracy\\t' + str(accuracy))\n",
    "print('Precision\\t' + str(precision))\n",
    "print('Recall\\t' + str(recall))\n",
    "print('f-measure\\t' + str(fmeasure))\n",
    "print('cohen_kappa_score\\t' + str(cohen_kappa_score))\n",
    "print('auc\\t' + str(auc_val))\n",
    "print('roc_auc\\t' + str(roc_auc_val))\n",
    "\n",
    "#print(\"Average of ROC-AUC score: %.3f\" % roc_auc_score(ytest, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Performance test\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(actual, predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'DNN'\n",
    "strategy = 'upsamping'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "from sklearn.metrics import classification_report\n",
    "now = datetime.datetime.now()\n",
    "\n",
    "out_string = '=========='+str(now)+'==============\\n'\n",
    "out_string += 'Strategy:\\t' + strategy + '\\n'\n",
    "out_string += str('Model Name:\\t' + model_name+'\\n')\n",
    "out_string += '-------------------------------------------------' + '\\n'\n",
    "\n",
    "out_string += 'Total Samples:\\t' + str(len(actual)) + '\\n'\n",
    "out_string += 'Positive Samples:\\t' + str(sum(actual)) + '\\n'\n",
    "out_string += 'Negative Samples:\\t' + str(len(actual)-sum(actual)) + '\\n'\n",
    "\n",
    "out_string += 'True Positive:\\t' + str(tp) + '\\n'\n",
    "out_string += 'True Negative:\\t' + str(tn) + '\\n'\n",
    "out_string += 'False Positive:\\t' + str(fp) + '\\n'\n",
    "out_string += 'False Negative:\\t' + str(fn) + '\\n'\n",
    "\n",
    "out_string += 'Accuracy:\\t' + str(accuracy) + '\\n'\n",
    "out_string += 'Precision:\\t' + str(precision) + '\\n'\n",
    "out_string += 'Recall:\\t' + str(recall) + '\\n'\n",
    "out_string += 'F-measure:\\t' + str(fmeasure) + '\\n'\n",
    "out_string += 'Cohen_Kappa_Score:\\t' + str(cohen_kappa_score) + '\\n'\n",
    "out_string += 'AUC:\\t' + str(auc_val) + '\\n'\n",
    "out_string += 'ROC_AUC:\\t' + str(roc_auc_val) + '\\n'\n",
    "out_string += '\\n'\n",
    "out_string += classification_report(actual, predicted)\n",
    "out_string += '\\n'\n",
    "print(out_string)\n",
    "with open(model_name+'_'+strategy+'_POP.txt', 'a+') as FO:\n",
    "    FO.write(out_string)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}