Data Processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preamble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Processing Application Data\n",
    "\n",
    "This is the main dataset which contains our response of interest (**TARGET**)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset Shape: (307511, 121)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"./data/application_train.csv\", index_col = \"SK_ID_CURR\")\n",
    "print(\"Dataset Shape: {}\".format(df.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Column names for feautres, response\n",
    "features = [x for x in df.columns if x != \"TARGET\"]\n",
    "response = \"TARGET\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Add an average external source feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-8c3292baed34>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'EXT_SOURCE_AVG'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'EXT_SOURCE_1'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'EXT_SOURCE_2'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"EXT_SOURCE_3\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcorr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/bbugbee/anaconda/envs/ml/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mcorr\u001b[0;34m(self, method, min_periods)\u001b[0m\n\u001b[1;32m   4724\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4725\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'pearson'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4726\u001b[0;31m             \u001b[0mcorrel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_algos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnancorr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ensure_float64\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmin_periods\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4727\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'spearman'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4728\u001b[0m             correl = _algos.nancorr_spearman(_ensure_float64(mat),\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "df['EXT_SOURCE_AVG'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', \"EXT_SOURCE_3\"]].mean(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Checking on missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_missing = len(df.index) - df.count()\n",
    "df_missing.sort_values(ascending = False)\n",
    "df_missing = df_missing.loc[lambda x: x > 100] #no more than 100 missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NAME_CONTRACT_TYPE',\n",
       " 'CODE_GENDER',\n",
       " 'FLAG_OWN_CAR',\n",
       " 'FLAG_OWN_REALTY',\n",
       " 'CNT_CHILDREN',\n",
       " 'AMT_INCOME_TOTAL',\n",
       " 'AMT_CREDIT',\n",
       " 'AMT_ANNUITY',\n",
       " 'NAME_INCOME_TYPE',\n",
       " 'NAME_EDUCATION_TYPE',\n",
       " 'NAME_FAMILY_STATUS',\n",
       " 'NAME_HOUSING_TYPE',\n",
       " 'REGION_POPULATION_RELATIVE',\n",
       " 'DAYS_BIRTH',\n",
       " 'DAYS_EMPLOYED',\n",
       " 'DAYS_REGISTRATION',\n",
       " 'DAYS_ID_PUBLISH',\n",
       " 'FLAG_MOBIL',\n",
       " 'FLAG_EMP_PHONE',\n",
       " 'FLAG_WORK_PHONE',\n",
       " 'FLAG_CONT_MOBILE',\n",
       " 'FLAG_PHONE',\n",
       " 'FLAG_EMAIL',\n",
       " 'CNT_FAM_MEMBERS',\n",
       " 'REGION_RATING_CLIENT',\n",
       " 'REGION_RATING_CLIENT_W_CITY',\n",
       " 'WEEKDAY_APPR_PROCESS_START',\n",
       " 'HOUR_APPR_PROCESS_START',\n",
       " 'REG_REGION_NOT_LIVE_REGION',\n",
       " 'REG_REGION_NOT_WORK_REGION',\n",
       " 'LIVE_REGION_NOT_WORK_REGION',\n",
       " 'REG_CITY_NOT_LIVE_CITY',\n",
       " 'REG_CITY_NOT_WORK_CITY',\n",
       " 'LIVE_CITY_NOT_WORK_CITY',\n",
       " 'ORGANIZATION_TYPE',\n",
       " 'DAYS_LAST_PHONE_CHANGE',\n",
       " 'FLAG_DOCUMENT_2',\n",
       " 'FLAG_DOCUMENT_3',\n",
       " 'FLAG_DOCUMENT_4',\n",
       " 'FLAG_DOCUMENT_5',\n",
       " 'FLAG_DOCUMENT_6',\n",
       " 'FLAG_DOCUMENT_7',\n",
       " 'FLAG_DOCUMENT_8',\n",
       " 'FLAG_DOCUMENT_9',\n",
       " 'FLAG_DOCUMENT_10',\n",
       " 'FLAG_DOCUMENT_11',\n",
       " 'FLAG_DOCUMENT_12',\n",
       " 'FLAG_DOCUMENT_13',\n",
       " 'FLAG_DOCUMENT_14',\n",
       " 'FLAG_DOCUMENT_15',\n",
       " 'FLAG_DOCUMENT_16',\n",
       " 'FLAG_DOCUMENT_17',\n",
       " 'FLAG_DOCUMENT_18',\n",
       " 'FLAG_DOCUMENT_19',\n",
       " 'FLAG_DOCUMENT_20',\n",
       " 'FLAG_DOCUMENT_21']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_features = [x for x in features if x not in list(df_missing.index)]\n",
    "full_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Process data for a given feature set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def clean_app_data(x, feat, resp):\n",
    "    \"\"\"Helper function to clean application data\"\"\"\n",
    "    cur = x[feat + [resp]]\n",
    "    cur = pd.get_dummies(cur.dropna(axis = 0))\n",
    "    return cur"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_proc = clean_app_data(df, full_features, response)\n",
    "print(\"Processed Shape: {}\".format(df_proc.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At this point all of data is numerical and all missing data has been dropped."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train - Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def split_data(df, resp, train_prob):\n",
    "    \"\"\"Helper function to get a train-test split for processed data\"\"\"\n",
    "    \n",
    "    feat = [x for x in df.columns if x != resp]\n",
    "    X = np.array(df[feat])\n",
    "    y = np.array(df[resp])\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size = train_prob)\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = split_data(df_proc, response, train_prob = 0.8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df[[\"EXT_SOURCE_1\", \"EXT_SOURCE_2\", \"EXT_SOURCE_3\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}