ZScore-Univariate feature selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "df = pd.read_csv(\"Telecom_customer churn (100000).csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_df = df.select_dtypes(include=['float64', 'int64']).copy()\n",
    "cat_df = df.select_dtypes(include=['object']).copy()\n",
    "\n",
    "# Categorical boolean mask\n",
    "categorical_feature_mask = cat_df.dtypes==object\n",
    "# filter categorical columns using mask and turn it into a list\n",
    "categorical_cols = cat_df.columns[categorical_feature_mask].tolist()\n",
    "\n",
    "import numpy as np\n",
    "#conData=np.log(0.00001 + 1)\n",
    "conData=0\n",
    "cat_df=cat_df.fillna(conData)\n",
    "num_df=num_df.fillna(conData)\n",
    "cat_df=cat_df.astype(str)\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "cat_df[categorical_cols] = cat_df[categorical_cols].apply(lambda col: le.fit_transform(col))\n",
    "\n",
    "#cat_df[categorical_cols].head(10)\n",
    "\n",
    "change_mou=num_df['change_mou']\n",
    "change_rev=num_df['change_rev']\n",
    "num_df=num_df.drop(['change_mou'], axis=1)\n",
    "num_df=num_df.drop(['change_rev'], axis=1)\n",
    "\n",
    "num_df=num_df.drop(['Customer_ID'], axis=1)\n",
    "\n",
    "churn=num_df['churn']\n",
    "num_df=num_df.drop(['churn'], axis=1)\n",
    "\n",
    "#BOX-COX\n",
    "#lemda=0.5\n",
    "#num_df=(num_df**lemda)\n",
    "#num_df=num_df-1\n",
    "#num_df[num_df < 0]=0\n",
    "#num_df=num_df.div(lemda)\n",
    "#End BOX-COX\n",
    "#Z-score\n",
    "num_df=(num_df-num_df.min())/(num_df.std(ddof=0)) ##Z-score\n",
    "\n",
    "#Log\n",
    "#num_df=round(np.log(num_df.add(1)),2)\n",
    "#num_df=num_df.replace([np.inf, -np.inf], np.nan)\n",
    "#End Log\n",
    "\n",
    "num_df=num_df.fillna(conData)\n",
    "\n",
    "result_df = pd.concat([num_df, cat_df], axis=1)\n",
    "np.nan_to_num(result_df)\n",
    "\n",
    "result_df[result_df < 0]=0\n",
    "\n",
    "result_df_op=result_df\n",
    "#result_df_op=result_df_op.drop(['churn'], axis=1)\n",
    "#result_df_op=np.nan_to_num(result_df_op)\n",
    "\n",
    "X=result_df_op\n",
    "#y=result_df['churn'] \n",
    "y=churn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 96)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature</th>\n",
       "      <th>Scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>crclscod</td>\n",
       "      <td>1803.718570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>eqpdays</td>\n",
       "      <td>820.785513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>hnd_price</td>\n",
       "      <td>583.727793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>asl_flag</td>\n",
       "      <td>415.391673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>custcare_Mean</td>\n",
       "      <td>393.119398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ccrndmou_Mean</td>\n",
       "      <td>392.580959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>threeway_Mean</td>\n",
       "      <td>360.499420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>cc_mou_Mean</td>\n",
       "      <td>355.525978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mou_Mean</td>\n",
       "      <td>347.100604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>mou_opkv_Mean</td>\n",
       "      <td>343.302661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>iwylis_vce_Mean</td>\n",
       "      <td>341.341551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>mou_rvce_Mean</td>\n",
       "      <td>323.898093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>opk_vce_Mean</td>\n",
       "      <td>317.567655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>mou_cvce_Mean</td>\n",
       "      <td>314.400780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>uniqsubs</td>\n",
       "      <td>312.015740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>complete_Mean</td>\n",
       "      <td>291.940049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>callwait_Mean</td>\n",
       "      <td>290.113452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>comp_vce_Mean</td>\n",
       "      <td>289.650584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>mouiwylisv_Mean</td>\n",
       "      <td>279.234403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>recv_vce_Mean</td>\n",
       "      <td>275.174813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>peak_vce_Mean</td>\n",
       "      <td>271.971061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>mou_peav_Mean</td>\n",
       "      <td>266.754388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>attempt_Mean</td>\n",
       "      <td>257.519286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>plcd_vce_Mean</td>\n",
       "      <td>255.066116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>avg3mou</td>\n",
       "      <td>252.092800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>owylis_vce_Mean</td>\n",
       "      <td>246.004022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>hnd_webcap</td>\n",
       "      <td>226.846793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>inonemin_Mean</td>\n",
       "      <td>223.335612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>lor</td>\n",
       "      <td>219.722044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>mouowylisv_Mean</td>\n",
       "      <td>209.651064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>totmou</td>\n",
       "      <td>32.552726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>callfwdv_Mean</td>\n",
       "      <td>23.653374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>blck_vce_Mean</td>\n",
       "      <td>21.597130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>actvsubs</td>\n",
       "      <td>18.693969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>infobase</td>\n",
       "      <td>16.703458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>income</td>\n",
       "      <td>15.974516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>numbcars</td>\n",
       "      <td>12.956686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>avg3rev</td>\n",
       "      <td>12.946961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>dwllsize</td>\n",
       "      <td>12.391973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>marital</td>\n",
       "      <td>11.766214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>ownrent</td>\n",
       "      <td>11.640799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rev_Mean</td>\n",
       "      <td>11.297862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>refurb_new</td>\n",
       "      <td>10.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>forgntvl</td>\n",
       "      <td>9.347267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>avg6rev</td>\n",
       "      <td>7.630345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>prizm_social_one</td>\n",
       "      <td>7.534467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>avgrev</td>\n",
       "      <td>4.854092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>truck</td>\n",
       "      <td>2.973973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>recv_sms_Mean</td>\n",
       "      <td>2.944229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>creditcd</td>\n",
       "      <td>2.698027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>datovr_Mean</td>\n",
       "      <td>2.353771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>totrev</td>\n",
       "      <td>0.755046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>adjrev</td>\n",
       "      <td>0.737768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>kid0_2</td>\n",
       "      <td>0.610982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>new_cell</td>\n",
       "      <td>0.517556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>kid16_17</td>\n",
       "      <td>0.333963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>kid3_5</td>\n",
       "      <td>0.248540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>rv</td>\n",
       "      <td>0.147649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>kid11_15</td>\n",
       "      <td>0.010772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>kid6_10</td>\n",
       "      <td>0.000403</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>96 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Feature       Scores\n",
       "76          crclscod  1803.718570\n",
       "74           eqpdays   820.785513\n",
       "64         hnd_price   583.727793\n",
       "77          asl_flag   415.391673\n",
       "21     custcare_Mean   393.119398\n",
       "22     ccrndmou_Mean   392.580959\n",
       "25     threeway_Mean   360.499420\n",
       "23       cc_mou_Mean   355.525978\n",
       "1           mou_Mean   347.100604\n",
       "39     mou_opkv_Mean   343.302661\n",
       "31   iwylis_vce_Mean   341.341551\n",
       "28     mou_rvce_Mean   323.898093\n",
       "37      opk_vce_Mean   317.567655\n",
       "26     mou_cvce_Mean   314.400780\n",
       "47          uniqsubs   312.015740\n",
       "43     complete_Mean   291.940049\n",
       "45     callwait_Mean   290.113452\n",
       "19     comp_vce_Mean   289.650584\n",
       "32   mouiwylisv_Mean   279.234403\n",
       "17     recv_vce_Mean   275.174813\n",
       "33     peak_vce_Mean   271.971061\n",
       "35     mou_peav_Mean   266.754388\n",
       "42      attempt_Mean   257.519286\n",
       "15     plcd_vce_Mean   255.066116\n",
       "58           avg3mou   252.092800\n",
       "29   owylis_vce_Mean   246.004022\n",
       "82        hnd_webcap   226.846793\n",
       "24     inonemin_Mean   223.335612\n",
       "69               lor   219.722044\n",
       "30   mouowylisv_Mean   209.651064\n",
       "..               ...          ...\n",
       "50            totmou    32.552726\n",
       "44     callfwdv_Mean    23.653374\n",
       "11     blck_vce_Mean    21.597130\n",
       "48          actvsubs    18.693969\n",
       "86          infobase    16.703458\n",
       "71            income    15.974516\n",
       "72          numbcars    12.956686\n",
       "60           avg3rev    12.946961\n",
       "88          dwllsize    12.391973\n",
       "85           marital    11.766214\n",
       "83           ownrent    11.640799\n",
       "0           rev_Mean    11.297862\n",
       "81        refurb_new    10.910400\n",
       "73          forgntvl     9.347267\n",
       "63           avg6rev     7.630345\n",
       "78  prizm_social_one     7.534467\n",
       "55            avgrev     4.854092\n",
       "67             truck     2.973973\n",
       "18     recv_sms_Mean     2.944229\n",
       "95          creditcd     2.698027\n",
       "7        datovr_Mean     2.353771\n",
       "51            totrev     0.755046\n",
       "52            adjrev     0.737768\n",
       "90            kid0_2     0.610982\n",
       "75          new_cell     0.517556\n",
       "94          kid16_17     0.333963\n",
       "91            kid3_5     0.248540\n",
       "68                rv     0.147649\n",
       "93          kid11_15     0.010772\n",
       "92           kid6_10     0.000403\n",
       "\n",
       "[96 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train=X\n",
    "y_train=y\n",
    "#Univariate Selection\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "select_feature = SelectKBest(chi2, k=80).fit(X_train, y_train)\n",
    "selected_features_df = pd.DataFrame({'Feature':list(X_train.columns),\n",
    "                                     'Scores':select_feature.scores_})\n",
    "selected_features_df.sort_values(by='Scores', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 80)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train=y\n",
    "X_train=X\n",
    "x_train_chi = select_feature.transform(X)\n",
    "x_train_chi.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8948 41490 7098 42464\n",
      "pod:  0.8567854404584158\n",
      "pof:  0.8225940758951584\n",
      "AUC:  0.5170956822816287\n"
     ]
    }
   ],
   "source": [
    "#Import Gaussian Naive Bayes model\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "#Create a Gaussian Classifier\n",
    "classifier = GaussianNB()\n",
    "\n",
    "classifier.fit(X,y_train)\n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = X, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 15 folds for each of 100 candidates, totalling 1500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:   15.2s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   50.3s\n",
      "[Parallel(n_jobs=-1)]: Done 434 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 784 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1234 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1500 out of 1500 | elapsed:  5.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'var_smoothing': 1.0}\n",
      "GaussianNB(priors=None, var_smoothing=1.0)\n",
      "0.5204933333333334\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import RepeatedKFold, GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "cv_method = RepeatedKFold(n_splits=5, \n",
    "                          n_repeats=3, \n",
    "                          random_state=999)\n",
    "\n",
    "classifier=GaussianNB();\n",
    "#params = {\n",
    "#          \"priors\" : \"None\",\n",
    "#          \"var_smoothing\" : 1e-9\n",
    "#}\n",
    "#create an list of var_smoothing to cross validate\n",
    "#steps = [1e-8, 1e-7, 1e-6, 1e-5, 1e-4]\n",
    "\n",
    "params_NB = {'var_smoothing': np.logspace(0,-9, num=100)}\n",
    "\n",
    "grid = GridSearchCV(estimator=classifier, \n",
    "                     param_grid=params_NB, \n",
    "                     cv=cv_method,\n",
    "                     verbose=1, \n",
    "                     scoring='accuracy', n_jobs=-1)\n",
    "\n",
    "grid.fit(x_train_chi, y_train)\n",
    "\n",
    " \n",
    "print(grid.best_params_)\n",
    "print(grid.best_estimator_)\n",
    "print(grid.best_score_)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26710 23728 21590 27972\n",
      "pod:  0.5643840038739357\n",
      "pof:  0.4704389547563345\n",
      "AUC:  0.5469725245588006\n"
     ]
    }
   ],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10386 40052 7995 41567\n",
      "pod:  0.8386868972196441\n",
      "pof:  0.794083825686982\n",
      "AUC:  0.522301535766331\n",
      "accuracy:  0.51953\n"
     ]
    }
   ],
   "source": [
    "#Import Gaussian Naive Bayes model\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import RepeatedKFold, GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "#Create a Gaussian Classifier\n",
    "classifier = GaussianNB(priors=None, var_smoothing=1.0)\n",
    "\n",
    "classifier.fit(x_train_chi,y_train)\n",
    "\n",
    "\n",
    "cv_method = RepeatedKFold(n_splits=10, \n",
    "                          n_repeats=3, \n",
    "                          random_state=999)\n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = x_train_chi, y = y_train, cv = 5)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "print ('accuracy: ',accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.54812\n"
     ]
    }
   ],
   "source": [
    " #accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "accuracy=(26836+27976)/(27976+22462+22726+26836)\n",
    "print ('accuracy: ',accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 36 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed: 555.6min\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed: 3133.4min\n",
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed: 3152.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 8, 'multi_class': 'ovr', 'penalty': 'l2', 'solver': 'liblinear'}\n",
      "LogisticRegression(C=8, class_weight=None, dual=False, fit_intercept=True,\n",
      "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
      "                   multi_class='ovr', n_jobs=None, penalty='l2',\n",
      "                   random_state=None, solver='liblinear', tol=0.0001, verbose=0,\n",
      "                   warm_start=False)\n",
      "0.57708\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# Specify parameters\n",
    "c_values = list(np.arange(1, 10))\n",
    "\n",
    "\n",
    "param_grid = [\n",
    "    {'C': c_values, 'penalty': ['l1'], 'solver' : ['liblinear'], 'multi_class' : ['ovr']},    \n",
    "    {'C': c_values, 'penalty': ['l2'], 'solver' : ['liblinear', 'newton-cg', 'lbfgs'], 'multi_class' : ['ovr']}\n",
    "]\n",
    " \n",
    "\n",
    "grid = GridSearchCV(LogisticRegression(), param_grid, verbose=1, cv=10, n_jobs=-1)\n",
    "grid.fit(x_train_chi, y_train)\n",
    "\n",
    "print(grid.best_params_)\n",
    "print(grid.best_estimator_)\n",
    "print(grid.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29385 21053 21016 28546\n",
      "pod:  0.5759654574068843\n",
      "pof:  0.4174035449462707\n",
      "AUC:  0.5792809562303068\n",
      "accuracy:  0.57931\n"
     ]
    }
   ],
   "source": [
    "#LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import metrics\n",
    "classifier = LogisticRegression(C=8, class_weight=None, dual=False, fit_intercept=True,\n",
    "                   intercept_scaling=1, max_iter=100,\n",
    "                   multi_class='ovr', n_jobs=None, penalty='l2',\n",
    "                   random_state=None, solver='liblinear', tol=0.0001, verbose=0,\n",
    "                   warm_start=False)\n",
    "classifier.fit(x_train_chi, y_train) \n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = x_train_chi, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "print ('accuracy: ',accuracy)\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  48 out of  48 | elapsed: 88.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'metric': 'manhattan', 'n_neighbors': 19, 'weights': 'uniform'}\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='manhattan',\n",
      "                     metric_params=None, n_jobs=None, n_neighbors=19, p=2,\n",
      "                     weights='uniform')\n",
      "0.54095\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier  \n",
    "\n",
    "# Specify parameters\n",
    "c_values = list(np.arange(1, 10))\n",
    "param_grid ={\n",
    "   'n_neighbors': [3,5,11,19],\n",
    "   'weights': ['uniform', 'distance'],\n",
    "   'metric': ['euclidean', 'manhattan']\n",
    "}\n",
    "\n",
    "grid = GridSearchCV(KNeighborsClassifier(), param_grid, verbose=1, cv=3, n_jobs=-1)\n",
    "grid.fit(x_train_chi, y_train)\n",
    "\n",
    "print(grid.best_params_)\n",
    "print(grid.best_estimator_)\n",
    "print(grid.best_score_)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 42 candidates, totalling 420 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed: 410.7min\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed: 1993.6min\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn import svm\n",
    "\n",
    "# Specify parameters\n",
    "\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "gammas = [0.001, 0.01, 0.1, 1, 1e-3, 1e-4]\n",
    "param_grid = {'C': Cs, 'gamma' : gammas}\n",
    "\n",
    "tuned_parameters = [{'kernel': ['rbf', 'linear'], \n",
    "                     'gamma': [0.001, 0.01, 0.1, 1, 1e-3, 1e-4], \n",
    "                     'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]} ]\n",
    "\n",
    "grid = GridSearchCV(svm.SVC(), param_grid, verbose=1, cv=10, n_jobs=-1)\n",
    "grid.fit(x_train_chi, y_train)\n",
    "\n",
    "print(grid.best_params_)\n",
    "print(grid.best_estimator_)\n",
    "print(grid.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29029 21409 24489 25073\n",
      "pod:  0.5058916105080505\n",
      "pof:  0.4244617153733296\n",
      "AUC:  0.5407149475673604\n",
      "accuracy:  0.54102\n"
     ]
    }
   ],
   "source": [
    "# KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier  \n",
    "classifier = KNeighborsClassifier(metric='manhattan', weights='uniform', n_neighbors=19 )  \n",
    "classifier.fit(X_train, y_train) \n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = x_train_chi, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "print ('accuracy: ',accuracy)\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27220 23218 24365 25197\n",
      "pod:  0.5083935272991404\n",
      "pof:  0.46032753082992983\n",
      "AUC:  0.5240329982346053\n",
      "accuracy:  0.52417\n"
     ]
    }
   ],
   "source": [
    "# KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier  \n",
    "classifier = KNeighborsClassifier(n_neighbors=5)  \n",
    "classifier.fit(X_train, y_train) \n",
    "\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = X, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "print ('accuracy: ',accuracy)\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000], 'max_features': ['auto', 'sqrt'], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None], 'min_samples_split': [2, 5, 10], 'min_samples_leaf': [1, 2, 4], 'bootstrap': [True, False]}\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 288 candidates, totalling 864 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-29c8a0868983>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     33\u001b[0m grid_search = GridSearchCV(estimator = RandomForestClassifier(), param_grid = param_grid, \n\u001b[0;32m     34\u001b[0m                           cv = 3, n_jobs = -1, verbose = 2)\n\u001b[1;32m---> 35\u001b[1;33m \u001b[0mgrid_search\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_train_chi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     36\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrid\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[0;32m    720\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mresults_container\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    721\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 722\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    723\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    724\u001b[0m         \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresults_container\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[1;34m(self, evaluate_candidates)\u001b[0m\n\u001b[0;32m   1189\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1190\u001b[0m         \u001b[1;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1191\u001b[1;33m         \u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1192\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1193\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[1;34m(candidate_params)\u001b[0m\n\u001b[0;32m    709\u001b[0m                                \u001b[1;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    710\u001b[0m                                in product(candidate_params,\n\u001b[1;32m--> 711\u001b[1;33m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[0;32m    712\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    713\u001b[0m                 \u001b[0mall_candidate_params\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcandidate_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    928\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    929\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 930\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    931\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    932\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    831\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    832\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'supports_timeout'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 833\u001b[1;33m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    834\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    835\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[1;34m(future, timeout)\u001b[0m\n\u001b[0;32m    519\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[0;32m    520\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 521\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    522\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mLokyTimeoutError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    523\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\concurrent\\futures\\_base.py\u001b[0m in \u001b[0;36mresult\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    425\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    426\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 427\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    428\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    429\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mCANCELLED\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCANCELLED_AND_NOTIFIED\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    294\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    295\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 296\u001b[1;33m                 \u001b[0mwaiter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    297\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    298\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "# Create the parameter grid based on the results of random search \n",
    "param_grid = {\n",
    "    'bootstrap': [True],\n",
    "    'max_depth': [80, 90, 100, 110],\n",
    "    'max_features': [2, 3],\n",
    "    'min_samples_leaf': [3, 4, 5],\n",
    "    'min_samples_split': [8, 10, 12],\n",
    "    'n_estimators': [100, 200, 300, 1000]\n",
    "}\n",
    "\n",
    "\n",
    "#clf = RandomForestClassifier()\n",
    "\n",
    "# Choose some parameter combinations to try\n",
    "#parameters = {'n_estimators': [4, 6, 9], \n",
    "#              'max_features': ['log2', 'sqrt','auto'], \n",
    "#              'criterion': ['entropy', 'gini'],\n",
    "#              'max_depth': [2, 3, 5, 10], \n",
    "#              'min_samples_split': [2, 3, 5],\n",
    "#              'min_samples_leaf': [1,5,8]\n",
    "#             }\n",
    "\n",
    "#model_params = {\n",
    "#    'n_estimators': [50, 150, 250],\n",
    "#    'max_features': ['sqrt', 0.25, 0.5, 0.75, 1.0],\n",
    "#    'min_samples_split': [2, 4, 6]\n",
    "#}\n",
    "\n",
    "# Instantiate the grid search model\n",
    "grid_search = GridSearchCV(estimator = RandomForestClassifier(), param_grid = param_grid, \n",
    "                          cv = 3, n_jobs = -1, verbose = 2)\n",
    "grid_search.fit(x_train_chi, y_train)\n",
    "\n",
    "print(grid.best_params_)\n",
    "print(grid.best_estimator_)\n",
    "print(grid.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': True, 'max_depth': 110, 'max_features': 3, 'min_samples_leaf': 3, 'min_samples_split': 8, 'n_estimators': 1000}\n",
      "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "                       max_depth=110, max_features=3, max_leaf_nodes=None,\n",
      "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                       min_samples_leaf=3, min_samples_split=8,\n",
      "                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
      "                       n_jobs=None, oob_score=False, random_state=None,\n",
      "                       verbose=0, warm_start=False)\n",
      "0.58342\n"
     ]
    }
   ],
   "source": [
    "print(grid_search.best_params_)\n",
    "print(grid_search.best_estimator_)\n",
    "print(grid_search.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29509 20929 20280 29282\n",
      "pod:  0.5908155441669021\n",
      "pof:  0.4149450810896546\n",
      "AUC:  0.5879352315386237\n",
      "accuracy:  0.58791\n"
     ]
    }
   ],
   "source": [
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "classifier =RandomForestClassifier(bootstrap= True, max_depth= 110, max_features= 3, min_samples_leaf= 3, min_samples_split= 8, n_estimators= 1000)\n",
    "classifier.fit(x_train_chi, y_train) \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = x_train_chi, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "print ('accuracy: ',accuracy)\n",
    "\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26400 24038 20635 28927\n",
      "pod:  0.5836527985149913\n",
      "pof:  0.4765851143978746\n",
      "AUC:  0.5535338420585584\n"
     ]
    }
   ],
   "source": [
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "classifier =RandomForestClassifier(n_estimators=100,max_depth=2,random_state=0)\n",
    "classifier.fit(X_train, y_train) \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = X, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "param_grid = {\n",
    "    \"loss\":[\"deviance\"],\n",
    "    \"learning_rate\": [0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2],\n",
    "    \"min_samples_split\": np.linspace(0.1, 0.5, 12),\n",
    "    \"min_samples_leaf\": np.linspace(0.1, 0.5, 12),\n",
    "    \"max_depth\":[3,5,8],\n",
    "    \"max_features\":[\"log2\",\"sqrt\"],\n",
    "    \"criterion\": [\"friedman_mse\",  \"mae\"],\n",
    "    \"subsample\":[0.5, 0.618, 0.8, 0.85, 0.9, 0.95, 1.0],\n",
    "    \"n_estimators\":[10]\n",
    "    }\n",
    "\n",
    "grid = GridSearchCV(GradientBoostingClassifier(), param_grid, cv=10, n_jobs=-1)\n",
    "\n",
    "grid.fit(x_train_chi, y_train)\n",
    "\n",
    "print(grid.best_params_)\n",
    "print(grid.best_estimator_)\n",
    "print(grid.best_score_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26622 23816 23751 25811\n",
      "pod:  0.5207820507646987\n",
      "pof:  0.4721836710416749\n",
      "AUC:  0.5242991898615119\n",
      "accuracy:  0.52433\n"
     ]
    }
   ],
   "source": [
    "#Decision Tree\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "classifier = DecisionTreeClassifier(criterion='entropy', splitter='best')\n",
    "classifier.fit(x_train_chi, y_train) \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = x_train_chi, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "print ('accuracy: ',accuracy)\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26256 24182 23741 25821\n",
      "pod:  0.5209838182478512\n",
      "pof:  0.4794401046829771\n",
      "AUC:  0.5207718567824371\n"
     ]
    }
   ],
   "source": [
    "#Decision Tree\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "classifier = DecisionTreeClassifier()\n",
    "classifier.fit(X_train, y_train) \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = X_train, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28101 22337 18301 31261\n",
      "pod:  0.630745329082765\n",
      "pof:  0.4428605416551013\n",
      "AUC:  0.593942393713832\n",
      "accuracy:  0.59362\n"
     ]
    }
   ],
   "source": [
    "#Gradient Boosting\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "#classifier = GradientBoostingClassifier(max_features=None, max_depth=3, criterion='friedman_mse')\n",
    "classifier = GradientBoostingClassifier()\n",
    "classifier.fit(x_train_chi, y_train) \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = x_train_chi, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "print ('accuracy: ',accuracy)\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28087 22351 18218 31344\n",
      "pod:  0.63241999919293\n",
      "pof:  0.44313811015504184\n",
      "AUC:  0.5946409445189441\n",
      "accuracy:  0.59431\n"
     ]
    }
   ],
   "source": [
    "#Gradient Boosting\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "classifier = GradientBoostingClassifier()\n",
    "classifier.fit(X_train, y_train) \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "y_pred  = cross_val_predict(estimator = classifier, X = X_train, y = y_train, cv = 10)\n",
    "tn, fp, fn, tp  = confusion_matrix(y_train, y_pred).ravel()\n",
    "print(tn, fp, fn, tp)\n",
    "pod=tp/(tp+fn)\n",
    "\n",
    "print('pod: ',pod)\n",
    "pof=fp/(fp+tn)\n",
    "print ('pof: ',pof)\n",
    "auc_val=(1+pod-pof)/2\n",
    "print ('AUC: ',auc_val)\n",
    "\n",
    "accuracy=(tp+tn)/(tp+fn+fp+tn)\n",
    "print ('accuracy: ',accuracy)\n",
    "\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#roc_auc = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10,  scoring='roc_auc')\n",
    "#print('roc_auc: ',roc_auc.mean())\n",
    "\n",
    "#from sklearn.metrics import roc_curve, auc\n",
    "#fpr, tpr, thresholds = roc_curve(y_train, y_pred)\n",
    "#auc = auc(fpr, tpr)\n",
    "#print('auc: ',auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 71999 samples, validate on 18000 samples\n",
      "Epoch 1/30\n",
      "  608/71999 [..............................] - ETA: 9:31 - loss: 0.7065 - accuracy: 0.4951   "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\callbacks\\callbacks.py:95: RuntimeWarning: Method (on_train_batch_end) is slow compared to the batch update (0.316353). Check your callbacks.\n",
      "  % (hook_name, delta_t_median), RuntimeWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\callbacks\\callbacks.py:95: RuntimeWarning: Method (on_train_batch_end) is slow compared to the batch update (0.158677). Check your callbacks.\n",
      "  % (hook_name, delta_t_median), RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71999/71999 [==============================] - 13s 175us/step - loss: 0.6867 - accuracy: 0.5487 - val_loss: 0.6794 - val_accuracy: 0.5807\n",
      "Epoch 2/30\n",
      "71999/71999 [==============================] - 7s 102us/step - loss: 0.6776 - accuracy: 0.5739 - val_loss: 0.6735 - val_accuracy: 0.5894\n",
      "Epoch 3/30\n",
      "71999/71999 [==============================] - 8s 106us/step - loss: 0.6734 - accuracy: 0.5801 - val_loss: 0.6793 - val_accuracy: 0.5837\n",
      "Epoch 4/30\n",
      "71999/71999 [==============================] - 7s 102us/step - loss: 0.6712 - accuracy: 0.5860 - val_loss: 0.6825 - val_accuracy: 0.5721\n",
      "Epoch 5/30\n",
      "71999/71999 [==============================] - 8s 107us/step - loss: 0.6697 - accuracy: 0.5877 - val_loss: 0.6751 - val_accuracy: 0.5984\n",
      "y2_pred:  [[0.1995708 ]\n",
      " [0.3651774 ]\n",
      " [0.7590049 ]\n",
      " ...\n",
      " [0.53678375]\n",
      " [0.71236444]\n",
      " [0.6727204 ]]\n",
      "y22_pred:  [[0.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [1.]]\n",
      "pod 1st:  0.585636473673593\n",
      "Train on 71999 samples, validate on 18000 samples\n",
      "Epoch 1/30\n",
      "71999/71999 [==============================] - 8s 106us/step - loss: 0.6849 - accuracy: 0.5563 - val_loss: 0.6814 - val_accuracy: 0.5661\n",
      "Epoch 2/30\n",
      "71999/71999 [==============================] - 7s 104us/step - loss: 0.6763 - accuracy: 0.5767 - val_loss: 0.6735 - val_accuracy: 0.5843\n",
      "Epoch 3/30\n",
      "71999/71999 [==============================] - 7s 98us/step - loss: 0.6730 - accuracy: 0.5834 - val_loss: 0.6693 - val_accuracy: 0.5984\n",
      "Epoch 4/30\n",
      "71999/71999 [==============================] - 7s 102us/step - loss: 0.6710 - accuracy: 0.5853 - val_loss: 0.6600 - val_accuracy: 0.6153\n",
      "Epoch 5/30\n",
      "71999/71999 [==============================] - 7s 100us/step - loss: 0.6692 - accuracy: 0.5884 - val_loss: 0.6736 - val_accuracy: 0.5897\n",
      "Epoch 6/30\n",
      "71999/71999 [==============================] - 8s 109us/step - loss: 0.6671 - accuracy: 0.5926 - val_loss: 0.6675 - val_accuracy: 0.6016\n",
      "Epoch 7/30\n",
      "71999/71999 [==============================] - 8s 111us/step - loss: 0.6661 - accuracy: 0.5935 - val_loss: 0.6750 - val_accuracy: 0.5742\n",
      "y2_pred:  [[0.4183093 ]\n",
      " [0.5514712 ]\n",
      " [0.70328707]\n",
      " ...\n",
      " [0.54850507]\n",
      " [0.6069394 ]\n",
      " [0.5965266 ]]\n",
      "y22_pred:  [[0.]\n",
      " [1.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [1.]]\n",
      "pod 1st:  0.725842243292314\n",
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/30\n",
      "72000/72000 [==============================] - 8s 118us/step - loss: 0.6859 - accuracy: 0.5512 - val_loss: 0.6856 - val_accuracy: 0.5613\n",
      "Epoch 2/30\n",
      "72000/72000 [==============================] - 8s 106us/step - loss: 0.6763 - accuracy: 0.5762 - val_loss: 0.6704 - val_accuracy: 0.5899\n",
      "Epoch 3/30\n",
      "72000/72000 [==============================] - 8s 110us/step - loss: 0.6723 - accuracy: 0.5836 - val_loss: 0.6663 - val_accuracy: 0.6069\n",
      "Epoch 4/30\n",
      "72000/72000 [==============================] - 7s 101us/step - loss: 0.6693 - accuracy: 0.5894 - val_loss: 0.6836 - val_accuracy: 0.5793\n",
      "Epoch 5/30\n",
      "72000/72000 [==============================] - 8s 107us/step - loss: 0.6683 - accuracy: 0.5912 - val_loss: 0.6690 - val_accuracy: 0.6040\n",
      "Epoch 6/30\n",
      "72000/72000 [==============================] - 8s 107us/step - loss: 0.6671 - accuracy: 0.5920 - val_loss: 0.6624 - val_accuracy: 0.6027\n",
      "Epoch 7/30\n",
      "72000/72000 [==============================] - 8s 115us/step - loss: 0.6663 - accuracy: 0.5944 - val_loss: 0.6711 - val_accuracy: 0.6027\n",
      "Epoch 8/30\n",
      "72000/72000 [==============================] - 8s 107us/step - loss: 0.6656 - accuracy: 0.5938 - val_loss: 0.6641 - val_accuracy: 0.6055\n",
      "Epoch 9/30\n",
      "72000/72000 [==============================] - 9s 125us/step - loss: 0.6650 - accuracy: 0.5952 - val_loss: 0.6782 - val_accuracy: 0.5758\n",
      "y2_pred:  [[0.5837752 ]\n",
      " [0.6213813 ]\n",
      " [0.5388638 ]\n",
      " ...\n",
      " [0.5261584 ]\n",
      " [0.30623233]\n",
      " [0.5836499 ]]\n",
      "y22_pred:  [[1.]\n",
      " [1.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [0.]\n",
      " [1.]]\n",
      "pod 1st:  0.7552461662631155\n",
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/30\n",
      "72000/72000 [==============================] - 8s 105us/step - loss: 0.6857 - accuracy: 0.5550 - val_loss: 0.6778 - val_accuracy: 0.5823\n",
      "Epoch 2/30\n",
      "72000/72000 [==============================] - 7s 101us/step - loss: 0.6757 - accuracy: 0.5770 - val_loss: 0.6716 - val_accuracy: 0.5972\n",
      "Epoch 3/30\n",
      "72000/72000 [==============================] - 7s 100us/step - loss: 0.6722 - accuracy: 0.5816 - val_loss: 0.6779 - val_accuracy: 0.5792\n",
      "Epoch 4/30\n",
      "72000/72000 [==============================] - 7s 100us/step - loss: 0.6703 - accuracy: 0.5868 - val_loss: 0.6690 - val_accuracy: 0.5861\n",
      "Epoch 5/30\n",
      "72000/72000 [==============================] - 7s 97us/step - loss: 0.6687 - accuracy: 0.5880 - val_loss: 0.6713 - val_accuracy: 0.6004\n",
      "Epoch 6/30\n",
      "72000/72000 [==============================] - 7s 100us/step - loss: 0.6674 - accuracy: 0.5907 - val_loss: 0.6657 - val_accuracy: 0.6052\n",
      "Epoch 7/30\n",
      "72000/72000 [==============================] - 8s 109us/step - loss: 0.6666 - accuracy: 0.5919 - val_loss: 0.6653 - val_accuracy: 0.6132\n",
      "Epoch 8/30\n",
      "72000/72000 [==============================] - 7s 99us/step - loss: 0.6657 - accuracy: 0.5926 - val_loss: 0.6782 - val_accuracy: 0.5647\n",
      "Epoch 9/30\n",
      "72000/72000 [==============================] - 7s 98us/step - loss: 0.6651 - accuracy: 0.5947 - val_loss: 0.6860 - val_accuracy: 0.5496\n",
      "Epoch 10/30\n",
      "72000/72000 [==============================] - 7s 96us/step - loss: 0.6647 - accuracy: 0.5935 - val_loss: 0.6663 - val_accuracy: 0.5989\n",
      "y2_pred:  [[0.33405697]\n",
      " [0.44543585]\n",
      " [0.59525275]\n",
      " ...\n",
      " [0.33614314]\n",
      " [0.47612265]\n",
      " [0.55191755]]\n",
      "y22_pred:  [[0.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [0.]\n",
      " [0.]\n",
      " [1.]]\n",
      "pod 1st:  0.6230831315577078\n",
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/30\n",
      "72000/72000 [==============================] - 8s 113us/step - loss: 0.6842 - accuracy: 0.5569 - val_loss: 0.6845 - val_accuracy: 0.5501\n",
      "Epoch 2/30\n",
      "72000/72000 [==============================] - 7s 104us/step - loss: 0.6723 - accuracy: 0.5841 - val_loss: 0.6673 - val_accuracy: 0.5979\n",
      "Epoch 3/30\n",
      "72000/72000 [==============================] - 8s 107us/step - loss: 0.6695 - accuracy: 0.5887 - val_loss: 0.6792 - val_accuracy: 0.5743\n",
      "Epoch 4/30\n",
      "72000/72000 [==============================] - 8s 104us/step - loss: 0.6671 - accuracy: 0.5927 - val_loss: 0.6767 - val_accuracy: 0.5782\n",
      "Epoch 5/30\n",
      "72000/72000 [==============================] - 8s 109us/step - loss: 0.6658 - accuracy: 0.5931 - val_loss: 0.6675 - val_accuracy: 0.5938\n",
      "y2_pred:  [[0.61497843]\n",
      " [0.60983074]\n",
      " [0.49547952]\n",
      " ...\n",
      " [0.481464  ]\n",
      " [0.5105566 ]\n",
      " [0.39646426]]\n",
      "y22_pred:  [[1.]\n",
      " [1.]\n",
      " [0.]\n",
      " ...\n",
      " [0.]\n",
      " [1.]\n",
      " [0.]]\n",
      "pod 1st:  0.6678773204196933\n",
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/30\n",
      "72000/72000 [==============================] - 9s 120us/step - loss: 0.6863 - accuracy: 0.5512 - val_loss: 0.6743 - val_accuracy: 0.5928\n",
      "Epoch 2/30\n",
      "72000/72000 [==============================] - 8s 106us/step - loss: 0.6748 - accuracy: 0.5781 - val_loss: 0.6697 - val_accuracy: 0.5942\n",
      "Epoch 3/30\n",
      "72000/72000 [==============================] - 7s 103us/step - loss: 0.6706 - accuracy: 0.5865 - val_loss: 0.6685 - val_accuracy: 0.5989\n",
      "Epoch 4/30\n",
      "72000/72000 [==============================] - 8s 109us/step - loss: 0.6687 - accuracy: 0.5900 - val_loss: 0.6952 - val_accuracy: 0.5494\n",
      "Epoch 5/30\n",
      "72000/72000 [==============================] - 8s 104us/step - loss: 0.6666 - accuracy: 0.5924 - val_loss: 0.6720 - val_accuracy: 0.5938\n",
      "Epoch 6/30\n",
      "72000/72000 [==============================] - 8s 108us/step - loss: 0.6661 - accuracy: 0.5932 - val_loss: 0.6665 - val_accuracy: 0.5986\n",
      "Epoch 7/30\n",
      "72000/72000 [==============================] - 8s 115us/step - loss: 0.6651 - accuracy: 0.5975 - val_loss: 0.6645 - val_accuracy: 0.6043\n",
      "Epoch 8/30\n",
      "72000/72000 [==============================] - 8s 114us/step - loss: 0.6640 - accuracy: 0.5958 - val_loss: 0.6742 - val_accuracy: 0.6008\n",
      "Epoch 9/30\n",
      "72000/72000 [==============================] - 8s 105us/step - loss: 0.6637 - accuracy: 0.5966 - val_loss: 0.6855 - val_accuracy: 0.5763\n",
      "Epoch 10/30\n",
      "72000/72000 [==============================] - 8s 110us/step - loss: 0.6631 - accuracy: 0.5971 - val_loss: 0.6849 - val_accuracy: 0.5764\n",
      "y2_pred:  [[0.61537176]\n",
      " [0.54313123]\n",
      " [0.51405555]\n",
      " ...\n",
      " [0.5548211 ]\n",
      " [0.4051618 ]\n",
      " [0.45178077]]\n",
      "y22_pred:  [[1.]\n",
      " [1.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [0.]\n",
      " [0.]]\n",
      "pod 1st:  0.7076271186440678\n",
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72000/72000 [==============================] - 7s 104us/step - loss: 0.6831 - accuracy: 0.5587 - val_loss: 0.7110 - val_accuracy: 0.4744\n",
      "Epoch 2/30\n",
      "72000/72000 [==============================] - 8s 106us/step - loss: 0.6712 - accuracy: 0.5841 - val_loss: 0.6709 - val_accuracy: 0.5935\n",
      "Epoch 3/30\n",
      "72000/72000 [==============================] - 7s 102us/step - loss: 0.6655 - accuracy: 0.5912 - val_loss: 0.7069 - val_accuracy: 0.5129\n",
      "Epoch 4/30\n",
      "72000/72000 [==============================] - 8s 109us/step - loss: 0.6624 - accuracy: 0.5982 - val_loss: 0.6743 - val_accuracy: 0.5801\n",
      "Epoch 5/30\n",
      "72000/72000 [==============================] - 7s 101us/step - loss: 0.6607 - accuracy: 0.6008 - val_loss: 0.6658 - val_accuracy: 0.5900\n",
      "Epoch 6/30\n",
      "72000/72000 [==============================] - 8s 115us/step - loss: 0.6591 - accuracy: 0.6019 - val_loss: 0.6677 - val_accuracy: 0.6066\n",
      "Epoch 7/30\n",
      "72000/72000 [==============================] - 7s 100us/step - loss: 0.6578 - accuracy: 0.6052 - val_loss: 0.6806 - val_accuracy: 0.5721\n",
      "Epoch 8/30\n",
      "72000/72000 [==============================] - 8s 106us/step - loss: 0.6573 - accuracy: 0.6052 - val_loss: 0.6584 - val_accuracy: 0.6154\n",
      "Epoch 9/30\n",
      "72000/72000 [==============================] - 7s 101us/step - loss: 0.6560 - accuracy: 0.6050 - val_loss: 0.6844 - val_accuracy: 0.5727\n",
      "Epoch 10/30\n",
      "72000/72000 [==============================] - 8s 105us/step - loss: 0.6561 - accuracy: 0.6055 - val_loss: 0.6842 - val_accuracy: 0.5788\n",
      "Epoch 11/30\n",
      "72000/72000 [==============================] - 7s 101us/step - loss: 0.6552 - accuracy: 0.6065 - val_loss: 0.6588 - val_accuracy: 0.6108\n",
      "y2_pred:  [[0.42616197]\n",
      " [0.3762729 ]\n",
      " [0.5880836 ]\n",
      " ...\n",
      " [0.6717525 ]\n",
      " [0.58033395]\n",
      " [0.4577837 ]]\n",
      "y22_pred:  [[0.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [0.]]\n",
      "pod 1st:  0.5560936238902341\n",
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/30\n",
      "72000/72000 [==============================] - 8s 112us/step - loss: 0.6843 - accuracy: 0.5562 - val_loss: 0.6736 - val_accuracy: 0.5861\n",
      "Epoch 2/30\n",
      "72000/72000 [==============================] - 8s 106us/step - loss: 0.6724 - accuracy: 0.5845 - val_loss: 0.6744 - val_accuracy: 0.5781\n",
      "Epoch 3/30\n",
      "72000/72000 [==============================] - 7s 102us/step - loss: 0.6680 - accuracy: 0.5923 - val_loss: 0.6811 - val_accuracy: 0.5568\n",
      "Epoch 4/30\n",
      "72000/72000 [==============================] - 7s 97us/step - loss: 0.6652 - accuracy: 0.5962 - val_loss: 0.6784 - val_accuracy: 0.5636\n",
      "y2_pred:  [[0.53518647]\n",
      " [0.22331902]\n",
      " [0.64273244]\n",
      " ...\n",
      " [0.48312846]\n",
      " [0.53722477]\n",
      " [0.23649743]]\n",
      "y22_pred:  [[1.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [0.]\n",
      " [1.]\n",
      " [0.]]\n",
      "pod 1st:  0.42736077481840196\n",
      "Train on 72000 samples, validate on 18001 samples\n",
      "Epoch 1/30\n",
      "72000/72000 [==============================] - 8s 107us/step - loss: 0.6861 - accuracy: 0.5512 - val_loss: 0.6889 - val_accuracy: 0.5555\n",
      "Epoch 2/30\n",
      "72000/72000 [==============================] - 7s 101us/step - loss: 0.6773 - accuracy: 0.5759 - val_loss: 0.6701 - val_accuracy: 0.5977\n",
      "Epoch 3/30\n",
      "72000/72000 [==============================] - 7s 102us/step - loss: 0.6734 - accuracy: 0.5817 - val_loss: 0.6780 - val_accuracy: 0.5871\n",
      "Epoch 4/30\n",
      "72000/72000 [==============================] - 7s 103us/step - loss: 0.6704 - accuracy: 0.5868 - val_loss: 0.6918 - val_accuracy: 0.5362\n",
      "Epoch 5/30\n",
      "72000/72000 [==============================] - 7s 101us/step - loss: 0.6690 - accuracy: 0.5881 - val_loss: 0.6818 - val_accuracy: 0.5711\n",
      "y2_pred:  [[0.67304564]\n",
      " [0.44074044]\n",
      " [0.5265009 ]\n",
      " ...\n",
      " [0.47974697]\n",
      " [0.45325384]\n",
      " [0.42895633]]\n",
      "y22_pred:  [[1.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [0.]\n",
      " [0.]\n",
      " [0.]]\n",
      "pod 1st:  0.6277239709443099\n",
      "Train on 72000 samples, validate on 18001 samples\n",
      "Epoch 1/30\n",
      "72000/72000 [==============================] - 7s 100us/step - loss: 0.6861 - accuracy: 0.5497 - val_loss: 0.6839 - val_accuracy: 0.5603\n",
      "Epoch 2/30\n",
      "72000/72000 [==============================] - 8s 105us/step - loss: 0.6770 - accuracy: 0.5749 - val_loss: 0.6759 - val_accuracy: 0.5685\n",
      "Epoch 3/30\n",
      "72000/72000 [==============================] - 7s 100us/step - loss: 0.6732 - accuracy: 0.5831 - val_loss: 0.6760 - val_accuracy: 0.5817\n",
      "Epoch 4/30\n",
      "72000/72000 [==============================] - 7s 95us/step - loss: 0.6701 - accuracy: 0.5883 - val_loss: 0.6984 - val_accuracy: 0.5209\n",
      "Epoch 5/30\n",
      "72000/72000 [==============================] - 7s 101us/step - loss: 0.6687 - accuracy: 0.5910 - val_loss: 0.6836 - val_accuracy: 0.5560\n",
      "y2_pred:  [[0.59524417]\n",
      " [0.49396303]\n",
      " [0.6721357 ]\n",
      " ...\n",
      " [0.53130335]\n",
      " [0.45972976]\n",
      " [0.37329096]]\n",
      "y22_pred:  [[1.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [0.]\n",
      " [0.]]\n",
      "pod 1st:  0.5905972558514931\n",
      "pod:  [0.585636473673593, 0.725842243292314, 0.7552461662631155, 0.6230831315577078, 0.6678773204196933, 0.7076271186440678, 0.5560936238902341, 0.42736077481840196, 0.6277239709443099, 0.5905972558514931]\n",
      "pof:  [0.35289452815226013, 0.5402458366375892, 0.5935765265662173, 0.4417129262490087, 0.49246629659000796, 0.5681998413957177, 0.563441712926249, 0.4282315622521808, 0.4774935554233591, 0.4257386476303787]\n",
      "auc:  [0.6163709727606663, 0.5927982033273624, 0.580834819848449, 0.5906851026543495, 0.5877055119148427, 0.5697136386241751, 0.49632595548199254, 0.4995646062831106, 0.5751152077604754, 0.5824293041105573]\n",
      "tn_list:  [3264, 2319, 2050, 2816, 2560, 2178, 2202, 2884, 2635, 2896]\n",
      "fp_list:  [1780, 2725, 2994, 2228, 2484, 2866, 2842, 2160, 2408, 2147]\n",
      "fn_list:  [2054, 1359, 1213, 1868, 1646, 1449, 2200, 2838, 1845, 2029]\n",
      "tp_list:  [2903, 3598, 3743, 3088, 3310, 3507, 2756, 2118, 3111, 2927]\n",
      "25804 24634 18501 31061\n"
     ]
    }
   ],
   "source": [
    "df_x_train_chi = pd.DataFrame(x_train_chi)\n",
    "df_x_train_chi.head()\n",
    "\n",
    "#FNN\n",
    "def get_FNN_Predict(X2_train, X2_test, y2_train, y2_test):\n",
    "    from keras.models import Sequential\n",
    "    from keras.layers import Dense\n",
    "    #create model\n",
    "    model = Sequential()\n",
    "    #get number of columns in training data\n",
    "    n_cols = X2_train.shape[1]\n",
    "    model.add(Dense(10, activation='relu', input_shape=(n_cols,)))\n",
    "    model.add(Dense(250, activation='relu'))\n",
    "    model.add(Dense(250, activation='relu'))\n",
    "    model.add(Dense(250, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    #compile model using mse as a measure of model performance\n",
    "    #model.compile(optimizer='adam', loss='mean_squared_error')\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    #model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])  \n",
    "    from keras.callbacks import EarlyStopping\n",
    "    #set early stopping monitor so the model stops training when it won't improve anymore\n",
    "    early_stopping_monitor = EarlyStopping(patience=3)\n",
    "    #train model\n",
    "    model.fit(X2_train, y2_train, validation_split=0.2, epochs=30, callbacks=[early_stopping_monitor])\n",
    "    y2_pred = model.predict(X2_test)\n",
    "    print('y2_pred: ',y2_pred)\n",
    "    y22_pred=y2_pred.round()\n",
    "    print('y22_pred: ',y22_pred)\n",
    "    return y22_pred\n",
    "\n",
    "pod_list = []\n",
    "pof_list = []\n",
    "auc_val_list = []\n",
    "tn_list= []\n",
    "fp_list= []\n",
    "fn_list= []\n",
    "tp_list= []\n",
    "    \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "folds = StratifiedKFold(n_splits=10)\n",
    "for train_index, test_index in folds.split(df_x_train_chi,y_train):\n",
    "    X2_train, X2_test, y2_train, y2_test=df_x_train_chi.iloc[train_index], df_x_train_chi.iloc[test_index],y_train.iloc[train_index], y_train.iloc[test_index]\n",
    "    y_pred = get_FNN_Predict(X2_train, X2_test, y2_train, y2_test)\n",
    "    tn, fp, fn, tp  = confusion_matrix(y2_test, y_pred).ravel()\n",
    "    tn_list.append(tn)\n",
    "    fp_list.append(fp)\n",
    "    fn_list.append(fn)\n",
    "    tp_list.append(tp)\n",
    "    pod=tp/(tp+fn)\n",
    "    print('pod 1st: ',pod)\n",
    "    pof=fp/(fp+tn)\n",
    "    auc_val=(1+pod-pof)/2\n",
    "    #break\n",
    "    pod_list.append(pod)\n",
    "    pof_list.append(pof)\n",
    "    auc_val_list.append(auc_val)\n",
    "\n",
    "print('pod: ',pod_list)\n",
    "print ('pof: ',pof_list)\n",
    "print ('auc: ',auc_val_list)\n",
    "\n",
    "print('tn_list: ',tn_list)\n",
    "print ('fp_list: ',fp_list)\n",
    "print ('fn_list: ',fn_list)\n",
    "print ('tp_list: ',tp_list)\n",
    "\n",
    "\n",
    "tn=sum(tn_list)\n",
    "fp=sum(fp_list) \n",
    "fn=sum(fn_list) \n",
    "tp=sum(tp_list)\n",
    "print(tn, fp, fn, tp)\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 71999 samples, validate on 18000 samples\n",
      "Epoch 1/5\n",
      "71999/71999 [==============================] - 44s 617us/step - loss: 0.6900 - acc: 0.5316 - val_loss: 0.6969 - val_acc: 0.4944\n",
      "Epoch 2/5\n",
      "71999/71999 [==============================] - 43s 599us/step - loss: 0.6870 - acc: 0.5447 - val_loss: 0.6877 - val_acc: 0.5361\n",
      "Epoch 3/5\n",
      "71999/71999 [==============================] - 42s 583us/step - loss: 0.6861 - acc: 0.5479 - val_loss: 0.7022 - val_acc: 0.4983\n",
      "Epoch 4/5\n",
      "71999/71999 [==============================] - 43s 595us/step - loss: 0.6854 - acc: 0.5515 - val_loss: 0.6969 - val_acc: 0.5080\n",
      "Epoch 5/5\n",
      "71999/71999 [==============================] - 42s 587us/step - loss: 0.6847 - acc: 0.5523 - val_loss: 0.6986 - val_acc: 0.5087\n",
      "y22_pred:  [[1.]\n",
      " [0.]\n",
      " [0.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [1.]]\n",
      "pod 1st:  0.7318942909017551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 71999 samples, validate on 18000 samples\n",
      "Epoch 1/5\n",
      "71999/71999 [==============================] - 44s 617us/step - loss: 0.6895 - acc: 0.5354 - val_loss: 0.6974 - val_acc: 0.5008\n",
      "Epoch 2/5\n",
      "71999/71999 [==============================] - 43s 598us/step - loss: 0.6869 - acc: 0.5467 - val_loss: 0.6898 - val_acc: 0.5336\n",
      "Epoch 3/5\n",
      "71999/71999 [==============================] - 43s 600us/step - loss: 0.6860 - acc: 0.5487 - val_loss: 0.6873 - val_acc: 0.5397\n",
      "Epoch 4/5\n",
      "71999/71999 [==============================] - 43s 602us/step - loss: 0.6847 - acc: 0.5538 - val_loss: 0.6890 - val_acc: 0.5369\n",
      "Epoch 5/5\n",
      "71999/71999 [==============================] - 43s 601us/step - loss: 0.6826 - acc: 0.5600 - val_loss: 0.6953 - val_acc: 0.5167\n",
      "y22_pred:  [[0.]\n",
      " [1.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [1.]]\n",
      "pod 1st:  0.7653822876739964\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/5\n",
      "72000/72000 [==============================] - 42s 583us/step - loss: 0.6890 - acc: 0.5356 - val_loss: 0.6930 - val_acc: 0.5186\n",
      "Epoch 2/5\n",
      "72000/72000 [==============================] - 42s 588us/step - loss: 0.6858 - acc: 0.5501 - val_loss: 0.6852 - val_acc: 0.5510\n",
      "Epoch 3/5\n",
      "72000/72000 [==============================] - 42s 586us/step - loss: 0.6851 - acc: 0.5511 - val_loss: 0.6841 - val_acc: 0.5526\n",
      "Epoch 4/5\n",
      "72000/72000 [==============================] - 42s 588us/step - loss: 0.6845 - acc: 0.5546 - val_loss: 0.6779 - val_acc: 0.5817\n",
      "Epoch 5/5\n",
      "72000/72000 [==============================] - 44s 608us/step - loss: 0.6834 - acc: 0.5571 - val_loss: 0.6821 - val_acc: 0.5619\n",
      "y22_pred:  [[0.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [0.]]\n",
      "pod 1st:  0.5724374495560937\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/5\n",
      "72000/72000 [==============================] - 44s 617us/step - loss: 0.6895 - acc: 0.5331 - val_loss: 0.7026 - val_acc: 0.4932\n",
      "Epoch 2/5\n",
      "72000/72000 [==============================] - 43s 597us/step - loss: 0.6866 - acc: 0.5469 - val_loss: 0.6908 - val_acc: 0.5282\n",
      "Epoch 3/5\n",
      "72000/72000 [==============================] - 42s 589us/step - loss: 0.6854 - acc: 0.5501 - val_loss: 0.6861 - val_acc: 0.5448\n",
      "Epoch 4/5\n",
      "72000/72000 [==============================] - 44s 611us/step - loss: 0.6840 - acc: 0.5564 - val_loss: 0.6956 - val_acc: 0.5173\n",
      "Epoch 5/5\n",
      "72000/72000 [==============================] - 45s 619us/step - loss: 0.6813 - acc: 0.5626 - val_loss: 0.6801 - val_acc: 0.5677\n",
      "y22_pred:  [[0.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [0.]\n",
      " [1.]\n",
      " [0.]]\n",
      "pod 1st:  0.5799031476997578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/5\n",
      "72000/72000 [==============================] - 44s 611us/step - loss: 0.6895 - acc: 0.5363 - val_loss: 0.7021 - val_acc: 0.4774\n",
      "Epoch 2/5\n",
      "72000/72000 [==============================] - 43s 598us/step - loss: 0.6869 - acc: 0.5448 - val_loss: 0.7027 - val_acc: 0.4995\n",
      "Epoch 3/5\n",
      "72000/72000 [==============================] - 44s 607us/step - loss: 0.6858 - acc: 0.5484 - val_loss: 0.7124 - val_acc: 0.4644\n",
      "Epoch 4/5\n",
      "72000/72000 [==============================] - 44s 616us/step - loss: 0.6852 - acc: 0.5507 - val_loss: 0.6891 - val_acc: 0.5363\n",
      "Epoch 5/5\n",
      "72000/72000 [==============================] - 43s 592us/step - loss: 0.6845 - acc: 0.5544 - val_loss: 0.6908 - val_acc: 0.5326\n",
      "y22_pred:  [[0.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [0.]]\n",
      "pod 1st:  0.6646489104116223\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/5\n",
      "72000/72000 [==============================] - 44s 611us/step - loss: 0.6892 - acc: 0.5370 - val_loss: 0.7135 - val_acc: 0.4475\n",
      "Epoch 2/5\n",
      "72000/72000 [==============================] - 43s 601us/step - loss: 0.6861 - acc: 0.5486 - val_loss: 0.6857 - val_acc: 0.5513\n",
      "Epoch 3/5\n",
      "72000/72000 [==============================] - 44s 616us/step - loss: 0.6844 - acc: 0.5532 - val_loss: 0.6905 - val_acc: 0.5349\n",
      "Epoch 4/5\n",
      "72000/72000 [==============================] - 44s 604us/step - loss: 0.6813 - acc: 0.5649 - val_loss: 0.6837 - val_acc: 0.5572\n",
      "Epoch 5/5\n",
      "72000/72000 [==============================] - 44s 614us/step - loss: 0.6799 - acc: 0.5674 - val_loss: 0.6854 - val_acc: 0.5482\n",
      "y22_pred:  [[0.]\n",
      " [1.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [1.]]\n",
      "pod 1st:  0.6180387409200968\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/5\n",
      "72000/72000 [==============================] - 43s 604us/step - loss: 0.6893 - acc: 0.5325 - val_loss: 0.6992 - val_acc: 0.4893\n",
      "Epoch 2/5\n",
      "72000/72000 [==============================] - 44s 607us/step - loss: 0.6854 - acc: 0.5506 - val_loss: 0.6987 - val_acc: 0.5003\n",
      "Epoch 3/5\n",
      "72000/72000 [==============================] - 43s 594us/step - loss: 0.6843 - acc: 0.5531 - val_loss: 0.6821 - val_acc: 0.5643\n",
      "Epoch 4/5\n",
      "72000/72000 [==============================] - 45s 625us/step - loss: 0.6835 - acc: 0.5575 - val_loss: 0.7204 - val_acc: 0.4906\n",
      "Epoch 5/5\n",
      "72000/72000 [==============================] - 44s 606us/step - loss: 0.6829 - acc: 0.5583 - val_loss: 0.7014 - val_acc: 0.5068\n",
      "y22_pred:  [[0.]\n",
      " [0.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [0.]]\n",
      "pod 1st:  0.7173123486682809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 72000 samples, validate on 18000 samples\n",
      "Epoch 1/5\n",
      "72000/72000 [==============================] - 45s 626us/step - loss: 0.6896 - acc: 0.5342 - val_loss: 0.6861 - val_acc: 0.5492\n",
      "Epoch 2/5\n",
      "72000/72000 [==============================] - 43s 591us/step - loss: 0.6866 - acc: 0.5452 - val_loss: 0.6843 - val_acc: 0.5549\n",
      "Epoch 3/5\n",
      "72000/72000 [==============================] - 44s 608us/step - loss: 0.6858 - acc: 0.5502 - val_loss: 0.6921 - val_acc: 0.5183\n",
      "Epoch 4/5\n",
      "72000/72000 [==============================] - 43s 604us/step - loss: 0.6853 - acc: 0.5505 - val_loss: 0.6943 - val_acc: 0.5243\n",
      "Epoch 5/5\n",
      "72000/72000 [==============================] - 44s 605us/step - loss: 0.6845 - acc: 0.5542 - val_loss: 0.6834 - val_acc: 0.5616\n",
      "y22_pred:  [[1.]\n",
      " [1.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [0.]\n",
      " [0.]]\n",
      "pod 1st:  0.4261501210653753\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 72000 samples, validate on 18001 samples\n",
      "Epoch 1/5\n",
      "72000/72000 [==============================] - 43s 603us/step - loss: 0.6894 - acc: 0.5340 - val_loss: 0.6939 - val_acc: 0.5097\n",
      "Epoch 2/5\n",
      "72000/72000 [==============================] - 43s 598us/step - loss: 0.6867 - acc: 0.5475 - val_loss: 0.6948 - val_acc: 0.5010\n",
      "Epoch 3/5\n",
      "72000/72000 [==============================] - 43s 603us/step - loss: 0.6854 - acc: 0.5503 - val_loss: 0.7112 - val_acc: 0.4596\n",
      "Epoch 4/5\n",
      "72000/72000 [==============================] - 44s 610us/step - loss: 0.6836 - acc: 0.5576 - val_loss: 0.6959 - val_acc: 0.5162\n",
      "Epoch 5/5\n",
      "72000/72000 [==============================] - 42s 582us/step - loss: 0.6813 - acc: 0.5640 - val_loss: 0.7027 - val_acc: 0.4965\n",
      "y22_pred:  [[1.]\n",
      " [1.]\n",
      " [0.]\n",
      " ...\n",
      " [1.]\n",
      " [1.]\n",
      " [1.]]\n",
      "pod 1st:  0.7808716707021792\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 72000 samples, validate on 18001 samples\n",
      "Epoch 1/5\n",
      "72000/72000 [==============================] - 42s 590us/step - loss: 0.6892 - acc: 0.5355 - val_loss: 0.6825 - val_acc: 0.5675\n",
      "Epoch 2/5\n",
      "72000/72000 [==============================] - 42s 587us/step - loss: 0.6860 - acc: 0.5508 - val_loss: 0.6949 - val_acc: 0.5087\n",
      "Epoch 3/5\n",
      "72000/72000 [==============================] - 43s 592us/step - loss: 0.6841 - acc: 0.5541 - val_loss: 0.7021 - val_acc: 0.5001\n",
      "Epoch 4/5\n",
      "72000/72000 [==============================] - 41s 574us/step - loss: 0.6815 - acc: 0.5656 - val_loss: 0.7044 - val_acc: 0.4930\n",
      "Epoch 5/5\n",
      "72000/72000 [==============================] - 44s 610us/step - loss: 0.6796 - acc: 0.5675 - val_loss: 0.6979 - val_acc: 0.5105\n",
      "y22_pred:  [[1.]\n",
      " [1.]\n",
      " [1.]\n",
      " ...\n",
      " [1.]\n",
      " [0.]\n",
      " [1.]]\n",
      "pod 1st:  0.7138821630347054\n",
      "pod:  [0.7318942909017551, 0.7653822876739964, 0.5724374495560937, 0.5799031476997578, 0.6646489104116223, 0.6180387409200968, 0.7173123486682809, 0.4261501210653753, 0.7808716707021792, 0.7138821630347054]\n",
      "pof:  [0.5864393338620143, 0.6245043616177637, 0.47006344171292624, 0.4411181601903251, 0.5721649484536082, 0.5249801744647106, 0.6748612212529739, 0.334853291038858, 0.6874876065833829, 0.6067816775728733]\n",
      "auc:  [0.5727274785198704, 0.5704389630281164, 0.5511870039215836, 0.5693924937547163, 0.546241980979007, 0.5465292832276931, 0.5212255637076535, 0.5456484150132587, 0.5466920320593982, 0.553550242730916]\n",
      "22580 27858 16997 32565\n"
     ]
    }
   ],
   "source": [
    "#RNN\n",
    "def get_RNN_Predict(X2_train, X2_test, y2_train, y2_test):\n",
    "    import pandas as pd\n",
    "    from keras.models import Sequential\n",
    "    from keras.layers import Dense,Dropout, LSTM, GRU\n",
    "    from keras.layers import Embedding\n",
    "    max_features = 10000 # number of words to consider as features\n",
    "    import numpy as np\n",
    "    #create model \n",
    "    model = Sequential()\n",
    "    model.add(Embedding(max_features, 32))\n",
    "    model.add(LSTM(32))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(optimizer='rmsprop',loss='binary_crossentropy', metrics=['acc'])\n",
    "    history = model.fit(X2_train, y2_train, epochs=5, batch_size=128, validation_split=0.2)\n",
    "    y2_pred = model.predict(X2_test)\n",
    "    y22_pred=y2_pred.round()\n",
    "    print('y22_pred: ',y22_pred)\n",
    "    return y22_pred\n",
    "\n",
    "pod_list = []\n",
    "pof_list = []\n",
    "auc_val_list = []\n",
    "tn_list= []\n",
    "fp_list= []\n",
    "fn_list= []\n",
    "tp_list= []\n",
    "    \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "folds = StratifiedKFold(n_splits=10)\n",
    "for train_index, test_index in folds.split(df_x_train_chi,y_train):\n",
    "    X2_train, X2_test, y2_train, y2_test=df_x_train_chi.iloc[train_index], df_x_train_chi.iloc[test_index],y_train.iloc[train_index], y_train.iloc[test_index]\n",
    "    y_pred = get_RNN_Predict(X2_train, X2_test, y2_train, y2_test)\n",
    "    tn, fp, fn, tp  = confusion_matrix(y2_test, y_pred).ravel()\n",
    "    tn_list.append(tn)\n",
    "    fp_list.append(fp)\n",
    "    fn_list.append(fn)\n",
    "    tp_list.append(tp)\n",
    "    \n",
    "    pod=tp/(tp+fn)\n",
    "    print('pod 1st: ',pod)\n",
    "    pof=fp/(fp+tn)\n",
    "    auc_val=(1+pod-pof)/2\n",
    "    #break\n",
    "    pod_list.append(pod)\n",
    "    pof_list.append(pof)\n",
    "    auc_val_list.append(auc_val)\n",
    "\n",
    "print('pod: ',pod_list)\n",
    "print ('pof: ',pof_list)\n",
    "print ('auc: ',auc_val_list)\n",
    "\n",
    "tn=sum(tn_list)\n",
    "fp=sum(fp_list) \n",
    "fn=sum(fn_list) \n",
    "tp=sum(tp_list)\n",
    "print(tn, fp, fn, tp)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}