Random Forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from IPython.display import display, clear_output\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>hypertension</th>\n",
       "      <th>heart_disease</th>\n",
       "      <th>ever_married</th>\n",
       "      <th>work_type</th>\n",
       "      <th>Residence_type</th>\n",
       "      <th>avg_glucose_level</th>\n",
       "      <th>bmi</th>\n",
       "      <th>smoking_status</th>\n",
       "      <th>stroke</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>30669</td>\n",
       "      <td>Male</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>children</td>\n",
       "      <td>Rural</td>\n",
       "      <td>95.12</td>\n",
       "      <td>18.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30468</td>\n",
       "      <td>Male</td>\n",
       "      <td>58.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Private</td>\n",
       "      <td>Urban</td>\n",
       "      <td>87.96</td>\n",
       "      <td>39.2</td>\n",
       "      <td>never smoked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16523</td>\n",
       "      <td>Female</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>Private</td>\n",
       "      <td>Urban</td>\n",
       "      <td>110.89</td>\n",
       "      <td>17.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56543</td>\n",
       "      <td>Female</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Private</td>\n",
       "      <td>Rural</td>\n",
       "      <td>69.04</td>\n",
       "      <td>35.9</td>\n",
       "      <td>formerly smoked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>46136</td>\n",
       "      <td>Male</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>Never_worked</td>\n",
       "      <td>Rural</td>\n",
       "      <td>161.28</td>\n",
       "      <td>19.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  gender   age  hypertension  heart_disease ever_married  \\\n",
       "0  30669    Male   3.0             0              0           No   \n",
       "1  30468    Male  58.0             1              0          Yes   \n",
       "2  16523  Female   8.0             0              0           No   \n",
       "3  56543  Female  70.0             0              0          Yes   \n",
       "4  46136    Male  14.0             0              0           No   \n",
       "\n",
       "      work_type Residence_type  avg_glucose_level   bmi   smoking_status  \\\n",
       "0      children          Rural              95.12  18.0              NaN   \n",
       "1       Private          Urban              87.96  39.2     never smoked   \n",
       "2       Private          Urban             110.89  17.6              NaN   \n",
       "3       Private          Rural              69.04  35.9  formerly smoked   \n",
       "4  Never_worked          Rural             161.28  19.1              NaN   \n",
       "\n",
       "   stroke  \n",
       "0       0  \n",
       "1       0  \n",
       "2       0  \n",
       "3       0  \n",
       "4       0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset= pd.read_csv(\"stroke prediction.csv\")\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.drop(columns=[\"id\",\"Residence_type\"],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[\"gender\"]=dataset[\"gender\"].astype('category')\n",
    "dataset[\"ever_married\"]=dataset[\"ever_married\"].astype('category')\n",
    "# dataset[\"Residence_type\"]=dataset[\"Residence_type\"].astype('category')\n",
    "dataset[\"work_type\"]=dataset[\"work_type\"].astype('category')\n",
    "dataset[\"smoking_status\"]=dataset.smoking_status.astype(\"category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_imputed=dataset.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gender               category\n",
       "age                   float64\n",
       "hypertension            int64\n",
       "heart_disease           int64\n",
       "ever_married         category\n",
       "work_type            category\n",
       "avg_glucose_level     float64\n",
       "bmi                   float64\n",
       "smoking_status       category\n",
       "stroke                  int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_imputed.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "encoder=LabelEncoder()\n",
    "\n",
    "dataset_imputed.gender=encoder.fit_transform(dataset_imputed.gender)\n",
    "\n",
    "dataset_imputed.ever_married=encoder.fit_transform(dataset_imputed.ever_married)\n",
    "\n",
    "dataset_imputed.work_type=encoder.fit_transform(dataset_imputed.work_type)\n",
    "\n",
    "# dataset_imputed.Residence_type=encoder.fit_transform(dataset_imputed.Residence_type)\n",
    "\n",
    "dataset_imputed.smoking_status=dataset_imputed.smoking_status.map({'never smoked':0,'formerly smoked':1,'smokes':2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=dataset_imputed.drop(columns=\"stroke\",axis=1)\n",
    "y=dataset_imputed[\"stroke\"]\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train,X_test,y_train,y_test= train_test_split(X,y,test_size=0.33,random_state=88,stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.impute import KNNImputer\n",
    "\n",
    "imputer=KNNImputer()\n",
    "\n",
    "X_trimputed=pd.DataFrame(imputer.fit_transform(X_train),columns=X_train.columns)\n",
    "\n",
    "X_teimputed=pd.DataFrame(imputer.transform(X_test),columns=X_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_trimputed[\"smoking_status\"]=X_trimputed[\"smoking_status\"].apply(round)\n",
    "X_trimputed[\"smoking_status\"].value_counts()\n",
    "\n",
    "# X_trimputed[\"smoking_status\"]=X_trimputed[\"smoking_status\"].astype('category')\n",
    "X_trimputed[\"work_type\"]=X_trimputed[\"work_type\"].apply(int)\n",
    "# X_trimputed[\"work_type\"]=X_trimputed[\"work_type\"].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_teimputed[\"smoking_status\"]=X_teimputed[\"smoking_status\"].apply(round)\n",
    "X_teimputed[\"smoking_status\"].value_counts()\n",
    "\n",
    "# X_teimputed[\"smoking_status\"]=X_teimputed[\"smoking_status\"].astype('category')\n",
    "X_teimputed[\"work_type\"]=X_teimputed[\"work_type\"].apply(int)\n",
    "# X_teimputed[\"work_type\"]=X_teimputed[\"work_type\"].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_trimputed[\"gender\"]=X_trimputed[\"gender\"].astype(\"category\")\n",
    "X_trimputed[[\"hypertension\",\"heart_disease\",\"ever_married\"]]=X_trimputed[[\"hypertension\",\"heart_disease\",\"ever_married\"]].astype(\"int\")\n",
    "\n",
    "# X_teimputed[\"gender\"]=X_teimputed[\"gender\"].astype(\"category\")\n",
    "X_teimputed[[\"hypertension\",\"heart_disease\",\"ever_married\"]]=X_teimputed[[\"hypertension\",\"heart_disease\",\"ever_married\"]].astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_trimputed=pd.get_dummies(X_trimputed,drop_first=True)\n",
    "X_teimputed=pd.get_dummies(X_teimputed,drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scaler=MinMaxScaler()\n",
    "\n",
    "scaler.fit(X_trimputed)\n",
    "\n",
    "X_trimputed=pd.DataFrame(scaler.transform(X_trimputed),columns=X_trimputed.columns)\n",
    "\n",
    "X_teimputed=pd.DataFrame(scaler.transform(X_teimputed),columns=X_teimputed.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,roc_auc_score,f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>hypertension</th>\n",
       "      <th>heart_disease</th>\n",
       "      <th>ever_married</th>\n",
       "      <th>work_type</th>\n",
       "      <th>avg_glucose_level</th>\n",
       "      <th>bmi</th>\n",
       "      <th>smoking_status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.865723</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.187511</td>\n",
       "      <td>0.219114</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.987793</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.188400</td>\n",
       "      <td>0.196970</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.609375</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.061769</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.133301</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.125021</td>\n",
       "      <td>0.127040</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.731445</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.089731</td>\n",
       "      <td>0.219114</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29073</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.890137</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.065582</td>\n",
       "      <td>0.033800</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29074</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.523926</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.168658</td>\n",
       "      <td>0.226107</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29075</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.829102</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.640061</td>\n",
       "      <td>0.191142</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29076</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.279785</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.184926</td>\n",
       "      <td>0.209790</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29077</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.377441</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.094772</td>\n",
       "      <td>0.341492</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29078 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       gender       age  hypertension  heart_disease  ever_married  work_type  \\\n",
       "0         0.0  0.865723           0.0            0.0           1.0       0.50   \n",
       "1         0.0  0.987793           0.0            0.0           1.0       0.50   \n",
       "2         0.0  0.609375           0.0            0.0           1.0       0.50   \n",
       "3         0.5  0.133301           0.0            0.0           0.0       1.00   \n",
       "4         0.5  0.731445           0.0            0.0           1.0       0.00   \n",
       "...       ...       ...           ...            ...           ...        ...   \n",
       "29073     0.0  0.890137           0.0            0.0           1.0       0.00   \n",
       "29074     0.0  0.523926           0.0            0.0           0.0       0.00   \n",
       "29075     0.0  0.829102           1.0            0.0           1.0       0.75   \n",
       "29076     0.0  0.279785           0.0            0.0           0.0       0.50   \n",
       "29077     0.0  0.377441           0.0            0.0           1.0       0.75   \n",
       "\n",
       "       avg_glucose_level       bmi  smoking_status  \n",
       "0               0.187511  0.219114             1.0  \n",
       "1               0.188400  0.196970             0.0  \n",
       "2               0.061769  0.136364             0.5  \n",
       "3               0.125021  0.127040             0.0  \n",
       "4               0.089731  0.219114             1.0  \n",
       "...                  ...       ...             ...  \n",
       "29073           0.065582  0.033800             0.0  \n",
       "29074           0.168658  0.226107             0.0  \n",
       "29075           0.640061  0.191142             0.0  \n",
       "29076           0.184926  0.209790             0.5  \n",
       "29077           0.094772  0.341492             0.0  \n",
       "\n",
       "[29078 rows x 9 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_trimputed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating an Gradient boosting instance\n",
    "clf = RandomForestClassifier(random_state=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
       "                       criterion='gini', max_depth=None, max_features='auto',\n",
       "                       max_leaf_nodes=None, max_samples=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                       n_jobs=None, oob_score=False, random_state=40, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#training the model\n",
    "clf.fit(X_trimputed,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred=clf.predict(X_trimputed)\n",
    "y_test_pred=clf.predict(X_teimputed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     28553\n",
      "           1       1.00      1.00      1.00       525\n",
      "\n",
      "    accuracy                           1.00     29078\n",
      "   macro avg       1.00      1.00      1.00     29078\n",
      "weighted avg       1.00      1.00      1.00     29078\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_train,y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99     14064\n",
      "           1       0.00      0.00      0.00       258\n",
      "\n",
      "    accuracy                           0.98     14322\n",
      "   macro avg       0.49      0.50      0.50     14322\n",
      "weighted avg       0.96      0.98      0.97     14322\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_prob=clf.predict_proba(X_trimputed)[:,1]\n",
    "y_test_prob=clf.predict_proba(X_teimputed)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7580880261660978"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test,y_test_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV,RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters={'class_weight':[None,'balanced']\n",
    "            ,'n_estimators':np.arange(100,1000,100),\n",
    "            'max_depth':np.arange(2,20,1)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "search=RandomizedSearchCV(RandomForestClassifier(random_state=40,n_jobs=-1),parameters,n_jobs=-1,scoring='roc_auc',cv=5,verbose=2,n_iter=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 100 candidates, totalling 500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:   48.7s\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 341 tasks      | elapsed: 12.8min\n"
     ]
    }
   ],
   "source": [
    "search.fit(X_trimputed,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8496261005653265"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': 1,\n",
       " 'max_depth': 2,\n",
       " 'learning_rate': 0.8999999999999999,\n",
       " 'gamma': 2.2,\n",
       " 'base_score': 0.2}"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating an Gradient boosting instance\n",
    "clf = XGBClassifier(random_state=96, max_depth=9, learning_rate=2.3, gamma=2.2, base_score=0.3, n_estimators=200, subsample=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.3, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=2.2, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=2.3, max_delta_step=0, max_depth=9,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=200, n_jobs=0, num_parallel_tree=1,\n",
       "              objective='binary:logistic', random_state=96, reg_alpha=0,\n",
       "              reg_lambda=1, scale_pos_weight=1, subsample=0.7,\n",
       "              tree_method='exact', validate_parameters=1, verbosity=None)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#training the model\n",
    "clf.fit(X_trimputed,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred=clf.predict(X_trimputed)\n",
    "y_test_pred=clf.predict(X_teimputed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_prob=clf.predict_proba(X_trimputed)[:,1]\n",
    "y_test_prob=clf.predict_proba(X_teimputed)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.98      0.99     28553\n",
      "           1       0.25      0.28      0.26       525\n",
      "\n",
      "    accuracy                           0.97     29078\n",
      "   macro avg       0.62      0.63      0.62     29078\n",
      "weighted avg       0.97      0.97      0.97     29078\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_train,y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98     14064\n",
      "           1       0.07      0.09      0.08       258\n",
      "\n",
      "    accuracy                           0.96     14322\n",
      "   macro avg       0.53      0.53      0.53     14322\n",
      "weighted avg       0.97      0.96      0.96     14322\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5967858174370101"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test,y_test_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search (F1 score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV,RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters={'subsample':[0.7,0.8,0.9,1],\n",
    "            'base_score':[0.2,0.3,0.4,0.5,0.6],\n",
    "            'gamma':np.arange(0,2.5,0.1),\n",
    "            'learning_rate':np.arange(0.5,2.5,0.1),\n",
    "            'max_depth':[2,3,4,5,6,7,8,9,10]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "search=RandomizedSearchCV(XGBClassifier(random_state=40),parameters,scoring='f1',cv=5,n_jobs=-1,verbose=2,n_iter=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 500 candidates, totalling 2500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:    8.7s\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed:   38.7s\n",
      "[Parallel(n_jobs=-1)]: Done 341 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 624 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 989 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1434 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1961 tasks      | elapsed:  8.4min\n",
      "[Parallel(n_jobs=-1)]: Done 2500 out of 2500 | elapsed: 10.6min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, error_score=nan,\n",
       "                   estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                           colsample_bylevel=None,\n",
       "                                           colsample_bynode=None,\n",
       "                                           colsample_bytree=None, gamma=None,\n",
       "                                           gpu_id=None, importance_type='gain',\n",
       "                                           interaction_constraints=None,\n",
       "                                           learning_rate=None,\n",
       "                                           max_delta_step=None, max_depth=None,\n",
       "                                           min_child_weight=None, missing=nan,\n",
       "                                           monotone_constraints=None,\n",
       "                                           n...\n",
       "                                        'gamma': array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. , 1.1, 1.2,\n",
       "       1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2. , 2.1, 2.2, 2.3, 2.4]),\n",
       "                                        'learning_rate': array([0.5, 0.6, 0.7, 0.8, 0.9, 1. , 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7,\n",
       "       1.8, 1.9, 2. , 2.1, 2.2, 2.3, 2.4]),\n",
       "                                        'max_depth': [2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                      10],\n",
       "                                        'subsample': [0.7, 0.8, 0.9, 1]},\n",
       "                   pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
       "                   return_train_score=False, scoring='roc_auc', verbose=2)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.fit(X_trimputed,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8496261005653265"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': 1,\n",
       " 'max_depth': 2,\n",
       " 'learning_rate': 0.8999999999999999,\n",
       " 'gamma': 2.2,\n",
       " 'base_score': 0.2}"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_params_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}