XGBoost_Yiyang_Zeng_Github.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>XGBOOST Mini-Version</h2>\n",
    "<h3>Yiyang \"Joe\" Zeng</h3>\n",
    "<h3>Here I tried to create a simplified version of the XGBOOST machine that was originally proposed by Tianqi Chen and Carlos Guestrin. For the original paper, please refer to https://arxiv.org/abs/1603.02754</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, roc_curve, auc\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Step 1: Load Dataset</h2>\n",
    "<p> a. load dataset</p>\n",
    "<p> b. check missing values</p>\n",
    "<p> c. split dataset into 80% train and 20% test sets </p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>days_since_last_loan_expiration</th>\n",
       "      <th>number_of_monthly_transactions</th>\n",
       "      <th>length_of_customer_relationship</th>\n",
       "      <th>number_of_products</th>\n",
       "      <th>size_of_transactions</th>\n",
       "      <th>attrition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22</td>\n",
       "      <td>30</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>100.56</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>75.04</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34</td>\n",
       "      <td>26</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>53.44</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>29</td>\n",
       "      <td>34</td>\n",
       "      <td>4</td>\n",
       "      <td>57.34</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>32</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>158.48</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   days_since_last_loan_expiration  number_of_monthly_transactions  \\\n",
       "0                               22                              30   \n",
       "1                               17                              28   \n",
       "2                               34                              26   \n",
       "3                               12                              29   \n",
       "4                               10                              32   \n",
       "\n",
       "   length_of_customer_relationship  number_of_products  size_of_transactions  \\\n",
       "0                               12                   3                100.56   \n",
       "1                               22                   1                 75.04   \n",
       "2                               25                   4                 53.44   \n",
       "3                               34                   4                 57.34   \n",
       "4                                4                   3                158.48   \n",
       "\n",
       "   attrition  \n",
       "0          0  \n",
       "1          0  \n",
       "2          0  \n",
       "3          0  \n",
       "4          0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data\n",
    "dt = pd.read_csv(\"customer_attrition.csv\")\n",
    "dt = dt[['days_since_last_loan_expiration','number_of_monthly_transactions','length_of_customer_relationship','number_of_products','size_of_transactions','attrition']]\n",
    "dt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "days_since_last_loan_expiration    0\n",
       "number_of_monthly_transactions     0\n",
       "length_of_customer_relationship    0\n",
       "number_of_products                 0\n",
       "size_of_transactions               0\n",
       "attrition                          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check missing values\n",
    "dt.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define X and Y variables\n",
    "y = dt.iloc[: , -1]\n",
    "X = dt.iloc[: , :(dt.shape[1]-1)]\n",
    "\n",
    "# train & test split\n",
    "# 80% training data + 20% test data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Step 2: Define Helper Functions for 1st & 2nd Derivatives of the Logistic Loss Function</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# refer to https://github.com/dmlc/xgboost/blob/master/src/objective/regression_loss.h\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "# first order gradient\n",
    "def grad(preds, labels):\n",
    "    preds = sigmoid(preds)\n",
    "    return (preds - labels)\n",
    "\n",
    "# second order gradient\n",
    "def hess(preds, labels):\n",
    "    preds = sigmoid(preds)\n",
    "    return (preds * (1 - preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Step 3: Define a TreeNode Object</h2>\n",
    "<p>This is an object for a node in a tree model.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TreeNode(object):\n",
    "    def __init__(self, is_leaf = False, leaf_score = None, split_feature = None, split_threshold = None, \n",
    "                 left_child = None, right_child = None, NA_direction = 'left'):\n",
    "        self.is_leaf = is_leaf # if True, the node is a leaf and no further split necessary\n",
    "        self.leaf_score = leaf_score # prediction score\n",
    "        self.split_feature = split_feature\n",
    "        self.split_threshold = split_threshold\n",
    "        self.left_child = left_child\n",
    "        self.right_child = right_child\n",
    "        self.NA_direction = NA_direction # whether to put NA on left or right"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Step 4: Define a Single Tree Object</h2>\n",
    "<p>This is an object for a tree model.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tree(object):\n",
    "    def __init__(self, root = None, min_sample_split = None, col_sub_frac = None, lamda = None, \n",
    "                 gamma = None, num_thread = None, min_child_weight = None):\n",
    "        self.root = root\n",
    "        self.min_sample_split = min_sample_split # minimum samples required for a split\n",
    "        self.col_sub_frac = col_sub_frac # column subsampling fraction\n",
    "        self.lamda = lamda #lambda\n",
    "        self.gamma = gamma #gamma\n",
    "        self.min_child_weight = min_child_weight # minimum weight for a split\n",
    "    \n",
    "    def cal_leaf_score(self, Y):\n",
    "        # calculate score for a leaf\n",
    "        # Formula (5): -G / (H + Lambda)\n",
    "        return - (Y['grad'].sum() / (Y['hess'].sum() + self.lamda))\n",
    "    \n",
    "    def cal_split_gain(self, left_Y, right_Y, NA_grad, NA_hess, NA_direction = 'left'):\n",
    "        # calculate gain from a potential split\n",
    "        # Formula (7): 0.5 * [GL^2 / (HL+Lambda) + GR^2 / (HR+Lambda) - G^2 / (H+Lambda)] - Gamma\n",
    "        if (NA_direction == 'left'):\n",
    "            GL = left_Y['grad'].sum() + NA_grad\n",
    "            HL = left_Y['hess'].sum() + NA_hess\n",
    "            GR = right_Y['grad'].sum()\n",
    "            HR = right_Y['hess'].sum()\n",
    "        else:\n",
    "            GL = left_Y['grad'].sum()\n",
    "            HL = left_Y['hess'].sum()\n",
    "            GR = right_Y['grad'].sum() + NA_grad\n",
    "            HR = right_Y['hess'].sum() + NA_hess\n",
    "        gain = 0.5 * ( (GL**2/(HL+self.lamda)) + (GR**2/(HR+self.lamda)) - ((GL+GR)**2/(HL+HR+self.lamda)) ) - self.gamma\n",
    "        return gain\n",
    "    \n",
    "    def weighted_quantile_sketch(self, dt, feature):\n",
    "        # find the best split value for a given feature\n",
    "        best_split_value = None\n",
    "        best_gain = -np.inf\n",
    "        best_NA_direction = 'left'\n",
    "        \n",
    "        selected_dt = dt[[feature, 'label', 'grad', 'hess']]\n",
    "        \n",
    "        # divide selected_dt into data with/without NA\n",
    "        mask = selected_dt[feature].isnull()\n",
    "        NA_dt = selected_dt[mask]\n",
    "        Non_NA_dt = selected_dt[~mask]\n",
    "        \n",
    "        # for NA data, calculate NA_grad & NA_hess\n",
    "        NA_grad = NA_dt['grad'].sum()\n",
    "        NA_hess = NA_dt['hess'].sum()\n",
    "        \n",
    "        # for non NA data, order data by feature value\n",
    "        Non_NA_dt.reset_index(inplace = True)\n",
    "        Non_NA_dt['feature_index'] = Non_NA_dt[feature].argsort() # assign new index based on feature value\n",
    "        Non_NA_dt = Non_NA_dt.iloc[Non_NA_dt['feature_index']] # sort\n",
    "        \n",
    "        # weighted quantile sketch\n",
    "        hess_sum = Non_NA_dt['hess'].sum()\n",
    "        # formula (8)\n",
    "        Non_NA_dt['rank'] = Non_NA_dt.apply(lambda x : (1/hess_sum)*sum(Non_NA_dt[Non_NA_dt[feature] < x[feature]]['hess']),\n",
    "                                            axis=1)\n",
    "\n",
    "        # loop over feature value to find split gain\n",
    "        for j in range(Non_NA_dt.shape[0]-1):\n",
    "            \n",
    "            # look at the current rank and the next rank\n",
    "            rk_sk_j, rk_sk_j_1 = Non_NA_dt['rank'].iloc[j:j+2]\n",
    "            \n",
    "            # formula (9)\n",
    "            # compare | rk(sk,j) - rk(sk,j+1) | with eps\n",
    "            if (abs(rk_sk_j-rk_sk_j_1) >= self.eps):\n",
    "                continue\n",
    "            \n",
    "            # if | rk(sk,j) - rk(sk,j+1) | < eps, then propose a split\n",
    "            split_value = (Non_NA_dt[feature].iloc[j+1] + Non_NA_dt[feature].iloc[j])/2\n",
    "                \n",
    "            left_Y = Non_NA_dt.iloc[:(j+1)] # observations before this point are put in the left tree\n",
    "            right_Y = Non_NA_dt.iloc[(j+1):] # observations after this point are put in the right tree\n",
    "            \n",
    "            # decide on where to put NA\n",
    "            go_left = self.cal_split_gain(left_Y, right_Y, NA_grad, NA_hess, NA_direction = 'left')\n",
    "            go_right = self.cal_split_gain(left_Y, right_Y, NA_grad, NA_hess, NA_direction = 'right')\n",
    "            \n",
    "            if (go_left > go_right):\n",
    "                this_gain = go_left\n",
    "                this_direction = 'left'\n",
    "            else:\n",
    "                this_gain = go_right\n",
    "                this_direction = 'right'\n",
    "            \n",
    "            # always choose the best gain\n",
    "            if (this_gain > best_gain):\n",
    "                best_split_value = split_value\n",
    "                best_gain = this_gain\n",
    "                best_NA_direction = this_direction\n",
    "        \n",
    "        return feature, best_split_value, best_gain, best_NA_direction\n",
    "    \n",
    "    def find_best_split_value_and_feature(self, X, Y):\n",
    "        best_gain = -np.inf\n",
    "        best_feature, best_split_value, results = None, None, None\n",
    "        \n",
    "        features = list(X.columns) # get a list of all features\n",
    "        data = pd.concat([X, Y], axis = 1)\n",
    "        \n",
    "        results = []\n",
    "        for j in range(len(features)):\n",
    "            results.append(self.weighted_quantile_sketch(data, features[j]))\n",
    "        \n",
    "        # find the best split\n",
    "        best = sorted(results, key = lambda x: float(x[2]), reverse = True)[0]\n",
    "        best_feature = best[0]\n",
    "        best_split_value = best[1]\n",
    "        best_gain = best[2]\n",
    "        best_NA_direction = best[3]\n",
    "        \n",
    "        return best_feature, best_split_value, best_gain, best_NA_direction\n",
    "    \n",
    "    def split(self, X, Y, feature, split_value, NA_direction):\n",
    "        data = pd.concat([X, Y], axis = 1)\n",
    "        X_cols, Y_cols = list(X.columns), list(Y.columns)\n",
    "        \n",
    "        if(NA_direction == 'left'):\n",
    "            mask = (data[feature] >= split_value)\n",
    "            left = data[~mask] # left take all NA\n",
    "            right = data[mask]\n",
    "        else:\n",
    "            mask = (data[feature] < split_value)\n",
    "            left = data[mask]\n",
    "            right = data[~mask] # right take all NA\n",
    "        \n",
    "        return left[X_cols], left[Y_cols], right[X_cols], right[Y_cols]\n",
    "    \n",
    "    def build_tree(self, X, Y, depth):\n",
    "        # return a tree node when cannot split\n",
    "        if (X.shape[0] < self.min_sample_split) or (depth == 0) or (Y['hess'].sum() < self.min_child_weight):\n",
    "            l_score = self.cal_leaf_score(Y)\n",
    "            return TreeNode(is_leaf = True, leaf_score = l_score)\n",
    "        \n",
    "        # column sub-sampling to reduce overfitting\n",
    "        X_sub = X.sample(frac = self.col_sub_frac, axis = 1)\n",
    "        best_feature, best_split_value, best_gain, best_NA_direction = self.find_best_split_value_and_feature(X_sub, Y)\n",
    "        \n",
    "        if (best_gain <= 0):\n",
    "            l_score = self.cal_leaf_score(Y)\n",
    "            return TreeNode(is_leaf = True, leaf_score = l_score)\n",
    "        \n",
    "        # split data\n",
    "        left_X, left_Y, right_X, right_Y = self.split(X_sub, Y, best_feature, best_split_value, best_NA_direction)\n",
    "        left_child = self.build_tree(left_X, left_Y, depth - 1)\n",
    "        right_child = self.build_tree(right_X, right_Y, depth - 1)\n",
    "        \n",
    "        # merge two trees together\n",
    "        sub_tree = TreeNode(is_leaf = False, leaf_score = None, split_feature = best_feature, split_threshold = best_split_value, \n",
    "                 left_child = left_child, right_child = right_child, NA_direction = best_NA_direction)\n",
    "        \n",
    "        return sub_tree\n",
    "    \n",
    "    def fit(self, X, Y, max_depth = 3, min_child_weight = 1, col_sub_frac = 1, min_sample_split = 10, \n",
    "            lamda = 1, gamma = 0.05, eps = 0.001):\n",
    "        self.min_child_weight = min_child_weight\n",
    "        self.col_sub_frac = col_sub_frac\n",
    "        self.min_sample_split = min_sample_split\n",
    "        self.lamda = lamda\n",
    "        self.gamma = gamma\n",
    "        self.eps = eps\n",
    "        self.root = self.build_tree(X, Y, max_depth)\n",
    "    \n",
    "    def predict_one(self, tree_node, X):\n",
    "        # predict one observation\n",
    "        # print predicted score at leaf\n",
    "        \n",
    "        if tree_node.is_leaf == True:\n",
    "            return tree_node.leaf_score\n",
    "        # if the observation has a missing value for this feature and NA direction is left\n",
    "        elif (type(X[tree_node.split_feature].item()) != int) and (type(X[tree_node.split_feature].item()) != float) and (tree_node.NA_direction == 'left'):\n",
    "            return self.predict_one(tree_node.left_child, X)\n",
    "        \n",
    "        # if value of the feature is less than the split value\n",
    "        elif ((X[tree_node.split_feature] < tree_node.split_threshold).item()):\n",
    "            return self.predict_one(tree_node.left_child, X)\n",
    "        \n",
    "        # all else\n",
    "        else:\n",
    "            return self.predict_one(tree_node.right_child, X)\n",
    "        \n",
    "    def predict(self, X):\n",
    "        # make predictions\n",
    "        preds = []\n",
    "        for n in range(X.shape[0]):\n",
    "            preds.append(self.predict_one(self.root, X.iloc[[n]]))\n",
    "        return np.array(preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Step 5: Define a XGBOOST Train Function</h2>\n",
    "<p>This function is to fit a xgboost machine to training data.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xgboost_train(X, Y, eta, max_round, max_depth, row_sub_frac, col_sub_frac, min_child_weight, \n",
    "                  min_sample_split, lamda, gamma, eps, metric):\n",
    "    \n",
    "    # initialize variables\n",
    "    trees = [] # tree models in this xgboost\n",
    "    initialize_pred = 1 # initialize a starting point for prediction\n",
    "    best_metric_value, best_round = -np.inf, None\n",
    "    metric_value_list = []\n",
    "    \n",
    "    # reset index for X & Y\n",
    "    X.reset_index(drop = True, inplace = True)\n",
    "    Y = Y.to_frame(name = 'label')\n",
    "    Y.reset_index(drop = True, inplace = True)\n",
    "    \n",
    "    # initialize Y\n",
    "    Y['y_pred'] = initialize_pred\n",
    "    Y['grad'] = grad(Y['y_pred'], Y['label'])\n",
    "    Y['hess'] = hess(Y['y_pred'], Y['label'])\n",
    "        \n",
    "    for i in range(max_round):\n",
    "        data = pd.concat([X, Y], axis = 1)\n",
    "        # in each boosting round, use row sub-sampling to reduce overfitting\n",
    "        data = data.sample(frac = row_sub_frac, axis = 0)\n",
    "            \n",
    "        Y_Selected = data[['label', 'y_pred', 'grad', 'hess']]\n",
    "        X_Selected = data[list(X.columns)]\n",
    "            \n",
    "        tree = Tree()\n",
    "        tree.fit(X_Selected, Y_Selected, max_depth = max_depth, min_child_weight = min_child_weight, col_sub_frac = col_sub_frac,\n",
    "                 min_sample_split = min_sample_split, lamda = lamda, gamma = gamma, eps = eps)\n",
    "        # predict on the whole training set\n",
    "        preds = tree.predict(X)\n",
    "        # eta is a shrinkage factor to prevent overfitting\n",
    "        Y['y_pred'] = Y['y_pred'] + eta * preds\n",
    "        Y['grad'] = grad(Y['y_pred'], Y['label'])\n",
    "        Y['hess'] = hess(Y['y_pred'], Y['label'])\n",
    "            \n",
    "        trees.append(tree)\n",
    "        \n",
    "        # print out current iteration and its corresponding metric value\n",
    "        print(\"Iteration: \", i)\n",
    "        test_perf = []\n",
    "        avg = Y['y_pred'].mean()\n",
    "        for j in Y['y_pred']:\n",
    "            if (j > avg):\n",
    "                test_perf.append(1)\n",
    "            else:\n",
    "                test_perf.append(0)\n",
    "        \n",
    "        # decide on metric to use\n",
    "        if (metric == 'f1'):\n",
    "            m = f1_score(Y['label'], test_perf)\n",
    "            print(\"F1-Score: \", m)\n",
    "            metric_value_list.append(m)\n",
    "        if (metric == 'accuracy'):\n",
    "            m = accuracy_score(Y['label'], test_perf)\n",
    "            print(\"Accuracy: \", m)\n",
    "        \n",
    "        # replace best metric\n",
    "        if (m > best_metric_value):\n",
    "            best_metric_value = m\n",
    "            best_round = i\n",
    "    \n",
    "    # too many training rounds may cause overfitting and/or decreased performance\n",
    "    # choose xgboost model when its training metric reaches its maximum\n",
    "    best_trees = trees[:(i+1)]\n",
    "    plt.plot(metric_value_list)\n",
    "    plt.show()\n",
    "    \n",
    "    return best_trees, eta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Step 6: Define a XGBOOST Predict Function</h2>\n",
    "<p>This function is to predict test data using trained xgboost machine.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xgboost_predict(trees, X_test, eta):\n",
    "    # initialize starting point for prediction\n",
    "    preds = np.ones(X_test.shape[0])\n",
    "    for tree in trees:\n",
    "        preds = preds + tree.predict(X_test) * eta\n",
    "    \n",
    "    adj_preds = []\n",
    "    avg = preds.mean()\n",
    "    for i in preds:\n",
    "        if (i > avg):\n",
    "            adj_preds.append(1)\n",
    "        else:\n",
    "            adj_preds.append(0)\n",
    "    \n",
    "    return adj_preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Step 7: Implement the Mini-XGBOOST Machine</h2>\n",
    "<p>This train function takes about 10-15 minutes to run for 80 training rounds. Let's see the performance.</p>\n",
    "<p>*Here, I did not do cross-validation and hyperparameter tunning for simplicity, as the purpose is to implement a mini-xgboost. I used some pre-defined hyperparameters.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration:  0\n",
      "F1-Score:  0.8762886597938144\n",
      "Iteration:  1\n",
      "F1-Score:  0.8772845953002611\n",
      "Iteration:  2\n",
      "F1-Score:  0.8877005347593583\n",
      "Iteration:  3\n",
      "F1-Score:  0.9282051282051282\n",
      "Iteration:  4\n",
      "F1-Score:  0.9390862944162437\n",
      "Iteration:  5\n",
      "F1-Score:  0.9567430025445293\n",
      "Iteration:  6\n",
      "F1-Score:  0.9543147208121828\n",
      "Iteration:  7\n",
      "F1-Score:  0.9494949494949495\n",
      "Iteration:  8\n",
      "F1-Score:  0.9545454545454546\n",
      "Iteration:  9\n",
      "F1-Score:  0.9569620253164557\n",
      "Iteration:  10\n",
      "F1-Score:  0.9569620253164557\n",
      "Iteration:  11\n",
      "F1-Score:  0.964824120603015\n",
      "Iteration:  12\n",
      "F1-Score:  0.9622166246851386\n",
      "Iteration:  13\n",
      "F1-Score:  0.9696969696969697\n",
      "Iteration:  14\n",
      "F1-Score:  0.9696969696969697\n",
      "Iteration:  15\n",
      "F1-Score:  0.9646464646464646\n",
      "Iteration:  16\n",
      "F1-Score:  0.9698492462311558\n",
      "Iteration:  17\n",
      "F1-Score:  0.9698492462311558\n",
      "Iteration:  18\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  19\n",
      "F1-Score:  0.9748743718592965\n",
      "Iteration:  20\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  21\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  22\n",
      "F1-Score:  0.9748743718592965\n",
      "Iteration:  23\n",
      "F1-Score:  0.9748743718592965\n",
      "Iteration:  24\n",
      "F1-Score:  0.9748743718592965\n",
      "Iteration:  25\n",
      "F1-Score:  0.9748743718592965\n",
      "Iteration:  26\n",
      "F1-Score:  0.9748743718592965\n",
      "Iteration:  27\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  28\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  29\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  30\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  31\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  32\n",
      "F1-Score:  0.9748743718592965\n",
      "Iteration:  33\n",
      "F1-Score:  0.9748743718592965\n",
      "Iteration:  34\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  35\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  36\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  37\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  38\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  39\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  40\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  41\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  42\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  43\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  44\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  45\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  46\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  47\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  48\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  49\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  50\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  51\n",
      "F1-Score:  0.9724310776942355\n",
      "Iteration:  52\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  53\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  54\n",
      "F1-Score:  0.975\n",
      "Iteration:  55\n",
      "F1-Score:  0.975\n",
      "Iteration:  56\n",
      "F1-Score:  0.975\n",
      "Iteration:  57\n",
      "F1-Score:  0.975\n",
      "Iteration:  58\n",
      "F1-Score:  0.975\n",
      "Iteration:  59\n",
      "F1-Score:  0.975\n",
      "Iteration:  60\n",
      "F1-Score:  0.975\n",
      "Iteration:  61\n",
      "F1-Score:  0.975\n",
      "Iteration:  62\n",
      "F1-Score:  0.975\n",
      "Iteration:  63\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  64\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  65\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  66\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  67\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  68\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  69\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  70\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  71\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  72\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  73\n",
      "F1-Score:  0.9774436090225564\n",
      "Iteration:  74\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  75\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  76\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  77\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  78\n",
      "F1-Score:  0.9773299748110832\n",
      "Iteration:  79\n",
      "F1-Score:  0.9773299748110832\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt81PWd7/HXJzdu4U6EQLgqAhGRS0SsbnHVtmhbrdW2oPZ23IM9rb3s1t0jpz3uLnt67Nm6u263tl3bta5aRaXata5d6qL2tFaQIIJyNaKSmADhlpCEXCb57B/zCwxhkAGSzDDf9/PxyIP5/X7fmflMZnjnO9/f7/f9mbsjIiJhyEl3ASIi0nsU+iIiAVHoi4gERKEvIhIQhb6ISEAU+iIiAVHoi4gERKEvIhIQhb6ISEDy0l1AVyNGjPAJEyakuwwRkTPK2rVr97h70YnaZVzoT5gwgfLy8nSXISJyRjGzd1Npl9LwjpktMLOtZlZhZnck2T7ezFaa2QYze9HMShK2/a2ZbTSzzWb2fTOz1F+GiIh0pxOGvpnlAvcCVwGlwCIzK+3S7G7gQXefASwF7oru+wHgEmAGMB24EJjfbdWLiMhJSaWnPxeocPft7t4KLAOu7dKmFFgZ3X4hYbsDfYECoA+QD+w63aJFROTUpBL6Y4DKhOWqaF2i9cD10e3rgIFmNtzdXyb+R6Am+lnh7pu7PoGZLTazcjMrr62tPdnXICIiKUol9JONwXedhP92YL6ZrSM+fPMeEDOzc4BpQAnxPxSXm9kHj3kw9/vcvczdy4qKTrjzWURETlEqR+9UAWMTlkuA6sQG7l4NfBLAzAqB6929zswWA6vcvSHa9mtgHvD/u6F2ERE5San09NcAk81sopkVAAuBpxMbmNkIM+t8rCXA/dHtHcS/AeSZWT7xbwHHDO+IiEjvOGFP391jZnYbsALIBe53941mthQod/engcuAu8zMiffivxLdfTlwOfA68SGh/3D3X3X/y5BT9eaug1QdOHR4uX9+LhdOGEZOzpl9ZO2m6np2HWxOuf2YIf04d+TAHqwo/VZv38tLFXuOrDDjo+cXM2VUZr3ufY2tPPrKDlra2tNdSlIFeTkM7JvPwL55DOybT15u9/1fGdQ3jznjh3Xb4yVjmXaN3LKyMtfJWb3jxa27+eIDa+j6EfjiJRP4y4+fl56iusEzG6q57ZF1J3Wfvvk5vHzHFQwdUNBDVaVXe4dzyXefZ2d9M51nyrjD8AEFPP3VSxkzpF96C4y0xjq48SerKH93P5l4Rk9Px+XMsUP45VcuOaX7mtlady87UbuMOyNXesc7exr52qPrmDpqEP/nE9Pp7Nj/4tUqfvbSO0wfPZjr55S8/4NkoE3V9fz5ExuYM34o3/rotKRHIXS1q76ZLz38Kr94tYo/+aNJPV5jOvx222521jfz45tns2B6MQBv1TbwiR+8xOIHy1n+pQ/QryA3zVXCX/1qI+Xv7uefFs3i4xeMTnc5SbXE2jnYHKOhOcbB5hixjo5ue+wBfXo+khX6AWpoibH4oXJycoz7PjuHscP6H952/pjBvLW7kSVPvc7kkYXMKBmSxkpPzv7GVm59uJxB/fL40c2zOWtg35TvO2vcEB55ZQe3XDqRbDxp/NFXKhlR2Icrpo08vO7sokL+cdFMbvnXcu54cgP3fGZmWl/7z1e/yyOrd/A/Ljs7YwMfoE9eLn0KcxlR2CfdpZwSzbIZmI4O55uPv0bF7gbuvXH2UYEPkJebww9unEVRYR9ufWgttQdb0lTpyYm1d/C1ZevYVdfCj2+ec1KBD3Dj3HFsr21k9dv7eqjC9Nld38zzW3Zzw5wS8nOP/i9/+dSR3P7hKfzba9X85Hfb01QhlL+zj796eiPzzy3i9g9PSVsdIVBPPzD3vlDBio27+PZHp3HJOSOSthle2Id//uwcbvjxH/iTB8u5cupZp/RcZnD1+cVMKio8nZKTWr19L68kBPSWnQf53Zt7+NsbZjBr3NCTfryPzRjN0mc28cjqHcybNLw7S027J9ZW0d7hfObCsUm3f/mys9lYXcd3f72F326rxVIaFOteb1TXMWZIP76/cBa5Z/hBBJlOoR+QlZt38ff/uY1PzBzNLZdOfN+208cM5ns3XMDtT6xnfeWBU37Oh1a9y69uu5SzBp1cz/v9lL+zj5v/ZTVt7UfvVbv1g5P4dFnyYDuRfgW5XD+7hEdW72BvQwvDz9Cv7l11dDiPralk3qRhTBwxIGkbM+N7N1xAQW4OlfsPJW3T02aUDOHOj01jcP/8tDx/SBT6gajY3cDXl73GeaMH8d3rZ6Q0dvvxC0Zz9fnFnOoRXtt2NXD9j/7ArQ+vZdniefTJO/0dhTV1h/jSw68yZkg/nvzyJQzqe+QjnJd7eqOVN140jgf+8A7L11Zx6/yzT7fUjPDy9r3s2NfEn33o3PdtN6BPHvcsnNVLVUk6aUw/APXNbSx+sJw+eTn882fL6Jufevjm5hh5uTmn9FM6ehB/9+kLWLfjAHf+cuMp//Ho1NzWzpceWsuh1hj3fa6MYQMKjnq+03XuyIGUjR/Ko6/soKMjsw5lPlWPvrKDwf3yWTB9VLpLkQyh0M9y7R3ON5a9xo59Tfzwptm9fjz21ecXc9sfn8Nj5ZU8vCqlazwk5e5866k3WF9Vx999emaPnUh140XjeGdvEy9v39sjj9+b9jW28puNu7hu1piT+kMv2U3DO1nuH57bxvNbdvM3157HRWnaQflnHzqXTTX1/PWvNvGfm3ef0kk3Ta3tvPL2Pr52xeQe7bVefX4xS5/ZxJInX2dSUfIx8DPFnoYWWts7WDj31PZzSHZS6Gex57fs4gcvVPDpshJunjc+bXXk5Bj3LJzJ/1y+geoDp76j8IuXTOAbV0zuxsqO1Tc/l9s/PIUnyivZ39jao8/V03LNuHneOKaOGpTuUiSDaBqGLFW1v4mPfv/3jB7Sj6e+/AF9vRfJcqlOw6Ax/SzUEmvnK4+so6PD+dFNsxX4InKYhney0P/9982srzzAj2+ezYTjHJstImFS6GegV97eR+W+psPLebnGR84blbTHvm3XQf7p+YrDhxi2xNr5z827ueXSiYcn1hIR6aTQzzB1TW0s+skq2rscJ77kqqlJTxj6p+crWLFxJ+MS5tD5xMzR3HHV1B6vVUTOPAr9DPPy9j20R2Px540eDMA3HlvHY2sqWfzBSUedSVvX1MaKjTtZeOFYll47PV0li8gZRDtyM8xLFXsZUJDLlaUjGTe8P+OG92fR3HFs39NI+bv7j2r79IZqWmMdfGqOjsMWkdQo9DPMSxV7mDtx2FFT4H50RjGFffJY9krlUW2Xl1cyddRApo/RcdgikhqFfgapPnCI7Xsaj5nyuH9BHtfMHM2/v15NfXMbEN+Bu76qjhvmlGTlRT9EpGco9DNI50Wrk81zv/DCsTS3dfD0a9UAPFFeSV6Ocd2sMb1ao4ic2RT6GeQPb+1lRGEBU5JMJnb+mMFMKx7EY2sqaWvv4Kl173H51LOyZt53EekdCv0M4e78vmIPF589gpwkVw4yMxZeOJbX36vjhy+8xZ6GVj51ihcMEZFwKfQzRMXuBmoPtnDpOcefCfMTM8dQkJfDPSu3MaKwgMumFPVihSKSDRT6afDT323nf//yjaMuKtI5nv+Bs5NftxZgcP98rp4+Cne4btaYYy5yLSJyIkqNXubu/Oyld3ho1bvc/9I7h9f/vmIv44f3Z2zCmbXJfOGSiYwc1IdFc8f1cKUiko0U+r2sav8h3jtwiKH987nr2c28umM/sfYOVm/f+769/E4zxw5h9f+6kklFhb1QrYhkG4V+L+u8DN99nytj1OC+fPWRdfzuzT0cbIlxaZJDNUVEupNCv5et2r6XYQMKKBs/lB/eNJvagy3c9sirAFx8dnouZygi4VDo9yJ3Z/X2fcybNAwzY0bJEL710Wk0trZz3uhBDBtQkO4SRSTLaZbNXlS5Lz6ef+v8SYfXfe7i8eyqb2ZasebPEZGel1JP38wWmNlWM6swszuSbB9vZivNbIOZvWhmJQnbxpnZb8xss5ltMrMJ3Vf+mWVVNJ5/8aQjwzhmxl8smMrHLxidrrJEJCAnDH0zywXuBa4CSoFFZlbapdndwIPuPgNYCtyVsO1B4HvuPg2YC+zujsLPRKu272X4gALOOUtH3ohIeqTS058LVLj7dndvBZYB13ZpUwqsjG6/0Lk9+uOQ5+7PAbh7g7s3ESB3Z9X2vcybNFyzYopI2qQS+mOAxIncq6J1idYD10e3rwMGmtlw4FzggJk9aWbrzOx70TeHo5jZYjMrN7Py2trak38VZ4DKfYeormtm3qRh6S5FRAKWSugn65Z6l+Xbgflmtg6YD7wHxIjvKP6jaPuFwCTgC8c8mPt97l7m7mVFRdk5n8zL2+PTLOiwTBFJp1RCvwpInM6xBKhObODu1e7+SXefBXwrWlcX3XddNDQUA34JzO6Wys8wq7bvY0RhAWfrTFoRSaNUQn8NMNnMJppZAbAQeDqxgZmNMLPOx1oC3J9w36Fm1tl9vxzYdPpln1k6x/Mv0ni+iKTZCUM/6qHfBqwANgOPu/tGM1tqZtdEzS4DtprZNmAk8J3ovu3Eh3ZWmtnrxIeKftLtryLD7djXRE1dM/MmaWhHRNIrpZOz3P1Z4Nku6+5MuL0cWH6c+z4HzDiNGtOio8Mx45R65nsbWvjFq1W0xjoA2LqrAYCLtRNXRNJMZ+Qex588WA7AfZ+dQ95JzFvf3NbOf3tgDeur6o5aP2XkQI3ni0jaKfSPY33lAfY2tvLdX2/h2x/rei5acu7OkidfZ31VHT++eTZXTBt5eFuumcbzRSTtNOFaEs1t7extbGX4gAJ++vu3+cXaqpTu99Pfvc1T697jzz50LgumF5Ofm3P4J9l1b0VEeptCP4ld9c0A/MWCKcybNIwlT73O+soD73uf326r5a5fb+bq80fx1cvP6Y0yRUROmkI/ieoD8dAfO7Q/P7xpDkWFfVj8UDm7oz8Gx7Y/xG2PvMqUUYO4+1MXaBhHRDKWQj+JmrpDAIwa3JdhAwr4yefKqDvUxt+u2Jq0/SOrd9DYEuPHN8+mf4F2k4hI5lLoJ1FTF+/RFw/uB0Dp6EFcN6uEX62vpq6p7ai2sfYOnlhbyWVTzmL88AG9XquIyMlQ6CdRU3eIIf3z6VdwZG64m+eNoyXWwfJXj96p+9ttteyqb+EzF47t+jAiIhlHoZ/Ezrrmw738TueNHsyscUP4+ep3cT8y39yyNZWMKOzD5VPP6u0yRUROmkI/ieoDzYwe3PeY9TddNJ7ttY28HF0Ba3d9M89v2c0Nc0rIP4kTuERE0kVJlURN3SFGJQn9j80oZnC/fH6+agcAy1+tor3DNbQjImcMhX4XzW3t7G9qY/SQfsds65ufy6fmlLBi40521Tfz2JpKLpo4jIkjtANXRM4MCv0uOo/cGTXo2J4+wI0XjSPW4dz+xHre3dukXr6InFEU+l10HqNfPCR56E8qKuSSc4bzuzf3MLBvHldNL+7N8kRETotCv4uaA0cfo5/MzReNB+ATM8ccdViniEim0+mjXRzu6SfZkdvpQ6Uj+dMrz+XTF5b0VlkiIt1Cod9FTV0zwwYU0Df/+D34vNwcvn7l5F6sSkSke2h4p4uauubj7sQVETnTKfS7qKlrZvRxduKKiJzpFPpdHO/ELBGRbKDQT3CotZ0DTW3ve+SOiMiZTKGfoPPIHQ3viEi2UugnOHI2rnr6IpKdFPoJOkNfPX0RyVYK/QQ1B+LDOyN1yKaIZCmFfoLqumaGn+DELBGRM5lCP8FOHa4pIllOoZ+gJsllEkVEsolCP4HOxhWRbKfQjzS1xqg71KbhHRHJaimFvpktMLOtZlZhZnck2T7ezFaa2QYze9HMSrpsH2Rm75nZD7qr8O5WHc2jP1rDOyKSxU4Y+maWC9wLXAWUAovMrLRLs7uBB919BrAUuKvL9r8Bfnv65facnZ0nZqmnLyJZLJWe/lygwt23u3srsAy4tkubUmBldPuFxO1mNgcYCfzm9MvtOdWdUzCopy8iWSyV0B8DVCYsV0XrEq0Hro9uXwcMNLPhZpYD/B3w5+/3BGa22MzKzay8trY2tcq7WWdPf+TgPml5fhGR3pBK6FuSdd5l+XZgvpmtA+YD7wEx4MvAs+5eyftw9/vcvczdy4qKilIoqfvV1B1iRGEBffJ0YpaIZK9ULpdYBYxNWC4BqhMbuHs18EkAMysErnf3OjO7GPgjM/syUAgUmFmDux+zMzjd3tnTRMnQ/ukuQ0SkR6US+muAyWY2kXgPfiFwY2IDMxsB7HP3DmAJcD+Au9+U0OYLQFkmBr67s2VnPQumj0p3KSIiPeqEwzvuHgNuA1YAm4HH3X2jmS01s2uiZpcBW81sG/Gdtt/poXp7xO6DLexvamPqqEHpLkVEpEel0tPH3Z8Fnu2y7s6E28uB5Sd4jAeAB066wl6wqaYegKmjBqa5EhGRnqUzcoEtNQcBmFqsnr6IZDeFPrC5pp4xQ/oxuF9+uksREelRCn1gy856De2ISBCCD/2WWDtv1TYyTUM7IhKA4EP/zV0NtHc4U4vV0xeR7Bd86G/ZGd+Jq56+iIRAoV9TT5+8HCYMH5DuUkREelzwob95Zz1TRg0kNyfZFEMiItkl6NB3dzbXHGSazsQVkUAEHfq1B1vY19iqnbgiEoygQ39ztBNXc+6ISCiCDv0t0Zw709TTF5FABB36m2vqKR7clyH9C9JdiohIrwg69LfsPKjpF0QkKMGGfkusnYrdDTopS0SCEmzov7W7kViHazplEQlKsKG/ZWd8J26pduKKSECCDf23ahvIyzFNvyAiQQk29A82xyjsm0debrC/AhEJULCJ19ASY0BBSpcIFhHJGsGGflNLOwP65Ka7DBGRXhVs6De2xuivnr6IBCbc0G+JUdhHoS8iYQk29Jta2+lfoOEdEQlLsKHfoJ6+iAQo2NBvam2nv3bkikhggg39hpYYA9TTF5HABBn6be0dtMY6dJy+iAQnyNBvamkHUE9fRIITZOg3tsYAGKCjd0QkMCmFvpktMLOtZlZhZnck2T7ezFaa2QYze9HMSqL1M83sZTPbGG37THe/gFPR2BKFvnr6IhKYE4a+meUC9wJXAaXAIjMr7dLsbuBBd58BLAXuitY3AZ9z9/OABcA9Zjaku4o/VY2tncM76umLSFhS6enPBSrcfbu7twLLgGu7tCkFVka3X+jc7u7b3P3N6HY1sBso6o7CT0dTZ09fO3JFJDCphP4YoDJhuSpal2g9cH10+zpgoJkNT2xgZnOBAuCtrk9gZovNrNzMymtra1Ot/ZQ1aHhHRAKVSuhbknXeZfl2YL6ZrQPmA+8BscMPYFYMPAR80d07jnkw9/vcvczdy4qKev6LQFOrjt4RkTClknpVwNiE5RKgOrFBNHTzSQAzKwSud/e6aHkQ8O/At919VXcUfboO9/R19I6IBCaVnv4aYLKZTTSzAmAh8HRiAzMbYWadj7UEuD9aXwA8RXwn7xPdV/bpaYoO2eyvnr6IBOaEoe/uMeA2YAWwGXjc3Tea2VIzuyZqdhmw1cy2ASOB70TrPw18EPiCmb0W/czs7hdxshqik7P656unLyJhSamr6+7PAs92WXdnwu3lwPIk93sYePg0a+x2TS0x+hfkkpOTbHeFiEj2CvaMXO3EFZEQhRn6Le3aiSsiQQo09NXTF5EwhRn6rTGdjSsiQQoy9Jta2zXvjogEKcjQb2iJ6Rh9EQlSkKHf1NJOoYZ3RCRAQYZ+Y0tMF0UXkSAFF/ruTmNrjEIN74hIgIIL/ea2Djoc+mt4R0QCFFzod14ft1DDOyISoPBCP5pWWT19EQlRgKGv6+OKSLiCC/3OufQ1DYOIhCi40G/Q8I6IBCy40O+8Pq4O2RSREAUX+kd6+hrTF5HwBBf6TS2dh2yqpy8i4Qku9Buj4R1NwyAiIQov9Fti5OcaffIU+iISniBDX0fuiEiowgv91naN54tIsMIL/ZaYjtwRkWCFF/qt7TobV0SCFVzoN7XENO+OiAQruNBv0I5cEQlYcKHfpB25IhKw4EJfO3JFJGThhb6ujysiAQsq9GPtHTS3dWhMX0SClVLom9kCM9tqZhVmdkeS7ePNbKWZbTCzF82sJGHb583szejn891Z/MlqatNVs0QkbCcMfTPLBe4FrgJKgUVmVtql2d3Ag+4+A1gK3BXddxjwl8BFwFzgL81saPeVf3I6r4+r4/RFJFSp9PTnAhXuvt3dW4FlwLVd2pQCK6PbLyRs/wjwnLvvc/f9wHPAgtMv+9QcuT6uQl9EwpRK6I8BKhOWq6J1idYD10e3rwMGmtnwFO/baw739HX0jogEKpXQtyTrvMvy7cB8M1sHzAfeA2Ip3hczW2xm5WZWXltbm0JJp6ZRF0UXkcClEvpVwNiE5RKgOrGBu1e7+yfdfRbwrWhdXSr3jdre5+5l7l5WVFR0ki8hdU2dwzs6ekdEApVK6K8BJpvZRDMrABYCTyc2MLMRZtb5WEuA+6PbK4APm9nQaAfuh6N1aXGkp6/hHREJ0wlD391jwG3Ew3oz8Li7bzSzpWZ2TdTsMmCrmW0DRgLfie67D/gb4n841gBLo3VpoR25IhK6lNLP3Z8Fnu2y7s6E28uB5ce57/0c6fmnlQ7ZFJHQBXVGbufwTr98De+ISJjCCv2WGP3yc8nNSXZQkYhI9gsr9HXVLBEJXFihr6tmiUjgAgv9dh2jLyJBCyz01dMXkbAFFfpNrTGN6YtI0IIK/cZWDe+ISNjCCn0N74hI4IILfV0qUURCFkzouzuNre26KLqIBC2Y0G+JddDe4fTX8I6IBCyY0O+cbE09fREJWTCh39Qan1ZZY/oiErJgQr/hcE9fwzsiEq5gQr8pmlZZPX0RCVkwoX/kqlnq6YtIuAIKfV01S0QkmNCvb24D0DQMIhK0YEJ/684G+uXnMnpIv3SXIiKSNsGE/qaaOqYWD9SlEkUkaEGEvruzqbqe0uJB6S5FRCStggj9qv2HqG+Ocd7owekuRUQkrYII/Y3V9QCUjlZPX0TCFkTob6qpJ8dg6qiB6S5FRCStwgj96jrOLiqkb75OzBKRsAUS+vUa2hERIYDQ39/YSnVdM+cp9EVEsj/0N9VEO3GLdeSOiEjWh/7G6jpAR+6IiECKoW9mC8xsq5lVmNkdSbaPM7MXzGydmW0ws6uj9flm9q9m9rqZbTazJd39Ak5kU3U9xYP7MmxAQW8/tYhIxjlh6JtZLnAvcBVQCiwys9Iuzb4NPO7us4CFwA+j9Z8C+rj7+cAc4FYzm9A9padmY3W9xvNFRCKp9PTnAhXuvt3dW4FlwLVd2jjQmayDgeqE9QPMLA/oB7QC9adddYqa29p5q7ZB0y+IiERSCf0xQGXCclW0LtFfATebWRXwLPDVaP1yoBGoAXYAd7v7vtMp+GRs2XmQDodSTb8gIgKkFvrJpqX0LsuLgAfcvQS4GnjIzHKIf0toB0YDE4FvmtmkY57AbLGZlZtZeW1t7Um9gPezKZp+QcM7IiJxqYR+FTA2YbmEI8M3nW4BHgdw95eBvsAI4EbgP9y9zd13Ay8BZV2fwN3vc/cydy8rKio6+VdxHBur6xjYN4+SoZpDX0QEUgv9NcBkM5toZgXEd9Q+3aXNDuAKADObRjz0a6P1l1vcAGAesKW7ij+RTTXx6ZTNNIe+iAikEPruHgNuA1YAm4kfpbPRzJaa2TVRs28C/93M1gOPAl9wdyd+1E8h8AbxPx4/c/cNPfA6jtHe4WypOajj80VEEqR0wVh3f5b4DtrEdXcm3N4EXJLkfg3ED9vsVe0dzrI1OzjU1q459EVEEmTVVcLbO5xnNlTz/ZVv8lZtI6XFg7hi6lnpLktEJGNkTehX7mviiw+soWJ3A+eOLORHN83mI+eNIkfXxBUROSxrQn/U4L6MG9afb1w5maunFyvsRUSSyJrQz8/N4f4vXJjuMkREMlrWz7IpIiJHKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIBafDDNzmFkt8O5pPMQIYE83ldOdMrUuyNzaMrUuyNzaMrUuyNzaMrUuOLnaxrv7CS9IknGhf7rMrNzdj7lQS7plal2QubVlal2QubVlal2QubVlal3QM7VpeEdEJCAKfRGRgGRj6N+X7gKOI1PrgsytLVPrgsytLVPrgsytLVPrgh6oLevG9EVE5PiysacvIiLHkTWhb2YLzGyrmVWY2R1pruV+M9ttZm8krBtmZs+Z2ZvRv0PTUNdYM3vBzDab2UYz+3oG1dbXzF4xs/VRbX8drZ9oZquj2h4zs4Leri2qI9fM1pnZMxlW1ztm9rqZvWZm5dG6THg/h5jZcjPbEn3eLs6QuqZEv6vOn3oz+0aG1Pan0Wf/DTN7NPo/0e2fs6wIfTPLBe4FrgJKgUVmVprGkh4AFnRZdwew0t0nAyuj5d4WA77p7tOAecBXot9TJtTWAlzu7hcAM4EFZjYP+H/AP0S17QduSUNtAF8HNicsZ0pdAH/s7jMTDu3LhPfzH4H/cPepwAXEf3dpr8vdt0a/q5nAHKAJeCrdtZnZGOBrQJm7TwdygYX0xOfM3c/4H+BiYEXC8hJgSZprmgC8kbC8FSiObhcDWzPg9/ZvwIcyrTagP/AqcBHxE1Pykr3PvVhPCfEguBx4BrBMqCt67neAEV3WpfX9BAYBbxPtM8yUupLU+WHgpUyoDRgDVALDiF/R8BngIz3xOcuKnj5HfmGdqqJ1mWSku9cARP+elc5izGwCMAtYTYbUFg2hvAbsBp4D3gIOuHssapKu9/Ue4C+Ajmh5eIbUBeDAb8xsrZktjtal+/2cBNQCP4uGxH5qZgMyoK6uFgKPRrfTWpu7vwfcDewAaoA6YC098DnLltBPdhV0HZZ0HGZWCPwC+Ia716e7nk7u3u7xr90lwFxgWrJmvVmTmX0M2O3uaxNXJ2mars/bJe4+m/jQ5lfM7INpqiNRHjAb+JG7zwIaSc8Q03FFY+PXAE+kuxaAaB/CtcBEYDQRRImQAAABw0lEQVQwgPh72tVpf86yJfSrgLEJyyVAdZpqOZ5dZlYMEP27Ox1FmFk+8cD/ubs/mUm1dXL3A8CLxPc7DDGzvGhTOt7XS4BrzOwdYBnxIZ57MqAuANy9Ovp3N/Gx6bmk//2sAqrcfXW0vJz4H4F015XoKuBVd98VLae7tiuBt9291t3bgCeBD9ADn7NsCf01wORoT3cB8a9tT6e5pq6eBj4f3f488fH0XmVmBvwLsNnd/z7DaisysyHR7X7E/xNsBl4AbkhXbe6+xN1L3H0C8c/V8+5+U7rrAjCzAWY2sPM28THqN0jz++nuO4FKM5sSrboC2JTuurpYxJGhHUh/bTuAeWbWP/p/2vk76/7PWTp3pHTzjpCrgW3Ex4G/leZaHiU+LtdGvNdzC/Fx4JXAm9G/w9JQ16XEvx5uAF6Lfq7OkNpmAOui2t4A7ozWTwJeASqIfxXvk8b39TLgmUypK6phffSzsfNznyHv50ygPHo/fwkMzYS6otr6A3uBwQnr0l4b8NfAlujz/xDQpyc+ZzojV0QkINkyvCMiIilQ6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhA/gseQSECNcyRUAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "### train the xgboost machine with some pre-defined hyperparameters\n",
    "model, eta = xgboost_train(X_train, y_train, \n",
    "                           eta = 0.4, # shrinkage factor that reduces influence of each individual tree\n",
    "                           max_round = 80, # the total number of training rounds\n",
    "                           max_depth = 3, # the max depth for each tree\n",
    "                           row_sub_frac = 0.95, # sampling percentage of observations used to reduce overfitting\n",
    "                           col_sub_frac = 1,  # sampling percentage of columns used to reduce overfitting\n",
    "                           min_child_weight = 1, # min weight for a tree to split\n",
    "                           min_sample_split = 10, # min sample required for a split\n",
    "                           lamda = 1, # lambda: L2 regularization term on weights\n",
    "                           gamma = 0, # gamma(regularization term): minimum loss reduction required for a split\n",
    "                           eps = 0.003, # epsilon for weighted quantile sketch\n",
    "                           metric = 'f1') # use f1 score as the evaluation metric to decide the best rounds or the number of trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[47  2]\n",
      " [ 6 45]]\n",
      "Test F1 Score:  0.9183673469387754\n",
      "Test Accuracy:  0.92\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFblJREFUeJzt3W2MXFd9x/Hffx/G3gd7d21vQrCdOBSnxURIiZZAhVRCSSPHL+w3lNoSaqkiLGhDVYEqpaJKUXhVUIuE5BasFlGQIARewAoZpSoNoo1wmkWBgB252pqHrILqJZ4Zxzuznpndf1/Mg2dnZ3burudh77nfj7Ty3DvXs+dm1z8O557zP+buAgCEZaDfDQAAdB7hDgABItwBIECEOwAEiHAHgAAR7gAQIMIdAAJEuANAgAh3AAjQUL++8b59+/zQoUP9+vYAEEs/+tGPfuPu0+2u61u4Hzp0SHNzc/369gAQS2b2yyjXMSwDAAEi3AEgQIQ7AASIcAeAABHuABCgtuFuZl80sytm9rMW75uZfc7M5s3sJTO7v/PNBABsRpSe+5ckHd3g/UckHa58nZb0T7feLADArWg7z93df2Bmhza45ISkL3t5v77zZjZpZne4+6871EYAiJ1CaVXpXEFXlwpKLxX02lKhdvz7v3Ob3nZgsqvfvxOLmPZLeqXueKFybl24m9lplXv3uvPOOzvwrQGg+9xd15ZLurpUqH2llwq6miusOXe1GuDXC3r9Rqnl5+0b3xGLcLcm55ruuu3uZyWdlaSZmRl25gbQF8vFlVovek1YNwR2eqmo15YKyuQKKq02j6zU0ID2jqW0p/J1195RTY3ePK7/mhpNaWp0WEOD3Z/L0olwX5B0sO74gKRXO/C5ANDW6qormy82hHJlGKQusOvPLRVWmn6WmTQ5MrwmqO+/a3JNWE+NpbS3EtR7x1MaGR6UWbM+bn91ItxnJT1mZk9JeoekLOPtALYqX1hZF8av1Q+DXL/Zu05XhkFadKo1Mjy4JpTfND1eC+VyYA9rz9iO2p8TI8MaHNh+Qb0VbcPdzL4m6UFJ+8xsQdLfShqWJHf/vKRzko5JmpeUk/Sn3WosgHhZWXVlcuUAfu165c9aj7qoq0s3dDVX/jO9VNTVpYLyxea96gHTzaGNsZQO3zauqbGU9jQZAqmeH0kN9viOt48os2VOtXnfJf15x1oEYFtyd+UKKzfHqXN1wx4NDxSrAZ7JF+UtetVjqUHtGS+H8PT4Dt1z+65yUI+vDezqMMjuncMaCKRX3Qt9K/kLoL9KK6tK54pNQ3nNues3p/DdKK02/ayhAVvTi37LG3ZrqjrkMTpcCegdmhob1t6xHZocHdbO4eT2qnuBcAcC4O66fqPUNJQbh0GqgZ7NF1t+3q6dQ7UhkDsmdurIG3eXHyLWBXjtweJYSrt3Dm3Lh4pJRrgD21ChtKpMruGBYpM51dUgTy8VVVhp3qseHrRaUO8dT+mNkyM3g7r6VTccMjmaUmqIslNxR7gDXdbpBTATdVP1DkyN6m0HJtbM+NgzNlwO8sowyPgOetVJRLgDm1S/AKa8yOVGw3zq6rnNL4C5c89o04Uv1al7vVoAg/gj3JFo9QtgWs6rXurOApg9YymNprbnAhjEH+GOoDRbAFM/da+6ACZdNwyy2QUwjQtfqsMgk6OpYBbAIP4Id2xbzRbA1Ba+1C2AqQ/wjRbATNXN8nhzkwUw9bM/kr4ABvFHuKMnurkAZu94SodvH18z46M+qFkAgyQi3LEl3VoAMzU2zAIYoAMId9QWwNRmeWwwDNLJBTDV4RAWwACdR7gHqNsLYNbVqa4bBmEBDLA9EO7bXOMCmOYbCtQFOAtgAIhw77kbpZW6UO7eApjGDQXKU/WGNcwCGCARCPdb0GoBTLMNBarn2i2AqYYyC2AA3ArCvc5ycWX9CsUuLoBZ26tmAQyAzklEuM9feV3zV66vmfnRbOpe1AUwvzU9rrffzQIYANtX8OG+XFzRsc/9lwp1c6xZAAMgdMGHezpXUKG0qr986LD+6O0HNTWaYgEMgOAFH+6ZXHmxzW/fvkt3TIz0uTUA0BvBz4urhvvE6HCfWwIAvZOAcC9IKj8QBYCkCD/cKzVQJum5A0iQ8MO9MiwzOULPHUByhB/u+YJSQwPaORz8rQJATfCJl80VNTU6zDJ9AIkSfLincwWGZAAkTvDhnskVmQYJIHGCD/dsvqjJEcIdQLIEH+6ZXJFpkAASJ1K4m9lRM7tkZvNm9niT9+80s2fN7EUze8nMjnW+qVuTzhVYwAQgcdqGu5kNSjoj6RFJRySdMrMjDZf9jaSn3f0+SScl/WOnG7oVy8UV3SitMuYOIHGi9NwfkDTv7pfdvSDpKUknGq5xSbsrryckvdq5Jm4dC5gAJFWUqpD7Jb1Sd7wg6R0N13xS0r+Z2UcljUl6qCOtu0WZfLmuDGPuAJImSs+92eqfxs3lTkn6krsfkHRM0lfMbN1nm9lpM5szs7nFxcXNt3aT0kvUlQGQTFHCfUHSwbrjA1o/7PKopKclyd1/KGmnpH2NH+TuZ919xt1npqent9biTchWe+4MywBImCjh/oKkw2Z2t5mlVH5gOttwza8kvVeSzOwtKod797vmbdTG3Om5A0iYtuHu7iVJj0l6RtLLKs+KuWBmT5rZ8cplH5f0ITP7iaSvSfqguzcO3fQc5X4BJFWkbfbc/Zykcw3nnqh7fVHSuzrbtFuXzpUrQo6wZyqAhAl6hWo2Vy49QEVIAEkTdLhTegBAUoUd7nnK/QJIprDDnZ47gIQi3AEgQGGHe76gSSpCAkigYMN9ubii5eKqJtioA0ACBRvuWRYwAUiwYMM9nSvXlWGjDgBJFGy436zlTs8dQPIEH+7swgQgiYIN91q5X4ZlACRQsOGervTcp+i5A0igYMM9kysqNUhFSADJFGy4Z/MFTYxSERJAMgUb7plKuV8ASKJgwz2dKzDHHUBiBRvumVyRaZAAEivYcM/mGZYBkFzBhjvlfgEkWZDhvlxcUb64wgImAIkVZLhTERJA0gUZ7jeLhtFzB5BMgYZ7ta4MPXcAyRRmuFeGZdiFCUBShRnu1Y06xhiWAZBMgYY7G3UASLYwwz1f1PCgaTRFRUgAyRRmuOeKmhhJURESQGIFGu4FNukAkGiBhjulBwAkW6RwN7OjZnbJzObN7PEW17zfzC6a2QUz+2pnm7k5mXx5WAYAkmqo3QVmNijpjKQ/kLQg6QUzm3X3i3XXHJb015Le5e5pM7utWw2OIpsr6K1v3N3PJgBAX0XpuT8gad7dL7t7QdJTkk40XPMhSWfcPS1J7n6ls83cnDS7MAFIuCjhvl/SK3XHC5Vz9e6RdI+ZPWdm583saLMPMrPTZjZnZnOLi4tba3Eb1YqQLGACkGRRwr3ZfEJvOB6SdFjSg5JOSfpnM5tc95fcz7r7jLvPTE9Pb7atkVyj9AAARAr3BUkH644PSHq1yTXfdveiu/9c0iWVw77nMpT7BYBI4f6CpMNmdreZpSSdlDTbcM23JL1Hksxsn8rDNJc72dCoKPcLABHC3d1Lkh6T9IyklyU97e4XzOxJMzteuewZSa+Z2UVJz0r6K3d/rVuN3kiacr8A0H4qpCS5+zlJ5xrOPVH32iV9rPLVV9kcwzIAENwK1Uy+2nNnWAZAcoUX7rmihgZMY1SEBJBgwYV7OlfU5CgVIQEkW3Dhns0XGG8HkHjBhXuG0gMAEGi403MHkHABhnuBmTIAEi+8cM8zLAMAQYX7jdKKcoUVhmUAJF5Q4Z6tVoRkWAZAwoUV7rWiYfTcASRbUOGeroT7FD13AAkXVLhnqAgJAJJCC3d2YQIASYGFO+V+AaAsqHBP5woaGjCN74hUph4AghVUuGfy5dIDVIQEkHRBhXs2V2S8HQAUWLhn8tSVAQApsHBPLxU1xcNUAAgr3LP5oiZG6LkDQFDhXi73S88dAIIJ90JpVUuFFerKAIACCvdMntIDAFAVTLjfXJ3KmDsABBPu1boy9NwBIKRwr9Vyp+cOAAGFO2PuAFAVULgzLAMAVeGEe76gQSpCAoCkiOFuZkfN7JKZzZvZ4xtc9z4zczOb6VwTo8nkipocoSIkAEgRwt3MBiWdkfSIpCOSTpnZkSbX7ZL0F5Ke73Qjo8jki5pgSAYAJEXruT8gad7dL7t7QdJTkk40ue5Tkj4tabmD7YsskyuwMTYAVEQJ9/2SXqk7XqicqzGz+yQddPfvdLBtm1IdlgEARAv3ZoPYXnvTbEDSZyV9vO0HmZ02szkzm1tcXIzeyggyOYZlAKAqSrgvSDpYd3xA0qt1x7sk3Svp+2b2C0nvlDTb7KGqu5919xl3n5ment56q5vI5ossYAKAiijh/oKkw2Z2t5mlJJ2UNFt9092z7r7P3Q+5+yFJ5yUdd/e5rrS4iUJpVddvlJjjDgAVbcPd3UuSHpP0jKSXJT3t7hfM7EkzO97tBkaRrdSVYRcmACiLtOLH3c9JOtdw7okW1z54683anGyl3O8Es2UAQFIgK1RvFg2j5w4AUiDhnqauDACsEUS4VytCsogJAMqCCPfqA1XmuQNAWRDhnskVNThg2kVFSACQFEq45wuaoCIkANQEEe7pXJGHqQBQJ4hwz1I0DADWCCLcM/mCJpkpAwA1YYQ7PXcAWCOccKfnDgA1sQ/34goVIQGgUezDvbqAiXAHgJtiH+7VomETjLkDQE0A4V6uK8OYOwDcFEC4s1EHADSKf7hXx9zZPxUAauIf7rnqLkz03AGgKoBwL2rAREVIAKgT/3CvlB4YGKAiJABUxT/cKT0AAOvEPtyz+SLj7QDQIPbhTs8dANaLfbincwU2xgaABrEP92yOYRkAaBTrcC+urOr1GyUWMAFAg1iH+zUqQgJAU7EO93SOcAeAZmId7tk8FSEBoJlYh3u1IiRTIQFgrTDCnWEZAFgjUrib2VEzu2Rm82b2eJP3P2ZmF83sJTP7npnd1fmmrpeubtTBbBkAWKNtuJvZoKQzkh6RdETSKTM70nDZi5Jm3P1tkr4p6dOdbmgz2XylIuROKkICQL0oPfcHJM27+2V3L0h6StKJ+gvc/Vl3z1UOz0s60NlmNpfJFTUxMkxFSABoECXc90t6pe54oXKulUclfbfZG2Z22szmzGxucXExeitbyOSLzJQBgCaihHuzbrE3vdDsA5JmJH2m2fvuftbdZ9x9Znp6OnorW8jkCppgpgwArBNlsHpB0sG64wOSXm28yMwekvQJSe929xudad7GMrmi9o3TcweARlF67i9IOmxmd5tZStJJSbP1F5jZfZK+IOm4u1/pfDObq+7CBABYq224u3tJ0mOSnpH0sqSn3f2CmT1pZscrl31G0rikb5jZj81stsXHdVT1gSoAYK1Icwjd/Zykcw3nnqh7/VCH29VWaWVVry+XWMAEAE3EdoVqNk/pAQBoJbbhnqmE+9QYY+4A0Ci+4V6pK8OYOwCsF9twp9wvALQW23BPLzHmDgCtxDbca2Pu9NwBYJ3Yhns2V5BRERIAmoptuGfyVIQEgFZiG+7pXJHxdgBoIbbhnslRVwYAWoltuGfzRUoPAEALsQ33DMMyANBSjMOdYRkAaCWW4V5aWdU1KkICQEuxDPdryyVJrE4FgFZiGe6ZHHVlAGAj8Qz3SumBCYZlAKCpeIZ7tefOsAwANBXTcKdoGABsJNbhzmwZAGgunuGeL1YqQhLuANBMPMM9V9DuncMapCIkADQV03AvaoohGQBoKZ7hni9qgoepANBSLMM9myswDRIANhDLcE/nKPcLABuJZbhncgXmuAPABmIX7iurrmvLJU0wLAMALcUu3K/lWcAEAO3ELtwzhDsAtBUp3M3sqJldMrN5M3u8yfs7zOzrlfefN7NDnW5oVbpWNIwxdwBopW24m9mgpDOSHpF0RNIpMzvScNmjktLu/mZJn5X0d51uaFWWujIA0FaUnvsDkubd/bK7FyQ9JelEwzUnJP1r5fU3Jb3XzLpSGyCTZ6MOAGgnSrjvl/RK3fFC5VzTa9y9JCkraW8nGtioVhGS2TIA0FKUcG/WA/ctXCMzO21mc2Y2t7i4GKV96+yfHNHDR27XbsIdAFoainDNgqSDdccHJL3a4poFMxuSNCHpauMHuftZSWclaWZmZl34R/HwW9+gh9/6hq38VQBIjCg99xckHTazu80sJemkpNmGa2Yl/Unl9fsk/Ye7bym8AQC3rm3P3d1LZvaYpGckDUr6ortfMLMnJc25+6ykf5H0FTObV7nHfrKbjQYAbCzKsIzc/Zykcw3nnqh7vSzpDzvbNADAVsVuhSoAoD3CHQACRLgDQIAIdwAIEOEOAAGyfk1HN7NFSb/c4l/fJ+k3HWxOHHDPycA9J8Ot3PNd7j7d7qK+hfutMLM5d5/pdzt6iXtOBu45GXpxzwzLAECACHcACFBcw/1svxvQB9xzMnDPydD1e47lmDsAYGNx7bkDADawrcN9O23M3SsR7vljZnbRzF4ys++Z2V39aGcntbvnuuveZ2ZuZrGfWRHlns3s/ZWf9QUz+2qv29hpEX637zSzZ83sxcrv97F+tLNTzOyLZnbFzH7W4n0zs89V/nu8ZGb3d7QB7r4tv1QuL/y/kt4kKSXpJ5KONFzzZ5I+X3l9UtLX+93uHtzzeySNVl5/JAn3XLlul6QfSDovaabf7e7Bz/mwpBclTVWOb+t3u3twz2clfaTy+oikX/S73bd4z78n6X5JP2vx/jFJ31V5J7t3Snq+k99/O/fct9XG3D3S9p7d/Vl3z1UOz6u8M1acRfk5S9KnJH1a0nIvG9clUe75Q5LOuHtaktz9So/b2GlR7tkl7a68ntD6Hd9ixd1/oCY70tU5IenLXnZe0qSZ3dGp77+dw31bbczdI1Huud6jKv8vf5y1vWczu0/SQXf/Ti8b1kVRfs73SLrHzJ4zs/NmdrRnreuOKPf8SUkfMLMFlfeP+GhvmtY3m/33vimRNuvok45tzB0jke/HzD4gaUbSu7vaou7b8J7NbEDSZyV9sFcN6oEoP+chlYdmHlT5/539p5nd6+6ZLretW6Lc8ylJX3L3vzez31V5d7d73X21+83ri67m13buuW9mY25ttDF3jES5Z5nZQ5I+Iem4u9/oUdu6pd0975J0r6Tvm9kvVB6bnI35Q9Wov9vfdveiu/9c0iWVwz6uotzzo5KeliR3/6GknSrXYAlVpH/vW7Wdwz2JG3O3vefKEMUXVA72uI/DSm3u2d2z7r7P3Q+5+yGVnzMcd/e5/jS3I6L8bn9L5YfnMrN9Kg/TXO5pKzsryj3/StJ7JcnM3qJyuC/2tJW9NSvpjyuzZt4pKevuv+7Yp/f7iXKbp83HJP2Pyk/ZP1E596TK/7il8g//G5LmJf23pDf1u809uOd/l/R/kn5c+Zrtd5u7fc8N135fMZ8tE/HnbJL+QdJFST+VdLLfbe7BPR+R9JzKM2l+LOnhfrf5Fu/3a5J+Lamoci/9UUkflvThup/xmcp/j592+veaFaoAEKDtPCwDANgiwh0AAkS4A0CACHcACBDhDgABItwBIECEOwAEiHAHgAD9P6aw8L92Zw7EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# make predictions on test set\n",
    "prediction = xgboost_predict(model, X_test, eta)\n",
    "print(confusion_matrix(y_test, prediction))\n",
    "print(\"Test F1 Score: \", f1_score(y_test, prediction))\n",
    "print(\"Test Accuracy: \", accuracy_score(y_test, prediction))\n",
    "fpr, tpr, _ = roc_curve(y_test, prediction)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "plt.plot(fpr, tpr, label = \"auc = \" + str(roc_auc))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}