bank_deposit_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2><B><left>Hello, This Notebook Contains example of Bank Deposite classification.</left></B></h2>\n",
    "<h4><I><left>- We try to predict if the bank campaign should contact with cretain customer.</left></I></h4>\n",
    "<h4><I><left>- We use Pandas library for data exploratory, Seaborn Library for visualize data(found in bank_visualization file).</left></I></h4>\n",
    "<h4><I><left>- We use Scikit-Learn library for training and testing models.</left></I></h4>\n",
    "<h4><I><left>- We implement different techniques for feature selection, and we use Grid_Search technique for parameters tunning.</left></I></h4>\n",
    "<h4><I><left>- Finally we save all results in 'results.csv' file</left></I></h4>\n",
    "<div class=\"alert alert-block alert-info\" style=\"margin-top: 20px\">\n",
    "    <ul>\n",
    "            <li><h3>Contents:</h3></li>\n",
    "   <ul>\n",
    "        <li><h4><a href=\"#exploratory\">Data Exploratory.</a></h4></li>\n",
    "        <li><h4><a href=\"#prep\">Prepocessing.</a></h4></li>\n",
    "         <ul>\n",
    "             <li><a href=\"#call\">call estimators</a></li>\n",
    "            <li><a href=\"#pipline\">create piplines</a></li>\n",
    "             <li><a href=\"#split\">split data</a></li>\n",
    "        </ul>\n",
    "        <li><h4><a href=\"#model\">Modeling.</a></h4></li>\n",
    "        <li><h4><a href=\"#results\">Results.</a></h4></li>\n",
    "         <ul>\n",
    "            <li><a href=\"#roc_auc\">plot destination metric</a></li>\n",
    "            <li><a href=\"#prev_res\">previous results</a></li>\n",
    "             <li><a href=\"#learning_curve\">plot learning curve</a></li>\n",
    "        </ul>\n",
    "    </ul>\n",
    "    </ul>\n",
    "</div>\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import utils as uts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"exploratory\"></a>\n",
    "<h3><B><left>- Data Exploratory:</left></B></h3>\n",
    "<h4><I><left>- reading data set and plotting some statistical values.</left></I></h4>\n",
    "<h4><I><left>- visualization processes found in another file (bank_deposit_data_visualization.ipynb).</left></I></h4>\n",
    "\n",
    "-----------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>job</th>\n",
       "      <th>marital</th>\n",
       "      <th>education</th>\n",
       "      <th>default</th>\n",
       "      <th>balance</th>\n",
       "      <th>housing</th>\n",
       "      <th>loan</th>\n",
       "      <th>contact</th>\n",
       "      <th>day</th>\n",
       "      <th>month</th>\n",
       "      <th>duration</th>\n",
       "      <th>campaign</th>\n",
       "      <th>pdays</th>\n",
       "      <th>previous</th>\n",
       "      <th>poutcome</th>\n",
       "      <th>deposit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>59</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>2343</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>1042</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>56</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>45</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>1467</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>technician</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>1270</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>1389</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55</td>\n",
       "      <td>services</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>2476</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>579</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>no</td>\n",
       "      <td>184</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>673</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11157</th>\n",
       "      <td>33</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>single</td>\n",
       "      <td>primary</td>\n",
       "      <td>no</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>20</td>\n",
       "      <td>apr</td>\n",
       "      <td>257</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11158</th>\n",
       "      <td>39</td>\n",
       "      <td>services</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>733</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>16</td>\n",
       "      <td>jun</td>\n",
       "      <td>83</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11159</th>\n",
       "      <td>32</td>\n",
       "      <td>technician</td>\n",
       "      <td>single</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>29</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>19</td>\n",
       "      <td>aug</td>\n",
       "      <td>156</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11160</th>\n",
       "      <td>43</td>\n",
       "      <td>technician</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>cellular</td>\n",
       "      <td>8</td>\n",
       "      <td>may</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>172</td>\n",
       "      <td>5</td>\n",
       "      <td>failure</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11161</th>\n",
       "      <td>34</td>\n",
       "      <td>technician</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>9</td>\n",
       "      <td>jul</td>\n",
       "      <td>628</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11162 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       age          job  marital  education default  balance housing loan  \\\n",
       "0       59       admin.  married  secondary      no     2343     yes   no   \n",
       "1       56       admin.  married  secondary      no       45      no   no   \n",
       "2       41   technician  married  secondary      no     1270     yes   no   \n",
       "3       55     services  married  secondary      no     2476     yes   no   \n",
       "4       54       admin.  married   tertiary      no      184      no   no   \n",
       "...    ...          ...      ...        ...     ...      ...     ...  ...   \n",
       "11157   33  blue-collar   single    primary      no        1     yes   no   \n",
       "11158   39     services  married  secondary      no      733      no   no   \n",
       "11159   32   technician   single  secondary      no       29      no   no   \n",
       "11160   43   technician  married  secondary      no        0      no  yes   \n",
       "11161   34   technician  married  secondary      no        0      no   no   \n",
       "\n",
       "        contact  day month  duration  campaign  pdays  previous poutcome  \\\n",
       "0       unknown    5   may      1042         1     -1         0  unknown   \n",
       "1       unknown    5   may      1467         1     -1         0  unknown   \n",
       "2       unknown    5   may      1389         1     -1         0  unknown   \n",
       "3       unknown    5   may       579         1     -1         0  unknown   \n",
       "4       unknown    5   may       673         2     -1         0  unknown   \n",
       "...         ...  ...   ...       ...       ...    ...       ...      ...   \n",
       "11157  cellular   20   apr       257         1     -1         0  unknown   \n",
       "11158   unknown   16   jun        83         4     -1         0  unknown   \n",
       "11159  cellular   19   aug       156         2     -1         0  unknown   \n",
       "11160  cellular    8   may         9         2    172         5  failure   \n",
       "11161  cellular    9   jul       628         1     -1         0  unknown   \n",
       "\n",
       "      deposit  \n",
       "0         yes  \n",
       "1         yes  \n",
       "2         yes  \n",
       "3         yes  \n",
       "4         yes  \n",
       "...       ...  \n",
       "11157      no  \n",
       "11158      no  \n",
       "11159      no  \n",
       "11160      no  \n",
       "11161      no  \n",
       "\n",
       "[11162 rows x 17 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try:\n",
    "    bank = pd.read_csv('../Data/bank1/bank.csv')\n",
    "    names_parser = uts.namesParser()\n",
    "    results_management = uts.resultsManagement('results.csv')\n",
    "except Exception as ex:\n",
    "    print('some Error!',ex)\n",
    "bank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x190bf4a2ec8>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAATaUlEQVR4nO3df9TedX3f8ecLUqS12oQSKE1gQYlVXOWHd4HptCt0ITBr6A9aOi0pzTlZd1inO2tXXM9ZWiineurmxCldJtHEapFiGTnOCVlE250WIQHkpywZWsgJhWgC6pg48L0/rk/0SnLn/tyEXPd9J/fzcc51ru/3/f18v9f74tzw4vvzSlUhSdJEjpjuBiRJM59hIUnqMiwkSV2GhSSpy7CQJHXNme4GRuHYY4+tRYsWTXcbknRI2bx589eqav54yw7LsFi0aBGbNm2a7jYk6ZCS5G/3t8zDUJKkrpGGRZK5SW5M8uUkDyX5B0mOSbIhyZb2Pq+NTZJrkmxNcm+SM4e2s7yN35Jk+Sh7liTta9R7Fu8HPltVrwZOAx4CrgA2VtViYGObB7gAWNxeK4FrAZIcA6wCzgbOAlbtDhhJ0tQYWVgkeTnwZuA6gKr6TlU9BSwD1rZha4GL2vQyYF0N3A7MTXICcD6woap2VtUuYAOwdFR9S5L2Nco9i1cAO4CPJLk7yYeTvBQ4vqoeB2jvx7XxC4DHhtbf1mr7q+8hycokm5Js2rFjx8H/NpI0i40yLOYAZwLXVtUZwP/h+4ecxpNxajVBfc9C1eqqGquqsfnzx73yS5J0gEYZFtuAbVX1xTZ/I4PweKIdXqK9Pzk0/sSh9RcC2yeoS5KmyMjCoqr+DngsyU+00nnAg8B6YPcVTcuBm9v0euDSdlXUOcDT7TDVLcCSJPPaie0lrSZJmiKjvinvt4CPJzkKeAS4jEFA3ZBkBfAocHEb+xngQmAr8EwbS1XtTHIVcGcbd2VV7Rxx35KkITkcf/xobGysvINbh6tHr/zJ6W5BM9BJ/+6+F72NJJuramy8Zd7BLUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKlrznQ3MFO9/nfWTXcLmoE2//Gl092CNC3cs5AkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqGmlYJPlqkvuS3JNkU6sdk2RDki3tfV6rJ8k1SbYmuTfJmUPbWd7Gb0myfJQ9S5L2NRV7Fj9TVadX1VibvwLYWFWLgY1tHuACYHF7rQSuhUG4AKuAs4GzgFW7A0aSNDWm4zDUMmBtm14LXDRUX1cDtwNzk5wAnA9sqKqdVbUL2AAsneqmJWk2G3VYFHBrks1JVrba8VX1OEB7P67VFwCPDa27rdX2V99DkpVJNiXZtGPHjoP8NSRpdhv14z7eWFXbkxwHbEjy5QnGZpxaTVDfs1C1GlgNMDY2ts9ySdKBG+meRVVtb+9PAjcxOOfwRDu8RHt/sg3fBpw4tPpCYPsEdUnSFBlZWCR5aZKX7Z4GlgD3A+uB3Vc0LQdubtPrgUvbVVHnAE+3w1S3AEuSzGsntpe0miRpiozyMNTxwE1Jdn/OJ6rqs0nuBG5IsgJ4FLi4jf8McCGwFXgGuAygqnYmuQq4s427sqp2jrBvSdJeRhYWVfUIcNo49a8D541TL+Dy/WxrDbDmYPcoSZoc7+CWJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktQ18rBIcmSSu5N8us2fnOSLSbYk+WSSo1r9JW1+a1u+aGgb72r1h5OcP+qeJUl7moo9i3cADw3Nvwd4X1UtBnYBK1p9BbCrqk4B3tfGkeRU4BLgtcBS4ENJjpyCviVJzUjDIslC4J8AH27zAc4FbmxD1gIXtellbZ62/Lw2fhlwfVU9W1VfAbYCZ42yb0nSnka9Z/EfgX8DfLfN/yjwVFU91+a3AQva9ALgMYC2/Ok2/nv1cdb5niQrk2xKsmnHjh0H+3tI0qw2srBI8hbgyaraPFweZ2h1lk20zvcLVauraqyqxubPn/+C+5Uk7d+cEW77jcBbk1wIHA28nMGextwkc9rew0Jgexu/DTgR2JZkDvAjwM6h+m7D60iSpsDI9iyq6l1VtbCqFjE4Qf25qnobcBvwS23YcuDmNr2+zdOWf66qqtUvaVdLnQwsBu4YVd+SpH2Ncs9if34XuD7JHwJ3A9e1+nXAx5JsZbBHcQlAVT2Q5AbgQeA54PKqen7q25ak2WtKwqKqPg98vk0/wjhXM1XVt4GL97P+1cDVo+tQkjQR7+CWJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXZMKiyQbJ1OTJB2eJvzxoyRHAz8EHJtkHpC26OXAj4+4N0nSDNH7pbx/BryTQTBs5vth8Q3ggyPsS5I0g0wYFlX1fuD9SX6rqj4wRT1JkmaYSf0Gd1V9IMkbgEXD61TVuhH1JUmaQSYVFkk+BrwSuAd4vpULMCwkaRaYVFgAY8CpVVWjbEaSNDNN9j6L+4EfG2UjkqSZa7J7FscCDya5A3h2d7Gq3jqSriRJM8pkw+L3R9mEJGlmm+zVUF8YdSOSpJlrso/7+GaSb7TXt5M8n+QbnXWOTnJHki8leSDJH7T6yUm+mGRLkk8mOarVX9Lmt7bli4a29a5WfzjJ+Qf+dSVJB2JSYVFVL6uql7fX0cAvAv+ps9qzwLlVdRpwOrA0yTnAe4D3VdViYBewoo1fAeyqqlOA97VxJDkVuAR4LbAU+FCSI1/Il5QkvTgH9NTZqvqvwLmdMVVV32qzP9Be1da7sdXXAhe16WVtnrb8vCRp9eur6tmq+gqwFTjrQPqWJB2Yyd6U9wtDs0cwuO+ie89F2wPYDJzC4FlS/xt4qqqea0O2AQva9ALgMYCqei7J08CPtvrtQ5sdXmf4s1YCKwFOOumkyXwtSdIkTfZqqJ8bmn4O+CqD/+OfUFU9D5yeZC5wE/Ca8Ya19+xn2f7qe3/WamA1wNjYmDcPStJBNNmroS57MR9SVU8l+TxwDjA3yZy2d7EQ2N6GbQNOBLYlmQP8CLBzqL7b8DqSpCkw2auhFia5KcmTSZ5I8qkkCzvrzG97FCT5QeBngYeA24BfasOWAze36fVtnrb8c+3xIuuBS9rVUicDi4E7Jv8VJUkv1mQPQ30E+ARwcZt/e6v94wnWOQFY285bHAHcUFWfTvIgcH2SPwTuBq5r468DPpZkK4M9iksAquqBJDcADzI4BHZ5O7wlSZoikw2L+VX1kaH5jyZ550QrVNW9wBnj1B9hnKuZqurbfD+M9l52NXD1JHuVJB1kk7109mtJ3p7kyPZ6O/D1UTYmSZo5JhsWvwH8MvB3wOMMzim8qJPekqRDx2QPQ10FLK+qXQBJjgHeyyBEJEmHucnuWbxud1AAVNVOxjkfIUk6PE02LI5IMm/3TNuzmOxeiSTpEDfZ/+D/e+Cvk9zI4O7pX8arkyRp1pjsHdzrkmxi8BDAAL9QVQ+OtDNJ0owx6UNJLRwMCEmahQ7oEeWSpNnFsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1jSwskpyY5LYkDyV5IMk7Wv2YJBuSbGnv81o9Sa5JsjXJvUnOHNrW8jZ+S5Llo+pZkjS+Ue5ZPAf866p6DXAOcHmSU4ErgI1VtRjY2OYBLgAWt9dK4FoYhAuwCjgbOAtYtTtgJElTY2RhUVWPV9VdbfqbwEPAAmAZsLYNWwtc1KaXAetq4HZgbpITgPOBDVW1s6p2ARuApaPqW5K0ryk5Z5FkEXAG8EXg+Kp6HAaBAhzXhi0AHhtabVur7a++92esTLIpyaYdO3Yc7K8gSbPayMMiyQ8DnwLeWVXfmGjoOLWaoL5noWp1VY1V1dj8+fMPrFlJ0rhGGhZJfoBBUHy8qv6ilZ9oh5do70+2+jbgxKHVFwLbJ6hLkqbIKK+GCnAd8FBV/YehReuB3Vc0LQduHqpf2q6KOgd4uh2mugVYkmReO7G9pNUkSVNkzgi3/Ubg14D7ktzTav8WeDdwQ5IVwKPAxW3ZZ4ALga3AM8BlAFW1M8lVwJ1t3JVVtXOEfUuS9jKysKiq/8n45xsAzhtnfAGX72dba4A1B687SdIL4R3ckqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1DWysEiyJsmTSe4fqh2TZEOSLe19XqsnyTVJtia5N8mZQ+ssb+O3JFk+qn4lSfs3yj2LjwJL96pdAWysqsXAxjYPcAGwuL1WAtfCIFyAVcDZwFnAqt0BI0maOiMLi6r6S2DnXuVlwNo2vRa4aKi+rgZuB+YmOQE4H9hQVTurahewgX0DSJI0YlN9zuL4qnocoL0f1+oLgMeGxm1rtf3VJUlTaKac4M44tZqgvu8GkpVJNiXZtGPHjoPanCTNdlMdFk+0w0u09ydbfRtw4tC4hcD2Cer7qKrVVTVWVWPz588/6I1L0mw21WGxHth9RdNy4Oah+qXtqqhzgKfbYapbgCVJ5rUT20taTZI0heaMasNJ/gz4R8CxSbYxuKrp3cANSVYAjwIXt+GfAS4EtgLPAJcBVNXOJFcBd7ZxV1bV3ifNJUkjNrKwqKpf3c+i88YZW8Dl+9nOGmDNQWxNkvQCzZQT3JKkGcywkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpK5DJiySLE3ycJKtSa6Y7n4kaTY5JMIiyZHAB4ELgFOBX01y6vR2JUmzxyERFsBZwNaqeqSqvgNcDyyb5p4kadaYM90NTNIC4LGh+W3A2cMDkqwEVrbZbyV5eIp6mw2OBb423U3MBHnv8uluQXvyb3O3VTkYW/l7+1twqITFeP8Uao+ZqtXA6qlpZ3ZJsqmqxqa7D2lv/m1OnUPlMNQ24MSh+YXA9mnqRZJmnUMlLO4EFic5OclRwCXA+mnuSZJmjUPiMFRVPZfkXwC3AEcCa6rqgWluazbx8J5mKv82p0iqqj9KkjSrHSqHoSRJ08iwkCR1GRaSpC7DQpLUZViIJFclecfQ/NVJ/mWS30lyZ5J7k/xBW/bSJP8tyZeS3J/kV6avc80mSRYleSjJf0nyQJJbk/xgktOT3N7+Tm9KMm+6ez0cGRYCuA5YDpDkCAb3sTwBLGbwXK7TgdcneTOwFNheVadV1d8HPjs9LWuWWgx8sKpeCzwF/CKwDvjdqnodcB+wahr7O2wZFqKqvgp8PckZwBLgbuCnhqbvAl7N4F/U+4CfTfKeJG+qqqenp2vNUl+pqnva9GbglcDcqvpCq60F3jwtnR3mDomb8jQlPgz8OvBjwBrgPOCPquo/7z0wyeuBC4E/SnJrVV05lY1qVnt2aPp5YO50NTLbuGeh3W5icIjppxjcKX8L8BtJfhggyYIkxyX5ceCZqvpT4L3AmdPVsAQ8DexK8qY2/2vAFyYYrwPknoUAqKrvJLkNeKqqngduTfIa4G+SAHwLeDtwCvDHSb4L/D/gn09Xz1KzHPiTJD8EPAJcNs39HJZ83IeA753Yvgu4uKq2THc/kmYWD0OJ9hO1W4GNBoWk8bhnIUnqcs9CktRlWEiSugwLSVKXYSEdoCS/n+S3p+Bz/rq9L0ryT0f9edJ4DAtphquqN7TJRYBhoWlhWEgvQJLfS/Jwkv8B/ESrvTLJZ5NsTvJXSV7d6h9N8iet9r+SvKXVj07ykST3Jbk7yc+0+muT3JHknvYE1cWt/q328e8G3tSW/6sp//Ka1byDW5qk9kysS4AzGPy7cxeDh9mtBn6zqrYkORv4EHBuW20R8NMMHnh3W5JTgMsBquonW7DcmuRVwG8C76+qjyc5CjhyrxauAH67qt4ywq8pjcuwkCbvTcBNVfUMQJL1wNHAG4A/b49FAXjJ0Do3VNV3gS1JHmHw9N5/CHwAoKq+nORvgVcBfwP8XpKFwF94g6RmEg9DSS/M3nexHsHgeVqnD71eM8H4AsI4quoTwFuB/wvckuTc8cZJ08GwkCbvL4Gfb7/O9jLg54BngK8kuRggA6cNrXNxkiOSvBJ4BfBw287b2vhXAScBDyd5BfBIVV0DrAdet9fnfxN42ei+nrR/hoU0SVV1F/BJ4B7gU8BftUVvA1Yk+RLwALBsaLWHGTwy+78zOK/xbQbnNI5Mcl/b3q9X1bPArwD3J7mHweGqdXu1cC/wXPtJW09wa0r5bChpRJJ8FPh0Vd043b1IL5Z7FpKkLvcsJEld7llIkroMC0lSl2EhSeoyLCRJXYaFJKnr/wPa2CiLdpqw7gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='deposit',data=bank)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "secondary    5869\n",
       "tertiary     3784\n",
       "primary      1509\n",
       "Name: education, dtype: int64"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bank.education.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "management       2575\n",
       "blue-collar      1990\n",
       "technician       1838\n",
       "admin.           1334\n",
       "services          923\n",
       "retired           778\n",
       "self-employed     405\n",
       "student           360\n",
       "unemployed        357\n",
       "entrepreneur      328\n",
       "housemaid         274\n",
       "Name: job, dtype: int64"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bank.job.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4><left>- helper methods.</left></h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper methods\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "def split_data(data,y_label,test_size=0.2):\n",
    "    bo = uts.booleanEncoder() \n",
    "    \n",
    "    X = data.loc[:,~data.columns.isin([y_label])]\n",
    "    y = data.loc[:,y_label]\n",
    "    X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2,random_state=42)\n",
    "    \n",
    "    y_train = bo.fit_transform(y_train)\n",
    "    y_test = bo.fit_transform(y_test)\n",
    "    return X_train, X_test,y_train,y_test\n",
    "\n",
    "def get_metrices(model,X_test,y_test):\n",
    "    y_predict= model.predict(X_test)\n",
    "    return accuracy_score(y_test, y_predict),precision_score(y_test, y_predict),recall_score(y_test, y_predict),f1_score(y_test, y_predict)\n",
    "\n",
    "def print_results(accuracy,precision,recall,f1):\n",
    "    print('Accuracy: {:.2f}'.format(accuracy))\n",
    "    print('Precision: {:.2f}'.format(precision))\n",
    "    print('Recall: {:.2f}'.format(recall))\n",
    "    print('F1: {:.2f}'.format(f1))\n",
    "    \n",
    "def descritize_column(column,bins):\n",
    "    return pd.cut(column.values,bins=bins[column.name],right=False).codes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"prep\"></a>\n",
    "<h3><B><left>- Preprocessing:</left></B></h3>\n",
    "<h4><I><left>- call our estimators from 'utils.py' file.</left></I></h4>\n",
    "<h4><I><left>- create preprocessing piplines for both numerical and categorical features.</left></I></h4>\n",
    "<h4><I><left>- splitting data then fitting on train, and transorming on test.</left></I></h4>\n",
    "\n",
    "--------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"call\"></a>\n",
    "<h4><I><left>- call our unknownImputer from 'utils.py' file for filling null values depends on values in another columns, we ellaborate that in our paper.</left></I></h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>job</th>\n",
       "      <th>marital</th>\n",
       "      <th>education</th>\n",
       "      <th>default</th>\n",
       "      <th>balance</th>\n",
       "      <th>housing</th>\n",
       "      <th>loan</th>\n",
       "      <th>contact</th>\n",
       "      <th>day</th>\n",
       "      <th>month</th>\n",
       "      <th>duration</th>\n",
       "      <th>campaign</th>\n",
       "      <th>pdays</th>\n",
       "      <th>previous</th>\n",
       "      <th>poutcome</th>\n",
       "      <th>deposit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>59</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>2343</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>1042</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>56</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>45</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>1467</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>technician</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>1270</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>1389</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55</td>\n",
       "      <td>services</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>2476</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>579</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>no</td>\n",
       "      <td>184</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>673</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11157</th>\n",
       "      <td>33</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>single</td>\n",
       "      <td>primary</td>\n",
       "      <td>no</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>20</td>\n",
       "      <td>apr</td>\n",
       "      <td>257</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11158</th>\n",
       "      <td>39</td>\n",
       "      <td>services</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>733</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>16</td>\n",
       "      <td>jun</td>\n",
       "      <td>83</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11159</th>\n",
       "      <td>32</td>\n",
       "      <td>technician</td>\n",
       "      <td>single</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>29</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>19</td>\n",
       "      <td>aug</td>\n",
       "      <td>156</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11160</th>\n",
       "      <td>43</td>\n",
       "      <td>technician</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>cellular</td>\n",
       "      <td>8</td>\n",
       "      <td>may</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>172</td>\n",
       "      <td>5</td>\n",
       "      <td>failure</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11161</th>\n",
       "      <td>34</td>\n",
       "      <td>technician</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>9</td>\n",
       "      <td>jul</td>\n",
       "      <td>628</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11162 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       age          job  marital  education default  balance housing loan  \\\n",
       "0       59       admin.  married  secondary      no     2343     yes   no   \n",
       "1       56       admin.  married  secondary      no       45      no   no   \n",
       "2       41   technician  married  secondary      no     1270     yes   no   \n",
       "3       55     services  married  secondary      no     2476     yes   no   \n",
       "4       54       admin.  married   tertiary      no      184      no   no   \n",
       "...    ...          ...      ...        ...     ...      ...     ...  ...   \n",
       "11157   33  blue-collar   single    primary      no        1     yes   no   \n",
       "11158   39     services  married  secondary      no      733      no   no   \n",
       "11159   32   technician   single  secondary      no       29      no   no   \n",
       "11160   43   technician  married  secondary      no        0      no  yes   \n",
       "11161   34   technician  married  secondary      no        0      no   no   \n",
       "\n",
       "        contact  day month  duration  campaign  pdays  previous poutcome  \\\n",
       "0       unknown    5   may      1042         1     -1         0  unknown   \n",
       "1       unknown    5   may      1467         1     -1         0  unknown   \n",
       "2       unknown    5   may      1389         1     -1         0  unknown   \n",
       "3       unknown    5   may       579         1     -1         0  unknown   \n",
       "4       unknown    5   may       673         2     -1         0  unknown   \n",
       "...         ...  ...   ...       ...       ...    ...       ...      ...   \n",
       "11157  cellular   20   apr       257         1     -1         0  unknown   \n",
       "11158   unknown   16   jun        83         4     -1         0  unknown   \n",
       "11159  cellular   19   aug       156         2     -1         0  unknown   \n",
       "11160  cellular    8   may         9         2    172         5  failure   \n",
       "11161  cellular    9   jul       628         1     -1         0  unknown   \n",
       "\n",
       "      deposit  \n",
       "0         yes  \n",
       "1         yes  \n",
       "2         yes  \n",
       "3         yes  \n",
       "4         yes  \n",
       "...       ...  \n",
       "11157      no  \n",
       "11158      no  \n",
       "11159      no  \n",
       "11160      no  \n",
       "11161      no  \n",
       "\n",
       "[11162 rows x 17 columns]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here we try to descritize ages.\n",
    "\n",
    "# import numpy as np\n",
    "# bins = {\n",
    "#     'age' : [0,18,30,40,55,np.inf],\n",
    "#     'balance' :  [0,500,2000,2800,3700,np.inf]\n",
    "# }\n",
    "# bank[['age','balance']]=bank[['age','balance']].apply(descritize_column,bins=bins) \n",
    "\n",
    "je_imputer = uts.unknownImputer(nan_column='job',related_column='education')\n",
    "bank = ui.fit_transform(bank)\n",
    "\n",
    "ej_imputer = uts.unknownImputer(nan_column='education',related_column='job')\n",
    "bank= ej_imputer.fit_transform(bank)\n",
    "bank"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"pipline\"></a>\n",
    "<h4><I><left>- create preprocessing piplines for Numerical, Categorical and Boolean features.</left></I></h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import OneHotEncoder, RobustScaler, StandardScaler, MinMaxScaler\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "\n",
    "\n",
    "numeric_preprocess = Pipeline([('fields',uts.featureSelector(['age','duration','campaign','day','pdays','previous','balance'])),\n",
    "#                                (\"polynomial_features\", PolynomialFeatures(degree=3)),\n",
    "                               ('scaler',StandardScaler())\n",
    "                              ])\n",
    "\n",
    "cat_preprocess = Pipeline([('fields',uts.featureSelector(['job','marital','education','contact','month'])),\n",
    "                           ('encoder', OneHotEncoder(sparse=False)),])\n",
    "\n",
    "boolean_preprocess = Pipeline([('fields',uts.featureSelector(['housing','loan'])),\n",
    "                               ('encoder',uts.booleanEncoder())])\n",
    "\n",
    "preprocess_features = FeatureUnion(transformer_list = [\n",
    "    ('num',numeric_preprocess),\n",
    "    ('cat',cat_preprocess),\n",
    "    ('bool',boolean_preprocess)\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"split\"></a>\n",
    "<h4><I><left>- split data then fit on train, transform on test.</left></I></h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = split_data(bank,'deposit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = preprocess_features.fit_transform(X_train)\n",
    "X_test = preprocess_features.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"model\"></a>\n",
    "<h3><B><left>- Modeling:</left></B></h3>\n",
    "<h4><I><left>- create model pipline.</left></I></h4>\n",
    "<h4><I><left>- implement grid search technique for tunning model parameters.</left></I></h4>\n",
    "\n",
    "----------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[00:13:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[00:13:35] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[00:13:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[00:13:37] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Accuracy: 0.85\n",
      "Precision: 0.82\n",
      "Recall: 0.87\n",
      "F1: 0.84\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.svm import SVC , LinearSVC\n",
    "from sklearn.feature_selection import SelectPercentile, SelectFromModel, RFE\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "select_from_model = SelectFromModel(\n",
    "   DecisionTreeClassifier(max_depth=10))\n",
    "\n",
    "model_pipline = Pipeline([\n",
    "#  (\"feature_selection\", select_from_model),\n",
    "#  ('classifier',SVC(kernel='poly',C=10,coef0=1,degree=2))\n",
    " ('classifier',XGBClassifier())\n",
    "#    ('classifier',SVC())\n",
    "#  ('classifier',DecisionTreeClassifier())\n",
    " ])\n",
    "\n",
    "param_grid = [{\n",
    "#    \"feature_selection__percentile\": [80,70,60],\n",
    "#     \"classifier__n_estimators\":[100,200], \n",
    "#     \"classifier__gamma\":[0.001,0.005,0.01],\n",
    "#       \"classifier__gamma\":[0.001,0.005,0.01],\n",
    "#       \"classifier__C\":[5,10],\n",
    "#     \"feature_selection__estimator__max_depth\": [7,10,15],\n",
    "#     \"classifier__max_depth\":[30],\n",
    " }]\n",
    "\n",
    "grid_search = GridSearchCV(model_pipline, param_grid, cv=3)\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "model = grid_search.best_estimator_\n",
    "acc,pre,rec,f1 = get_metrices(model,X_test,y_test)\n",
    "print_results(acc,pre,rec,f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"results\"></a>\n",
    "<h3><B><left>- Result: </left></B></h3>\n",
    "<h4><I><left>- plot ROC-AUC curve which is our destination metric.</left></I></h4>\n",
    "<h4><I><left>- show previous results, save new result into results file, the result contain(algorithm, all hyper parameters, score metrices).</left></I></h4>\n",
    "<h4><I><left>- plot learning curve.</left></I></h4>\n",
    "\n",
    "---------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"roc_auc\"></a>\n",
    "<h4><left>- plot ROC-AUC metric.</left></h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    y_predicted = grid_search.decision_function(X_test)\n",
    "except:\n",
    "    y_predicted = grid_search.predict_proba(X_test)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARUAAAEaCAYAAAA/uMM3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2deXhV1dWH35VAmBKSSIAqAYIICiiiIC2KAoKKyKCWTwG1IuBU26/U2aLgULVa2/q11VoLgqhAi6KgYrWiVKpiQUBGkQgoASRhDIMBQtb3xzkJNzf3JifJnbPe5zlP7tlnn7PXvcn9Ze+1115bVBXDMIxQkRRtAwzDSCxMVAzDCCkmKoZhhBQTFcMwQoqJimEYIcVExTCMkGKiYhhGSDFRSWBEZJqIqHsUi8i3IvIXEckMUPeHIjJPRHaLyGER+VJEJolIwwB1u4nI30XkOxEpEpFct60zIvPOjFjGRCXxeR84EcgBxgFDgGd9K4jIUGARsAsYAHQEHgJuAt4TkRSfuoOBz4BU4DqgEzAC2A78JrxvpTwiUj+S7RkeUVU7EvQApgFv+ZX9Dtjlc94YKADeCHD/2UAJcJdf3XlB2suoxBYB7gA2AIeBPOBx91oOoEAPv3sUGO5XZyTwAfA98Av35xC/+y4GjgIt3PNWwCxgj3u8DXSI9u8nUQ/rqdQhRORkYCDOF66US4As4En/+qq6DFgAjPKrG7BHoqp7K2n+MeAB4HGgC/A/wJbqvQNw738W6Ay8BrwFXONX5xrgPVXNF5HGwIdAEdAH6IXTq3rfvWaEmHrRNsAIOwNF5ACQDJT6R273ud7R/bkuyP1rgRvd1x2qqBsQEUkFfgmMV9UX3OJc4NPqPMflT6r6qs+zXwZmikiaqu4XkUbAFcDNbpUROL2kG7S0yyRyM5APDAb+UQMbjEqwnkri8xHQDegJ/AmYD/yxGvcLzrCj9HVN6Aw0wOn11JalfufzgUM4QgIwFMfOue55d6AdsF9EDrgCuw/IBNqHwB7DDxOVxOeQquaq6ipV/V8cv8gDPte/cn92DnJ/Jxw/iG/dTtW0oSoxKvGvV4kT9qDviaoeBWZzfAh0DTBHVQ+550nAChxh9T06An/1aL9RDUxU6h4PAfeIyEnu+Xs4sz53+VcUkbOB/sArPnV3AvcGerCIZARpcy2Oc7Z/kOsF7s8Tfcq6BakbiJeB/iLSGcdn9LLPtWXAKcBOV1x9j93VaMPwiIlKHUNVFwJrgPvd84M4PpPLROQFETlLRNqIyAhgHvAf4P986o7D8dO8LSIXiUiOiJwtIo9wXHz829zvPuNxEblBRNqLSE8RudW9/j2wGEfsuojIucBT1XhPHwPfADNwRO8Dn8uvADuAuSLSR0TaicgFIvI7EekQ4HFGLTFRqZv8HhgrIm0BVPV14AKgBc4XcgNOj2YycLGqHim9UVXn4sygHMLpEazHGX60Bu6upM37gCdwhl7rcGZusn2uj3F/LsEZltxfzff0CnAmMFNVj/nYe8h9bxtdO78EXsTxqeypZhuGB8R1iBuGYYQE66kYhhFSTFQMwwgpJiqGYYSUiIiKO6uQLyKrg1wXEfmju9p1pTuVaRhGHBKpMP1pwJ+B6UGuX4oTAt4B+CHwF/dnpWRlZWlOTk5oLDQMo4zPP/98p6o2r8m9EREVVf1IRHIqqTIMmO6uzVgsIhkicqKqbq/suTk5OSxd6h+1bRiB+dtHG3n6/a84eORY1ZXrOp8P/qamt8bKgsJWlF+xmueWVRAVEbkJJ88Hbdq0iYhxRvWxL3DdJVZEJdDakIABNKr6PPA8QI8ePSzIJsSYGBi1JVZEJQ8nIrOUbGBblGypE9RV8WiSksz4AR258YKTo21KzFBUVMycOesYNep4NlB5oubPixVRmQf8TERm4Tho91XlTzG8EU3xsC9w7FNUVMzll8/i3Xe/5uuvd/PAA31q/cyIiIqIzAT6AlkikgdMAuoDqOpzODkxBuEk7jkE3BAJuxKdv320kUfnVyufEmBiUFfwFRSAiRMX0qdPDhdc0LZWz43U7M/IKq4rcFskbElUatIjMfGou/gLCsCkSX1qLSgQO8MfoxZ46ZFMGNTJxMMAggvKgw/2DcnzTVQSgKff/yroNeuNGL6EW1DARCVuCTbcsR6JEYxICAqYqMQVVflNmqQkm6AYAYmUoICJSlzgxQlbOswxjECsXp3PRx8dj7wPl6CAiUrMU5kT1vwlhld69DiJt94axeDBM7j77vPCJihgohLTBBIUExKjplx4YTvWrPkp7dplhrUdE5UYxJywRm0pKipm585DZGc3LVcebkEBE5WYojLfiQmK4ZVSp+yXX+5k4cLR5OQE244pPJioxACViYkNd4zq4D/L07fvNFasuIWMjIZV3Bk6TFSiTDBHrImJUV0CTRuPHt0tooICJipRxRyxRqiIZBxKVZioRIlAgmJ+E6MmxJKggG3RETX81+uYoBg1IdYEBUxUosLfPtpYzilrgmLUhFgUFLDhT0QJNMtj63WMmvKHP3wac4IC1lOJGKU+FP9pY1uvY9SUO+44lyuuOA2IHUEB66lEBJvlMcJBSkoys2YN5/XX13H11adH25wyTFTCiIXbG6Hk6NFj1KuXhMjxHW1SUpJjSlDAhj9hI9hwxwTFqAlFRcUMGTKTiRM/xEnpHLtYTyUM2HDHCCW+szyljtmHH+5XrscSS5iohAGLQTFCRaBp4+TkpJgVFLDhT8ixGBQjVMRqHEpVmKiEGN9eisWgGDUlXgUFTFRCin8vxWJQjJoQz4ICJiohw985a70UoybEu6CAiUrI8HfOWi/FqAnXXfd6XAsKmKiEDHPOGqHgF7/4IU2a1AfiU1DAppRDwt8+2lju3ATFqCm9e7fhn/+8lkWLvuG++86Ptjk1wkQlBPjP+BhGbejduw29e7eJthk1xoY/tcRmfIyaUlRUzJgxc8nN3R1tU0KKiUotsbgUoyaUzvJMnbqCvn2nJZSwmKjUEuulGNXFf9p469b9vPnm+ihbFTpMVGqBOWiN6hIsDuWXv+wVRatCS8RERUQGish6EckVkXsDXG8jIh+KyHIRWSkigyJlW00xB61RHRIhsM0LEREVEUkGngEuBToDI0Wks1+1+4F/qOpZwAjg2UjYVlPMQWtUh7oiKBC5nkpPIFdVN6rqEWAWMMyvjgKlu0mnA9siZFuNMAet4ZW6JCgQOVFpBWzxOc9zy3x5ELhWRPKA+cDPAz1IRG4SkaUisrSgoCActnrCeimGF+qaoEDkRCVQRhn/nHgjgWmqmg0MAl4SkQr2qerzqtpDVXs0b948DKZWjTloDa/Uq5dUbi/jRBcUiFxEbR7Q2uc8m4rDm7HAQABV/VREGgJZQH5ELKwG5qA1vFKvXhIvv3wlAKedlpXwggKRE5UlQAcRaQdsxXHEjvKr8y3QH5gmIp2AhkD0xjdBMAetUV3q1Utixowfk5QUuykgQ0lEhj+qWgz8DHgXWIczy7NGRB4WkaFutTuAG0XkC2AmMFpjMG24OWiNyigqKmbKlGUVMt7XFUGBCC4oVNX5OA5Y37KJPq/XAudFyp6aYL0UozJ8nbJffrmTJ5+8KKYTVIcLi6j1iGV2MyrDf5bnqac+5f33N1ZxV2JiouIRy+xmBCPYtPFFF7WPolXRw0TFI5bZzQhEXYxDqQoTFQ9YXIoRCBOUwJioeMDiUgx/TFCCU21REZEW4TAkVrEZH8MfE5TK8SQqIpIuItNFpAjY5JYNEZGHwmpdDGBxKYY/Gzbs4pNPji9lM0Epj9eeyl+Aw0AH4Ihb9hnOep2Exnophj9nnNGS9967jrS0FBOUAHgNfhsAZKvqERFRAFXNF5GW4TMt+piD1gjGj36Uzdq1t5Gd3bTqynUMrz2VQuAE3wIRaQ3sCLlFMYQ5aA1wfCibNu2pUG6CEhivovICMFtEzgeSROQcYCrw17BZFgPY0Mcodcqee+4LrF+/M9rmxAVeReVxYC4wBWf18Azgn8AfwmRXzGFDn7qH7yzPd98doG/fF9m581C0zYp5vPpUmqnqU8BTvoUikgUknHz/7aONFcLyjbpFoGnjm2/uTlZW4yhaFR947akEWxmVkN+8p9//qtzQx/wpdQuLQ6kdXkWlwvptEUkFSkJrTmzgLyjmT6k7mKDUnkqHPyKyCSeXbCMR8e+tZAGvhcuwWGHNwwOjbYIRIUxQQkNVPpVxOL2UecCNPuUK7FDVNeEyzDAiiQlK6KhUVFR1AYCI/EBVCyNjUnTxD3gz6gbPPrvEBCVEeJr9UdVCETkdOB9n2CM+1x4Ok21RwQLe6ia/+MUPWb78O15+eaUJSi3xJCoiMhb4E7AAuAj4F07m+zfDZ1p0sIC3uklychLTpg3jyitP44orOkXbnLjG6+zPvcAgVR0CfO/+vAo4GDbLooCt9ak7HDlyrELG++TkJBOUEOBVVFqq6kL3dYm7c+DbwOVhsSpK2NCnblBUVMzQoTMZP/6fFYTFqD1eI2rzRKStqn4DbAAuw4mkPRo2y6KADX0SH99ZnlLH7NNPD6yTW2mEC6+i8jvgdOAb4NfAbKA+cHuY7Io6NvRJPAJNG2dmNjJBCTFeZ3+m+Lx+S0QygQaqui9slhlGCLE4lMhRo8TXqloE1BORx0Nsj2GEHBOUyFKlqIjI9SLyBxH5qYjUE5GmIvJbYDNwdtgtNIxaYIISeapa+/MkcB3wCU4+2h8BvYDPgd6q+kXYLYwQFkmbeJigRIeqfCojgAtUdYOIdALWACNV9e/hNy2y2HRy4nHDDXNNUKJAVcOfDFXdAKCq64BDiSgotrdPYnLnnb3IyGgImKBEkqp6KuImuC6dcyv2O0dVvw2XcZHC9vZJTLp3P4n337+O99/fyD339I62OXWGqkSlCY5D1nci/xuf1wrE9VjBeimJTffuJ9G9+0nRNqNOUdXwpz6Q4v4MdKSE1boIYL2UxKCoqJjrrnud1avzo21KnadSUVHVY1UdkTI0XFgvJf4pneV5+eWVXHjhiyYsUaZGwW81QUQGish6EckVkXuD1LlKRNaKyBoRmRFum2xVcvzjP21cUHCId97ZEGWr6jZe1/7UChFJBp7BycWSBywRkXmqutanTgfgPuA8Vd0jIi3CbZdNI8c3weJQ7rrrvChaZUSqp9ITyFXVjap6BJgFDPOrcyPwjKruAWev5nAbZUOf+MUC22IXz6Lihuj3EpHh7nkjEWnk8fZWwBaf8zy3zJeOQEcR+VhEFotIwDT2InKTiCwVkaUFBQVeza+ADX3iFxOU2MaTqIhIF+BL4CVgmlvcH2ePZU+PCFDmnx2nHtAB6IuzJGCyiGRUuEn1eVXtoao9mjdv7rH5itjQJz4xQYl9vPZU/gL8WlVP4XhipoU4ibC9kAe09jnPBrYFqDNXVY+q6iZgPY7IhAUb+sQfhw+boMQDXkXlDOBF97UCqOoBwOvGskuADiLSTkRScNYUzfOr8wbQD8r2aO5I8O1WQ4oNfeKD+vWTyc5uWnZughKbeJ39+QY4C1hWWiAiPYCvg97hg6oWi8jPgHdxInBfUNU1IvIwsFRV57nXLhaRtcAx4C5V3eX9rRiJTlKS8PzzQwDIzm5qghKjeBWVicDbIvIskCIidwG3Abd6bUhV5wPz/com+rxWnPSUCZui0qg9SUnC3/42xFJAxjCehj9uT2Iojl/kY+BU4CpVfSeMtoUNy50SHxQVFfPMM/+lpKS8T98EJbbxuplYpqouwfGNxD028xP7+M7yrFqVz7PPXkZSkolJPODVUbtVROaJyNXViE2JWWzmJ7bxnzb+618/Z/58C72PF7yKSjvgfeCXwA4ReUlELnXD7+MKC3qLbYLFoQwebOIfL3j1qexQ1T+q6o+AbjgxJE9RMdYk5rGhT+xigW2JQU3W/qS7RxpxuJeyDX1iExOUxMFrmH5HEZkkIuuBd4CGwAhVjeuxgw19YgMTlMTCa5zKEuB14H+B9+M1OZNNJcceJiiJh1dRaenuShjXmD8l9vjmm70sXXrcNWeCEv8EFRURGamqM93Tq4IFHKnq9HAYFg7MnxJ7nHpqFgsW/IT+/afzs5/1NEFJACrrqYwGSkXlxiB1FIgbUfHF/Cmxw5ln/oDVq3/KD36QGm1TjBAQVFRU9RKf115THMQs5k+JDYqKitm8eS+nnZZVrtwEJXHwOvsTMDxfRBaH1pzwYf6U6FPqlD3vvBdYvnx7tM0xwoTXOJXTgpTHjWPC/CnRxXeWZ/fu7xkw4CW+++5AtM0ywkClsz8iUpouMsXndSk5wLpwGBVuzJ8SWQJNG//85z1tyJOgVDWlvDXIawU+BxJus3YjtFgcSt2jUlFR1QfA8Z2o6tuRMSn0mJM2Opig1E0qi1M5T1U/dk/3i8gFgeqp6kdhsSyEmJM28pig1F0q66lM4biD9pUgdRRoE1KLwoA5aSOLCUrdprI4ldN8XrcOVi/eMCdt+HnhheUmKHWYGm17KiLni0ivUBtjJAa33NKDW2/tAZig1EW8Br8tFJHz3dd3AnOAOSJyTziNM+KTpCThz38exFtvjTRBqYNUZzOxT93XN+NsTfpD4KdhsMmIMw4fLq6Q8T4pSbjsMvNf1UW8ikoSUCIiJwP1VHWNqn4LnBA+04x4oKiomGHDZjFu3LwKwmLUTbzmU/kEeBo4CSdZE67A2A6CdZhAszxTpgy1fXnqOF57KqOBIpyE15Pcss7An8JgkxEHBBKUNm3STVAMbz0VVS0A7vYrewt4KxxGhRKLpg09FodiVIbX2Z96IvKAiHwlIgfdnw+ISP1wG1hbLJo2tJigGFXh1afyBHAeMB74BmgL3A9kAHeEx7TQYNG0ocMExfCCV1G5CjhLVXe652vcxE0riHFR8cWiaWuOCYrhFa+O2mSgxK+sBDCvXB3hxhvfNEExPOFVVF4F5olIfxHpICIDcKaWXwufaUYsce+959GiRRPABMWoHK/Dn7twppKnACfi7KE8C3goTHYZMUaXLi344IOf8M47udx557nRNseIYbxOKR8GfuUeNUJEBgL/hzOUmqyqvwlSbzgwGzhHVZfWtD0j9HTp0oIuXVpE2wwjxql0+OMOdT4Skd0i8r6I1Ch3iogkA88Al+IEzY0Ukc4B6qXhbK36WU3a8cdiVGpGUVExI0a8Wm7nQMPwSlU+lT/j5KYdDezECdWvCT2BXFXdqKpHcIZOwwLUewR4Eid6t9ZYjEr1KZ3l+fvf1zBgwHQTFqPaVCUq3YExqjoPZ5fCH9awnVbAFp/zPLesDBE5C2jtRuoGRURuEpGlIrK0oKCg0kYtRqV6+E8b79t3mAULrLdnVI+qRCVFVb8HUNX9QKMathNo6rlsSauIJAF/wEPMi6o+r6o9VLVH8+bNPRtgMSqVEywO5Z57ekfRKiMeqcpR20BEJvqcN/I7R1Uf9tBOHuCbkjIbZwaplDTgdGChuyDtBzhT2EPNWRt+LLDNCCVVico/gA4+56/6nXtNoLEE6CAi7XB8NCOAUWUPUd0HlG2uKyILgTtNUMKPCYoRaqra9+e6UDSiqsUi8jPgXZwp5RdUdY2IPAwsdX02RoQxQTHCgdfgt1qjqvOB+X5lE4PU7RsJm+oyR44cM0ExwkKNsukb8U/9+kl06lQ24jRBMUJGxHoqRmwhIvz+95cAkJ7e0ATFCBkJKyoWTVs1pcJiKSCNUOJ5+CMi/UTkryLyhnt+toj0CZ9ptcOiactTVFTM7373CceOlc9gYYJihBqv6SR/irNCeQvQzy0+AjwaJrtqjUXTHqd0lufOO//FT37yRgVhMYxQ4rWncgcwQFV/zfFkTeuATmGxKsTU5Wha/2njGTNWMXfu+ihbZSQyXkUlDSc3LRwPeKuH01sxYpRgcShXXhkX/wuMOMWrqPwHuNOv7Dbg36E1xwgVFthmRAuvsz8/B94SkRuBNBFZg9NLGRQ2y2pBXZ/5MUExoonXzG9bRaQ70Atog+Ow/VRVj1V+Z3SoyzM/JihGtPEcp6KqJcDH7hHT1NWZHxMUIxbwJCoisokgK5JVNaanVurSzM/27ftZuXJH2bkJihENvPZUxvmdn4jjZ5kZWnOM2tCuXSYLF46mb99p3HRTdxMUIyp49aks8C8TkQU4q45rmrfWCAMdOzZj1apbadascbRNMeootVml/D0Qc2OLujTzU1RUXG64U4oJihFNvPpU/POeNAYuA94LuUW1pK7M/JQ6ZT/5ZAv//Oe1nHtu66pvMowI4LWn0sHvyMDZxyckmeFCSV2Y+fGd5dm//wgDB77Mli37om2WYQAeeiruRmD/Av6hqiHZjydSJOLMT6Bp49tv70Xr1ulRtMowjlNlT8UNcPtTvAlKImJxKEY84HX487aIxGRIfl3BBMWIF7zGqSQBc0TkPzgh+mWBcKo6JhyGGccxQTHiCa+isgH4bTgNMQJjgmLEG5WKioiMVNWZqvpApAyqDYkYozJjxioTFCOuqMqn8teIWBEiEjFG5YYbunH33ecCJihGfFDV8CeusiInYoyKiPCb3wzgoovaM2BA4k2RG4lHVaKSLCL9qERcVPWD0JoUGuI1RqWoqJjkZKF+/eM9LRExQTHihqpEpQFOFv1goqLE4PqfeKXUKZuW1oAZM64sJyyGES9UJSoHYz1fSqIQaJZn1qwfk5xsO9Ma8YX9xcYAgQSlS5fmJihGXFLVX21cOWrjEYtDMRKNSkVFVdMiZUhdxATFSESsfx0lTFCMRMVEJQqYoBiJjIlKFPjpT982QTESloiJiogMFJH1IpIrIvcGuH67iKwVkZUiskBE2kbKtkgzYcL5ZGc3BUxQjMTD82ZitcHNHvcMcBGQBywRkXmqutan2nKgh6oeEpFbgSeBqyNhX6Rp3/4EFi68nnnz1vPLX/aKtjmGEVIi1VPpCeSq6kZVPQLMAob5VlDVD1X1kHu6GMiOkG1RoX37E0xQjIQkUqLSCie5Uyl5blkwxgLvBLogIjeJyFIRWVpQUBBCE8NDUVExw4f/g48++ibaphhGRIiUqAQKogu4jaqIXAv0IEhSKFV9XlV7qGqP5s2bh9DE0FM6y/Paa+sYNOgVExajThApUckDfDemyQa2+VcSkQHABGCoqh6OkG1hwX/a+ODBo/znP99G2SrDCD+REpUlQAcRaSciKcAIYJ5vBRE5Cycp1FBVzY+QXWEhWBzKr351fhStMozIEBFRUdVi4GfAu8A6nD2E1ojIwyIy1K32WyAVmC0iK0RkXpDHxTQW2GbUdSIypQygqvNxNnT3LZvo83pApGwJFyYohmERtSHDBMUwHExUQkBxcYkJimG4JIyoRHN7juRk4Yc/PB52Y4Ji1GUi5lMJN9HcnkNEykTE97Vh1EUSRlSivT2HiPDQQ/0i3q5hxBoJM/zxJdzbcxQVFfPYY4s44iNkhmE4JKSohJPSWZ4JEz7gqqtmm7AYhh8mKtXAf9p47tz1zJmzLspWGUZsYaLikWBxKCNGnB5Fqwwj9hDVgIuF44IePXro0qVLKSws5J9L15PRMAlByM5sFNJ2VJX8/IMUFRWXlaWnNyQjo2FI2zGMSFG/fn1atGhB06ZNA14Xkc9VtUdNnh33sz+FhYV8tTmPU9u1gXopiAidsjNC9vySEiU3dzepqYdJTXXKTjopjZNOst1LjPhEVfn+++/ZunUrQFBhqSlxP/zJz88npWkWUr8BIkKShG7/s1JBKSw8noXBBMWId0SExo0b06pVK/LzQ58QIO5F5ejRo2hy/bLzlk1DMyQxQTESnUaNGnH06NGQPzfuRQUc5S2leVqDkDyzuLiknA/FBMVINCSEvXpfEkJUwkFKSjKnntqMlJRkExTDqAYmKpXQoEE9OnduboISYfLz82nbti27d++OtilxS0FBAW3btmXnzp0Rb9tExaWkRDl48EiF8nr1QvcR9e3blwYNGpCamkp6ejrdunVj9uzZFep9+umnDBw4kPT0dFJTU+nevTsvvvhihXrbt2/n1ltvpW3btjRp0oQ2bdpw1VVX8fnnn4fM5mgwadIkrr/+ek444YRomxIy8vPzufLKK0lLS6N58+bcc889lJSUBK1/6NAhxo8fT3Z2NmlpaVx22WV8++3xHMfz58/nwgsvJCsri8zMTM4//3wWLVpUdr158+aMGjWKhx56KKzvKxAmKhx3yn755a5yjtlw8MADD3DgwAF27drF6NGjGTVqFLm5uWXX33vvPfr160evXr3YuHEj+fn53HPPPYwfP55JkyaV1du2bRvnnHMOW7ZsYf78+RQWFrJ27VqGDBnCnDlzwvoewJmWLC4urrpiNdm7dy/Tp09n3LhxNX5GOJyPteWaa64BIC8vj88++4zXX3+d3/424IYRANx1110sWbKEZcuWsWPHDrKyshg8eHCZEO3Zs4ef//zn5ObmUlBQwKhRo7j00kvZsuX4Tjhjxoxh6tSpFBYWhvfN+aOqcXt0795d165dq19s2VN2VJdjx0p0/fqdumTJVl2yZKsuXbpNi4qOVvs5XujTp48+8sgjZecHDhxQQGfPnl1Wdsopp+jo0aMr3Dt16lRNTk7WTZs2qarq2LFjtWPHjnrkyJFq2bBw4ULt3bu3ZmZmarNmzcra+vDDDzU5Oblc3UmTJmn//v3LzgF9+umntXv37tqwYUNdtGiR1q9fX/Pz88vqlJSUaE5Ojr744ouqqnrw4EG94447NCcnRzMzM/WSSy7RDRs2BLVv1qxZeuqpp5YrW7FihV5wwQXarFkzzcjI0IEDB2pubm7Z9euvv15HjRqlo0eP1szMTL3llltUVXXVqlV68cUXa7NmzbR169Z67733lvu8Ro8erdnZ2ZqamqqdOnXSV155pVqfpVc2btyoQDmbJ0+erDk5OUHvad68ub7xxhtl55s3b1ZA//3vfwe9JysrS+fMmVOurG3btvraa68FvWft2rUBy4GlWsPvZdwHvwUi5963a/eAiiOSKtn8m8uqVf/IkSP85S9/AaBjRydVw1dffUVubi7PPfdchfqjRo1i7Nix/Otf/+LGG29k/vz5jBkzhvr161eoG4yVK1dyySWX8NxzzzFy5EhKSkpYvHhxteyeMqysxYAAABHOSURBVGUKr7/+Ojk5ORQXF9OtWzdeeeUVxo8fD8DChQvZtWsXw4cPB2DcuHEUFhayePFiMjMzefTRRxk8eDCrVq0KaPuyZcvo3LlzuTInR82DnHvuuRQVFTFu3DiuvfZaPv3007I6s2fP5qWXXmLy5MkcPnyY/Px8+vTpw2OPPcabb75JQUEBw4YNo1GjRkyc6KRG7t27N0899RQZGRnMnj2bn/zkJ3Tr1q1C+6V07dq13BDEn7feeovevXtXKP/iiy9IT0+nffv2ZWVnn302mzdvprCwMGDwWekXtJTSHsqKFSu44IILKtRfuXIlu3bt4vTTyy8bOeOMM1i2bBlXXnllULtDjQ1/Isyjjz5KRkYGjRo14v7772fy5Ml07doVcJxrAK1aVdy8MSUlhaysrLJgpYKCgoD1KuO5555jyJAhjB49mgYNGtCoUSP69ateDpg777yT9u3bk5ycTIMGDbjhhhuYOnVq2fWpU6dy9dVX07hxY3bu3MnMmTN59tlnadmyJSkpKUyaNInt27fz2WefBXz+nj17KnzJunbtSr9+/WjQoAHp6elMmjSJxYsXc/DgwbI6vXv35uqrryY5OZnGjRszffp0zjzzTG6++WZSUlJo1aoV9913H9OnTy+7Z+zYsTRr1ozk5GRGjBhB165dWbhwYdD3vnLlSvbu3Rv0CCQoAPv37yc9Pb1cWUaGE/UdbGgyePBgnnjiCbZv387+/ft54IEHEJGA9fPz8xk+fDh33303HTp0KHetadOmEXd4m6hEmAkTJrB371527tzJoEGD+OCDD8qule64WBo+7cuRI0fYuXNnWZ3mzZsHrFcZmzdvLusV1ZScnJxy5yNHjuSrr75i2bJl7N+/n9dee40xY8YAsGnTJsARhYyMDDIyMjjhhBM4evRoubG/L5mZmRW+OF9//TVXXnklrVq1omnTppx33nkA5WY2/O3atGkTH3/8cVm7GRkZjBkzhu+++w5w/vNPnDiRU089lfT0dDIyMvjiiy8Ix1a6aWlp7Nu3r1zZ3r17y64F4umnn+aMM87gnHPO4bTTTqN79+6kpqaSlZVVrt62bdvo168fF198MY8//niF5xQWFkbc4Z2Qw5/KhiKxEimbmZnJ5MmTad++PXPnzmXYsGF06NCBk08+mRkzZtC/f/9y9WfNmoWIcNFFFwEwaNAgXn31VSZNmuR5CJSTk8OGDRsCXktNTeXYsWMcPnyYBg2cAMJt2ypsIklSUvn/QxkZGVx++eVMmzaNM888kzZt2tCrl7PxfNu2bQHYsGEDXreoPeuss5g7d265sltuuYWTTjqJlStX0qxZM1avXs0ZZ5xRbnjgb1fbtm0ZMGAAb78deCg8c+ZMJk+ezHvvvUfnzp1JSkqiR48e5Z7pT5cuXfjmm+Bb177zzjucf37FDePOPPNM9u3bx8aNGzn5ZCeB2PLly8nJyanQgyklPT2d559/vux8zZo13H777fTp06esbPPmzfTv358rrriCp556KuBzVq9ezejRo4PaHBZq6oyJhaO6jlp/p+ySJVt169bCSu8JJf6OWlXVhx9+WDt37qzHjh1TVdX58+drSkqKPvLII7pr1y49dOiQzp49WzMzM/VXv/pV2X15eXnaqlUrHTp0qK5du1aLi4v1wIEDOmPGDJ0wYULA9lesWKENGjTQ6dOn6+HDh/XQoUP64YcfqqpqYWGhpqam6p///Gc9duyYLlq0SDMzMys4ahctWlThue+++642a9ZMe/bsqU888US5a6NGjdLhw4drXl6eqqru2bNH58yZo/v37w9o4+7du7Vx48b67bfflpX17NlTb731Vi0uLtaCggK94oorFChzWl9//fU6duzYcs/Zvn27tmjRQqdMmaLff/+9Hjt2TL/++mt95513VFX12Wef1datW+t3332nR48e1SlTpmi9evV00qRJAe2qLQMGDNAf//jHum/fPt24caN27NhRH3/88aD1N27cqNu2bdOSkhJdt26d9uzZs5wDf926ddqqVaugv2tV1Q0bNmiTJk107969QeuEw1EbdWGozVFdUdm161DUBEU1sKjs27dPMzMzderUqWVlixYt0osuukjT0tK0cePG2q1bN50yZUqF523btk1vvvlmzc7O1saNG2vr1q31qquu0mXLlgW1YcGCBdqrVy9NT0/XrKwsHTNmTNm12bNna7t27TQ1NVWHDx+u48eP9yQqx44d09atW2tycrJu37693LWDBw/qhAkT9JRTTtHU1FTNzs7WkSNH6oEDB4LaePPNN5f7cn/88cd6+umna+PGjfW0007TKVOmVCkqqqpr1qzRIUOGaMuWLbVp06batWtXfeaZZ8rsGj58uKampmqLFi30jjvu0H79+oVNVHbs2KFXXHGFpqamarNmzfSuu+4q+0dS+p4HDhxYdv7mm29qmzZttFGjRtq6dWu9//779ejR47OSo0ePVkCbNGlS7nj55ZfL6tx333162223VWpXOEQl7vOpvPTSSxxNO7GsrGsVaQ+2b9/P1q37LfQ+htmxYwc9e/Zk+fLlCRUAF0l27txJ9+7dWbp0aaVDz3Xr1tGpU6cK5XU6n0p1OfHENFJTU0gL0cJDI/S0bNmyUt+FUTVZWVlR+wwTevanpEQpKanYEzNBMYzwkbCiUjrL8/XXuwMKi2EY4SEhRcV32njfvsN8/fXuSqcKDcMIHXEvKvuLiv3CmSvGoTRpkhK2hDSGEa9Utkq6NsS9qPx3ywEO7MpHjxUjSkwEthlGLKOqHDlyhK1bt9KkSZOQPz/uZ3+e/nQ3A3ceZsDJ+0k9VkLx0eM7BqanN2Tfvn34RUgbRp2nXr16pKenVwj7D8mzQ/7ECFOs8Oba/Uz59XKKNh1fMzJpUh8efLBXFC0zjLpJxIY/IjJQRNaLSK6I3BvgegMR+bt7/TMRyfHyXC0uIX9ObgBB6Rsq0w3DqAYRERURSQaeAS4FOgMjRcQ/acVYYI+qngL8AXjCy7N3L9higmIYMUSkeio9gVxV3aiqR4BZwDC/OsOA0kSsrwL9xcOUTXqvE6mX4QSzmaAYRvSJlKi0AnwTaOS5ZQHrqGoxsA9o5v8gEblJRJaKyNKCggLqNU2h5chTyRzQ2gTFMGKASIlKoB6HfzSalzqo6vOq2kNVe5QulKrXNIWm3VvW3krDMGpNpGZ/8oDWPufZgH8GoNI6eSJSD0gHKs2D9/nnn+/k88HfAFnATvHkhYk6WUDkN2OpPvFiJ5it4eDUmt4YKVFZAnQQkXbAVmAEMMqvzjzgeuBTYDjwgVYRW6+qzQFEZGlNl2lHmnixNV7sBLM1HIjI0preGxFRUdViEfkZ8C6QDLygqmtE5GGcZDDzgCnASyKSi9NDGREJ2wzDCC0RC35T1fnAfL+yiT6vi4D/iZQ9hmGEh7hf++PyfNVVYoZ4sTVe7ASzNRzU2M64TidpGEbskSg9FcMwYgQTFcMwQkpciUq4FiWGAw+23i4ia0VkpYgsEJG2sWinT73hIqIiErXpUC+2ishV7ue6RkRmRNpG14aqfvdtRORDEVnu/v4HRcNO15YXRCRfRFYHuS4i8kf3vawUkbOrfGhN9/aI9IEzFf01cDKQAnwBdPar81PgOff1CODvMWxrP6Cx+/rWaNjqxU63XhrwEbAY6BHDn2kHYDmQ6Z63iFE7nwdudV93BjZH4zN1278AOBtYHeT6IOAdnIj3HwGfVfXMeOqphG1RYhio0lZV/VBVD7mni3GijCONl88U4BHgSaAoksb54cXWG4FnVHUPgKrmR9hG8GanAqW70KdTMbo8YqjqR1QeuT4MmK4Oi4EMETmxkvpxJSohW5QYAbzY6stYnP8GkaZKO0XkLKC1qr4VScMC4OUz7Qh0FJGPRWSxiAyMmHXH8WLng8C1IpKHE7v188iYViOq+7ccV5nfQrYoMQJ4tkNErgV6AH0CXQ8zldopIkk4uW1GR8qgSvDymdbDGQL1xen5LRKR01V1b5ht88WLnSOBaar6OxHphRNJfrqqhicTde2o9ncqnnoq1VmUiNdFiWHCi62IyABgAjBUVQ/7X48AVdmZBpwOLBSRzThj6nlRctZ6/f3PVdWjqroJWI8jMpHEi51jgX8AqOqnQEOchYaxiKe/5XJEy0FUA4dSPWAj0I7jDrAufnVuo7yj9h8xbOtZOA69DrH8mfrVX0j0HLVePtOBwIvu6yycbnuzGLTzHWC0+7qT+yWVKP4d5BDcUXsZ5R21/63yedF6IzV884OAr9wv4wS37GGc//TgKP5sIBf4L3ByDNv6PrADWOEe82LRTr+6URMVj5+pAL8H1gKrgBExamdn4GNXcFYAF0fxM50JbAeO4vRKxgK3ALf4fKbPuO9llZffv4XpG4YRUuLJp2IYRhxgomIYRkgxUTEMI6SYqBiGEVJMVAzDCCkmKgmAiLwsIg9G246qcFfunl/J9fdE5JpI2mSEHhOVGEJENovI9yJywOc4KUq2vCwiR1wbdrtf+I61eaaqnqqqi9zn/1pEpvldv1hVX6lNG/6ISD03ZcNB973kichv3SUIXu4f4EYTGx4xUYk9hqhqqs8RtRWswGOqmooTpr0beCGKttSWLu57uRC4Dmc7GCMMmKjEASKSJCKvish3IrJXRBaKSKcgdVuIyHy33m4R+cjnWraIvC4iBSKySURu89K+qh7Eibw83X1OQzdxz3YR2SoivxeRFA/t54lIXxEZDNwNXOP2Hj53r/9HREaLSCMRKRSR03zu/YHbi2vmng8VkS/cdv4jIqd7fC9fAZ8A3XyePU5E1onIfhH5WkTGueXpwJtAG5+eYwv39/Ert+5OEZklIple2q8LmKjED2/hLI77AbAaeClIvbtw1p40d+s+ACAiye4zluAsXb8IuEtE+lfVsIik4Wz+ttwtmoizsrorzhqm84D7KmvfF3XSKDwJvOL2xrr7Xf8eeANnNW8pVwMLVHWXiJwD/A0Yh5Pa4gVgbqmwVfFeOrn25voU78BZ49IUJyfLn0Skq6ruA4YA3/r0HPOB2936F+AssDsI/LGqtusKJiqxxxvuf9+9IvIGgKqWqOo0Vd2vzv5IDwLdRaRJgPuPAicBbVT1iKr+2y3/EdBUVR9zy3NxNnCrbNO2e0VkL846lgbAGLf8GuBBVS1wv2QP4wwpKmu/usygvKiMcssAbgKeVdUlqnpMVUuHZedU8ryVInIQZ13Qv4C/ll5Q1TfVSaqkqvoBsAAI6lAGbgZ+papbfX4fV3n10yQ69iHEHperaoZ7XA5OL0NEnhSRjSJSyPH/soGWy/8G+AZY4HbP73LL2+J040sFay/OEOQHldjyG9eOE1X1cnXSCQCc6LZRyjccT9wTrP3q8j5OlrHuItIe6ALM9Xkv9/i9lxOpPHlQV5xUDqOAXkDj0gsiMlicnMa73WddTOWpCNoAb/q0vQonx0iLGr3TBMNEJT74Cc7K1wtxcsSc4pZXSKCjqoWq+ktVzQEux/ny9cFJA7DBR7AyVDVNVYfUwJ7tOF/sUtrg7JFdWfsVTK2sAXUy983G6a2MwsmTctC9vAV4yO+9NFbVf1TxzBJVnQksBe4HEJFGOKlHHwdaqmoG8B7HP9tAduYBF/m131BVv6us/bqCiUp8kAYcBnbh/Id9NFhFERkiIu1FRHDSaR5zj0+BIyJyh+toTRaRM0Ske7BnVcJMYKKIZIlIcxy/yctVtO/PDiDHrReMGTi+FN+hDziJo28TkXPEIdVtN9BwMBCPA7e4tjfAyXtSABxznci+fqYdQJbrVyrlOeAxEWnjvucWIjLUY9sJj4lKfDAVJ5HPNmANzuxFME4FPgAO4OTs+D9V/Y/7n38QTmLmzcBOHL9C0yDPqYyHcHKBrAJWAp/hfFGDth/gGX/H+TLvFpH/BmnnE6AYx+n7Xmmhqn6GswPBX4A9OD6fa70ar6orcET2TnVSTf4SeB1n2nw4jkO7tO5q4DVgszvcaYGTs+WfOEO8/a6dlflz6hSWT8UwjJBiPRXDMEKKiYphGCHFRMUwjJBiomIYRkgxUTEMI6SYqBiGEVJMVAzDCCkmKoZhhJT/B0fFPOwGif/dAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "fpr_lr, tpr_lr, _ = roc_curve(y_test, y_predicted)\n",
    "roc_auc = auc(fpr_lr, tpr_lr)\n",
    "\n",
    "plt.figure()\n",
    "plt.xlim([-0.01, 1.00])\n",
    "plt.ylim([-0.01, 1.01])\n",
    "plt.plot(fpr_lr, tpr_lr, lw=3, label='ROC curve (area = {:0.2f})'.format(roc_auc))\n",
    "plt.xlabel('False Positive Rate', fontsize=12)\n",
    "plt.ylabel('True Positive Rate', fontsize=12)\n",
    "plt.title('ROC curve', fontsize=14)\n",
    "plt.legend(loc='lower right', fontsize=13)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=3, linestyle='--')\n",
    "plt.axes().set_aspect('equal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"prev_res\"></a>\n",
    "<h4><left>- previous results and save new result</left></h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Scaler</th>\n",
       "      <th>Features</th>\n",
       "      <th>Feature Selection</th>\n",
       "      <th>Accurecy on Test</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precession</th>\n",
       "      <th>Parameters</th>\n",
       "      <th>Polynomial Features</th>\n",
       "      <th>Roc AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>StandardScaler</td>\n",
       "      <td>all</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.808330</td>\n",
       "      <td>0.780694</td>\n",
       "      <td>0.811100</td>\n",
       "      <td>{'c': 1}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>all</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.810121</td>\n",
       "      <td>0.774133</td>\n",
       "      <td>0.818632</td>\n",
       "      <td>{'c': 1}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>all</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.811464</td>\n",
       "      <td>0.777882</td>\n",
       "      <td>0.818540</td>\n",
       "      <td>(C=100)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>all</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.809225</td>\n",
       "      <td>0.773196</td>\n",
       "      <td>0.817641</td>\n",
       "      <td>(C=1000)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>all</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.778325</td>\n",
       "      <td>0.713215</td>\n",
       "      <td>0.801053</td>\n",
       "      <td>(C=0.001)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>(percentile=100)</td>\n",
       "      <td>SelectPercentile(percentile=100)</td>\n",
       "      <td>0.810121</td>\n",
       "      <td>0.774133</td>\n",
       "      <td>0.818632</td>\n",
       "      <td>(percentile=100))</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>(percentile=100)</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.810121</td>\n",
       "      <td>0.774133</td>\n",
       "      <td>0.818632</td>\n",
       "      <td>(C=1)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>(percentile=95)</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.815047</td>\n",
       "      <td>0.778819</td>\n",
       "      <td>0.824405</td>\n",
       "      <td>(C=10)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>(random_state=42)</td>\n",
       "      <td>SelectFromModel</td>\n",
       "      <td>0.802956</td>\n",
       "      <td>0.763824</td>\n",
       "      <td>0.812562</td>\n",
       "      <td>(C=1)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>(n_estimators=50, random_state=42))</td>\n",
       "      <td>RFE</td>\n",
       "      <td>0.805643</td>\n",
       "      <td>0.772259</td>\n",
       "      <td>0.811823</td>\n",
       "      <td>(C=0.1)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Model          Scaler                             Features  \\\n",
       "0  LogisticRegression  StandardScaler                                  all   \n",
       "1  LogisticRegression    RobustScaler                                  all   \n",
       "2  LogisticRegression    RobustScaler                                  all   \n",
       "3  LogisticRegression    RobustScaler                                  all   \n",
       "4  LogisticRegression    RobustScaler                                  all   \n",
       "5  LogisticRegression    RobustScaler                     (percentile=100)   \n",
       "6  LogisticRegression    RobustScaler                     (percentile=100)   \n",
       "7  LogisticRegression    RobustScaler                      (percentile=95)   \n",
       "8  LogisticRegression    RobustScaler                    (random_state=42)   \n",
       "9  LogisticRegression    RobustScaler  (n_estimators=50, random_state=42))   \n",
       "\n",
       "                  Feature Selection  Accurecy on Test    Recall  Precession  \\\n",
       "0                            manual          0.808330  0.780694    0.811100   \n",
       "1                            manual          0.810121  0.774133    0.818632   \n",
       "2                            manual          0.811464  0.777882    0.818540   \n",
       "3                            manual          0.809225  0.773196    0.817641   \n",
       "4                            manual          0.778325  0.713215    0.801053   \n",
       "5  SelectPercentile(percentile=100)          0.810121  0.774133    0.818632   \n",
       "6                  SelectPercentile          0.810121  0.774133    0.818632   \n",
       "7                  SelectPercentile          0.815047  0.778819    0.824405   \n",
       "8                   SelectFromModel          0.802956  0.763824    0.812562   \n",
       "9                               RFE          0.805643  0.772259    0.811823   \n",
       "\n",
       "          Parameters Polynomial Features  Roc AUC  \n",
       "0           {'c': 1}                 NaN      NaN  \n",
       "1           {'c': 1}                 NaN      NaN  \n",
       "2            (C=100)                 NaN      NaN  \n",
       "3           (C=1000)                 NaN      NaN  \n",
       "4          (C=0.001)                 NaN      NaN  \n",
       "5  (percentile=100))                 NaN      NaN  \n",
       "6              (C=1)                 NaN      NaN  \n",
       "7             (C=10)                 NaN      NaN  \n",
       "8              (C=1)                 NaN      NaN  \n",
       "9            (C=0.1)                 NaN      NaN  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_management.results.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Scaler</th>\n",
       "      <th>Features</th>\n",
       "      <th>Feature Selection</th>\n",
       "      <th>Accurecy on Test</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precession</th>\n",
       "      <th>Parameters</th>\n",
       "      <th>Polynomial Features</th>\n",
       "      <th>Roc AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>StandardScaler</td>\n",
       "      <td>percentile=60</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.785043</td>\n",
       "      <td>0.816307</td>\n",
       "      <td>0.754113</td>\n",
       "      <td>max_depth=7</td>\n",
       "      <td>-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>StandardScaler</td>\n",
       "      <td>percentile=50</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.799821</td>\n",
       "      <td>0.790066</td>\n",
       "      <td>0.790807</td>\n",
       "      <td>max_depth=7</td>\n",
       "      <td>degree=3</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>StandardScaler</td>\n",
       "      <td>percentile=80</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.813704</td>\n",
       "      <td>0.814433</td>\n",
       "      <td>0.799448</td>\n",
       "      <td>max_depth=7</td>\n",
       "      <td>degree=4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>-</td>\n",
       "      <td>percentile=50</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.808330</td>\n",
       "      <td>0.799438</td>\n",
       "      <td>0.799438</td>\n",
       "      <td>max_depth=7</td>\n",
       "      <td>degree=3</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>-</td>\n",
       "      <td>percentile=80</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.807882</td>\n",
       "      <td>0.835052</td>\n",
       "      <td>0.778846</td>\n",
       "      <td>max_depth=7</td>\n",
       "      <td>degree=4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>percentile=60</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.806538</td>\n",
       "      <td>0.816307</td>\n",
       "      <td>0.786811</td>\n",
       "      <td>max_depth=10</td>\n",
       "      <td>-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>percentile=80</td>\n",
       "      <td>SelectPercentile</td>\n",
       "      <td>0.810121</td>\n",
       "      <td>0.824742</td>\n",
       "      <td>0.787825</td>\n",
       "      <td>max_depth=15</td>\n",
       "      <td>-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>RobustScaler</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>0.811912</td>\n",
       "      <td>0.819119</td>\n",
       "      <td>0.793824</td>\n",
       "      <td>max_depth=15</td>\n",
       "      <td>-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>MinMaxScaler</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>0.807882</td>\n",
       "      <td>0.812559</td>\n",
       "      <td>0.791058</td>\n",
       "      <td>max_depth=10</td>\n",
       "      <td>-</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>XGBClassifier</td>\n",
       "      <td>StandardScaler</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>0.846843</td>\n",
       "      <td>0.866917</td>\n",
       "      <td>0.822222</td>\n",
       "      <td>base_score=0.5,booster='gbtree',colsample_byle...</td>\n",
       "      <td>-</td>\n",
       "      <td>0.918722</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Model          Scaler       Features Feature Selection  \\\n",
       "42  DecisionTreeClassifier  StandardScaler  percentile=60  SelectPercentile   \n",
       "43  DecisionTreeClassifier  StandardScaler  percentile=50  SelectPercentile   \n",
       "44  DecisionTreeClassifier  StandardScaler  percentile=80  SelectPercentile   \n",
       "45  DecisionTreeClassifier               -  percentile=50  SelectPercentile   \n",
       "46  DecisionTreeClassifier               -  percentile=80  SelectPercentile   \n",
       "47  RandomForestClassifier    RobustScaler  percentile=60  SelectPercentile   \n",
       "48  RandomForestClassifier    RobustScaler  percentile=80  SelectPercentile   \n",
       "49  RandomForestClassifier    RobustScaler              -                 -   \n",
       "50  RandomForestClassifier    MinMaxScaler              -                 -   \n",
       "51           XGBClassifier  StandardScaler              -                 -   \n",
       "\n",
       "    Accurecy on Test    Recall  Precession  \\\n",
       "42          0.785043  0.816307    0.754113   \n",
       "43          0.799821  0.790066    0.790807   \n",
       "44          0.813704  0.814433    0.799448   \n",
       "45          0.808330  0.799438    0.799438   \n",
       "46          0.807882  0.835052    0.778846   \n",
       "47          0.806538  0.816307    0.786811   \n",
       "48          0.810121  0.824742    0.787825   \n",
       "49          0.811912  0.819119    0.793824   \n",
       "50          0.807882  0.812559    0.791058   \n",
       "51          0.846843  0.866917    0.822222   \n",
       "\n",
       "                                           Parameters Polynomial Features  \\\n",
       "42                                        max_depth=7                   -   \n",
       "43                                        max_depth=7            degree=3   \n",
       "44                                        max_depth=7            degree=4   \n",
       "45                                        max_depth=7            degree=3   \n",
       "46                                        max_depth=7            degree=4   \n",
       "47                                       max_depth=10                   -   \n",
       "48                                       max_depth=15                   -   \n",
       "49                                       max_depth=15                   -   \n",
       "50                                       max_depth=10                   -   \n",
       "51  base_score=0.5,booster='gbtree',colsample_byle...                   -   \n",
       "\n",
       "     Roc AUC  \n",
       "42       NaN  \n",
       "43       NaN  \n",
       "44       NaN  \n",
       "45       NaN  \n",
       "46       NaN  \n",
       "47       NaN  \n",
       "48       NaN  \n",
       "49       NaN  \n",
       "50       NaN  \n",
       "51  0.918722  "
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = {\n",
    "    'Model':names_parser.get_element_name(model.get_params().get('classifier')),\n",
    "    'Scaler':names_parser.get_element_name(numeric_preprocess.get_params().get('scaler','-')),\n",
    "    'Features':names_parser.get_element_params(model.get_params().get('feature_selection','-')),\n",
    "    'Feature Selection':names_parser.get_element_name(model.get_params().get('feature_selection','')),\n",
    "    'Accurecy on Test':acc,\n",
    "    'Recall':rec,\n",
    "    'Precession':pre,\n",
    "    'Parameters':names_parser.get_element_params(model.get_params().get('classifier')),\n",
    "    'Polynomial Features':names_parser.get_element_params(numeric_preprocess.get_params().get('polynomial_features','-')),\n",
    "    'Roc AUC':roc_auc\n",
    "}\n",
    "results_management.add_to_results(result=result).tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_management.save_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>51</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Model</th>\n",
       "      <td>XGBClassifier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Scaler</th>\n",
       "      <td>StandardScaler</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Features</th>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Feature Selection</th>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Accurecy on Test</th>\n",
       "      <td>0.846843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.866917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precession</th>\n",
       "      <td>0.822222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Parameters</th>\n",
       "      <td>base_score=0.5,booster='gbtree',colsample_byle...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Polynomial Features</th>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Roc AUC</th>\n",
       "      <td>0.918722</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    51\n",
       "Model                                                    XGBClassifier\n",
       "Scaler                                                  StandardScaler\n",
       "Features                                                             -\n",
       "Feature Selection                                                    -\n",
       "Accurecy on Test                                              0.846843\n",
       "Recall                                                        0.866917\n",
       "Precession                                                    0.822222\n",
       "Parameters           base_score=0.5,booster='gbtree',colsample_byle...\n",
       "Polynomial Features                                                  -\n",
       "Roc AUC                                                       0.918722"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(results_management.results.loc[51])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"learning_curve\"></a>\n",
    "<h4><left>- plot learning curve.</left></h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x190d6093608>"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABCYAAAF3CAYAAAB5QUrKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeZxcdZ3v/9e3q7d0pzudhZCQBBKGIJCVEIIKhp0J6IAgIAw8JIOC4qiXqzLi/K6izMx1LuOol7ksA+5OUEIURIygMCAiiwkQMCyRAIGEEMi+9L58f3+cqu7qJUkl6eSku1/Px+M8ajmnTn3OOVXJt979Pd8TYoxIkiRJkiSloSjtAiRJkiRJ0sBlMCFJkiRJklJjMCFJkiRJklJjMCFJkiRJklJjMCFJkiRJklJjMCFJkiRJklKz02AihPD9EMK7IYSl25kfQgg3hhCWhxCeDyHM6P0yJUlSf2L7QpIk5RTSY+KHwJwdzD8TmJidrgRu2fOyJElSP/dDbF9IkiQKCCZijI8CG3awyDnAj2PiSaAmhDC6twqUJEn9j+0LSZKU0xtjTIwBVuY9XpV9TpIkaXfZvpAkaYAo7oV1hB6eiz0uGMKVJN0xqaysPOaII47ohbeXJEm94emnn14XYzwg7TqybF9IktTHFdq26I1gYhUwLu/xWGB1TwvGGG8DbgOYOXNmXLx4cS+8vSRJ6g0hhDfSriGP7QtJkvq4QtsWvXEqx73Ax7KjZ78X2BxjfLsX1itJkgYu2xeSJA0QO+0xEUL4KXASMCKEsAq4DigBiDHeCiwEzgKWA3XA3+2tYiVJUv9g+0KSJOXsNJiIMV68k/kR+Pteq0iSJPV7ti8kSVJOb4wxIUmSJEnSLmtubmbVqlU0NDSkXYr2QHl5OWPHjqWkpGS3Xm8wIUmSJElKxapVq6iqqmL8+PGE0NMFmbS/izGyfv16Vq1axYQJE3ZrHb0x+KUkSZIkSbusoaGB4cOHG0r0YSEEhg8fvke9XgwmJEmSJEmpMZTo+/b0GBpMSJIkSZIGpE2bNnHzzTfv1mvPOussNm3atMNlvvrVr/Lggw/u1voHEoMJSZIkSdKAtKNgorW1dYevXbhwITU1NTtc5vrrr+e0007b7fr2lp1t275mMCFJkiRJGpCuvfZaXn31VaZPn84111zDI488wsknn8zf/u3fMmXKFAA+/OEPc8wxxzBp0iRuu+229teOHz+edevWsWLFCo488kiuuOIKJk2axBlnnEF9fT0Ac+fOZcGCBe3LX3fddcyYMYMpU6bw8ssvA7B27VpOP/10ZsyYwSc/+UkOOeQQ1q1b16nO1tZW5s6dy+TJk5kyZQrf/va3AVi+fDmnnXYa06ZNY8aMGbz66qvEGLnmmmval73zzjsBety2//qv/2LWrFlMnz6dT37yk6kFFl6VQ5IkSZKUuquvhiVLened06fDd76z/fn/+q//ytKlS1mSfeNHHnmEP/3pTyxdurT9ChPf//73GTZsGPX19Rx77LF85CMfYfjw4Z3W88orr/DTn/6U22+/nQsvvJCf//znXHrppd3eb8SIETzzzDPcfPPNfPOb3+S73/0uX//61znllFP48pe/zP33398p/MhZsmQJb731FkuXLgVoP4Xkkksu4dprr+Xcc8+loaGBtrY2fvGLX7BkyRKee+451q1bx7HHHsvs2bMBOm3bSy+9xJ133skf//hHSkpK+PSnP828efP42Mc+tus7eg8ZTEiSJEmSlDVr1qxOl7288cYbufvuuwFYuXIlr7zySrdgYsKECUyfPh2AY445hhUrVvS47vPOO699mV/84hcAPPbYY+3rnzNnDkOHDu32ukMPPZTXXnuNz372s3zwgx/kjDPOYOvWrbz11luce+65AJSXl7ev7+KLLyaTyXDggQdy4oknsmjRIqqrqztt20MPPcTTTz/NscceC0B9fT0jR47c9R3WCwwmJEmSJEmp21HPhn2psrKy/f4jjzzCgw8+yBNPPEFFRQUnnXRSj5fFLCsra7+fyWTaT+XY3nKZTIaWlhYAYow7rWno0KE899xzPPDAA9x0003Mnz+f72xnh+1offnbFmPksssu4xvf+MZO339vc4wJSZIkSdKAVFVVxdatW7c7f/PmzQwdOpSKigpefvllnnzyyV6v4YQTTmD+/PkA/Pa3v2Xjxo3dllm3bh1tbW185CMf4Z/+6Z945plnqK6uZuzYsdxzzz0ANDY2UldXx+zZs7nzzjtpbW1l7dq1PProo8yaNavbOk899VQWLFjAu+++C8CGDRt44403en37CmEwIUmSJEkakIYPH87xxx/P5MmTueaaa7rNnzNnDi0tLUydOpWvfOUrvPe97+31Gq677jp++9vfMmPGDH7zm98wevRoqqqqOi3z1ltvcdJJJzF9+nTmzp3b3svhJz/5CTfeeCNTp07l/e9/P2vWrOHcc89l6tSpTJs2jVNOOYUbbriBUaNGdXvfo446in/+53/mjDPOYOrUqZx++um8/fbbvb59hQiFdBvZG2bOnBkXL16cyntLkqTuQghPxxhnpl3HnjjiiJnx5ZdtX0hSX/HSSy9x5JFHpl1GqhobG8lkMhQXF/PEE09w1VVXtQ/G2Zf0dCwLbVs4xoQkSeo3soOUS5LUZ7z55ptceOGFtLW1UVpayu233552SfucwYQkSeo3Nm2CGCGEtCuRJKkwEydO5Nlnn027jFQ5xoQkSeo3Ghth2bK0q5AkSbvCYEKSJPUr996bdgWSJGlXGExIkqR+Y9AggwlJkvoagwlJktRv1NTA44/D2rVpVyJJkgplMCFJkvqNmppk8MuFC9OuRJLUXw0ePBiA1atXc/755/e4zEknncTixTu+fPV3vvMd6urq2h+fddZZbBqgl5cymJAkSf1GRQWMGePpHJKkve+ggw5iwYIFu/36rsHEwoULqamp6Y3SelVra+tefw+DCUmS1K/8zd/AAw9AQ0PalUiS9ndf+tKXuPnmm9sff+1rX+Pf//3f2bZtG6eeeiozZsxgypQp/PKXv+z22hUrVjB58mQA6uvrueiii5g6dSof/ehHqa+vb1/uqquuYubMmUyaNInrrrsOgBtvvJHVq1dz8sknc/LJJwMwfvx41q1bB8C3vvUtJk+ezOTJk/nOd77T/n5HHnkkV1xxBZMmTeKMM87o9D45d911F5MnT2batGnMnj0bSMKFL37xi0yZMoWpU6fyH//xHwA89NBDHH300UyZMoXLL7+cxsbG9lquv/56TjjhBO666y5effVV5syZwzHHHMMHPvABXn755T3b8V0U9+raJEmSUnb22XDrrfDww3DmmWlXI0kq2NVXw5IlvbvO6dMh+8O+JxdddBFXX301n/70pwGYP38+999/P+Xl5dx9991UV1ezbt063vve93L22WcTQuhxPbfccgsVFRU8//zzPP/888yYMaN93r/8y78wbNgwWltbOfXUU3n++ef53Oc+x7e+9S0efvhhRowY0WldTz/9ND/4wQ946qmniDFy3HHHceKJJzJ06FBeeeUVfvrTn3L77bdz4YUX8vOf/5xLL7200+uvv/56HnjgAcaMGdN+ashtt93G66+/zrPPPktxcTEbNmygoaGBuXPn8tBDD3H44YfzsY99jFtuuYWrr74agPLych577DEATj31VG699VYmTpzIU089xac//Wn++7//excPxvbZY0KSJPUrJ58MlZWeziFJ2rmjjz6ad999l9WrV/Pcc88xdOhQDj74YGKM/OM//iNTp07ltNNO46233uKdd97Z7noeffTR9oBg6tSpTJ06tX3e/PnzmTFjBkcffTQvvPACL7744g5reuyxxzj33HOprKxk8ODBnHfeefzhD38AYMKECUyfPh2AY445hhUrVnR7/fHHH8/cuXO5/fbb20/DePDBB/nUpz5FcXHSN2HYsGEsW7aMCRMmcPjhhwNw2WWX8eijj7av56Mf/SgA27Zt4/HHH+eCCy5g+vTpfPKTn+Ttt9/e4TbsKntMSJKkfqW8HP76r5Ng4uabYTt/3JIk7W920LNhbzr//PNZsGABa9as4aKLLgJg3rx5rF27lqeffpqSkhLGjx9Pw07OEeypN8Xrr7/ON7/5TRYtWsTQoUOZO3fuTtcTY9zuvLKysvb7mUymx1M5br31Vp566il+/etfM336dJYsWUKMsVt9O3ofgMrKSgDa2tqoqalhSW/3ZsljjwlJktTvnH02rF4NzzyTdiWSpP3dRRddxM9+9jMWLFjQfpWNzZs3M3LkSEpKSnj44Yd54403driO2bNnM2/ePACWLl3K888/D8CWLVuorKxkyJAhvPPOO/zmN79pf01VVRVbt27tcV333HMPdXV11NbWcvfdd/OBD3yg4O159dVXOe6447j++usZMWIEK1eu5IwzzuDWW2+lpaUFgA0bNnDEEUewYsUKli9fDsBPfvITTjzxxG7rq66uZsKECdx1111AEmg899xzBddTCIMJSZLU75x1VtJT4le/SrsSSdL+btKkSWzdupUxY8YwevRoAC655BIWL17MzJkzmTdvHkccccQO13HVVVexbds2pk6dyg033MCsWbMAmDZtGkcffTSTJk3i8ssv5/jjj29/zZVXXsmZZ57ZPvhlzowZM5g7dy6zZs3iuOOO4xOf+ARHH310wdtzzTXXMGXKFCZPnszs2bOZNm0an/jEJzj44IOZOnUq06ZN44477qC8vJwf/OAHXHDBBUyZMoWioiI+9alP9bjOefPm8b3vfY9p06YxadKkHgcD3RNhZ9039paZM2fGnV3XVZIk7TshhKdjjDPTrmNP5LcvTjgB6ursNSFJ+7OXXnqJI488Mu0y1At6OpaFti3sMSFJkvqls8+GZ5+FnfS+lSRJKTOYkCRJ/dIFF0AmAzfckHYlkiRpRwwmJElSvzRhAlxxBdx2G7zyStrVSJKk7TGYkCRJ/dZ110FZGfzjP6ZdiSRpe9Ia91C9Z0+PocGEJEnqt0aNgi98ARYsgKeeSrsaSVJX5eXlrF+/3nCiD4sxsn79esrLy3d7HcW9WI8kSdJ+54tfhFtvhS99CR5+OLmMqCRp/zB27FhWrVrF2rVr0y5Fe6C8vJyxY8fu9usNJiRJUr9WVQVf/Sp85jOwcCF88INpVyRJyikpKWHChAlpl6GUeSqHJEnq9668Eg47DK69Flpb065GkiTlM5iQJEn9XkkJ/O//DUuXwo9/nHY1kiQpn8GEJEkaEM4/H2bNSk7rWLwYHGdNkqT9g8GEJEkaEEKAb38b1q2DY4+FI46A66+H5cvTrkySpIHNwS8lSdKA8f73w+rVyeVD77gDvvY1uO46mDQJhg+HQYOgoiK5LSqC9euTICM3jR0L996bjFchSZJ6h8GEJEkaUIYOhSuuSKaVK+HOO5PLiNbWwsaNSXBRV5cMkjl8OBxwALznPTBsGMybBx/4ADz4YBJmSJKkPWcwIUmSBqxx4+CLX0ymQlx5JZx+Opx4Ivz2tzBjxt6tT5KkgcAxJiRJkgo0aRI8+ihUVsLJJ8Pjj6ddkSRJfZ/BhCRJ0i447DB47DE48MCk98RDD6VdkSRJfZvBhCRJ0i4aNy7pOXHooTBnDvzbv0FbW9pVSZLUNznGhCRJ0m4YNQr+8IdkEM1/+IdkAM0f/SgZLDMNtbXw1lvw9tuwZk1yu2ED1NQkNY0YkdyOG5f09pAkaX9hMCFJkrSbampg/ny49Vb4n/8Tpk9PLkN64on7robFi+Fb30rqaG3d+fJFRfCJT8D11xtQSJL2DwYTkiRJeyAEuOoqeN/74KMfhVNOgcsvTy4reuyxcPjhkMl0f13u1I+i3TixtrUVfvWrJJD4wx+guho++1k45hgYPTrpzTF6dBKcbNkCa9fCunXJ7YMPwi23JAHKtdfC5z8PgwZ1rDtGeOcdWLEiuXzqpk3JtHEjbN4M27bB1q3JtG0bVFQkg4LmpiOOgPLy3dqVkqQBKsQYU3njmTNnxsWLF6fy3pIkqbsQwtMxxplp17En0m5fbN0KV18Nd96ZnFoBMHhwclnRIUOScGD9+uR248YkBCgrS4KBQYOSH/knnAAf/3hyG0Ln9a9Zk5wucvvt8OqrcMghyftdfnkSThTqL3+BL30J7rkHxo5NgpW334alS+HPf05q7ElpKVRVJdtUVZVMmzbBK69AS0uyTFERDB2a3GYyyW3ufnFxx1RSAuPHw7RpSU+TadOSx123WZLUdxXatjCYkCRJgMFEb2pthZdfTk6zyE0NDTB8eDLWw/DhyZTJQF0d1Ncn08aN8MADSU+EiROTwOHSS2HJEvjud+G++5J1f+ADSQ+Jc89NfuTvrt//Pukx8cwzScgweXLHdNhhMGxY0usiN22vJ0RTUxJOvPBCMq1fn/QIaWtL6m1tTe63tCRTczM0NsLy5UlIkmuOVlUlAU4uzAgh2UcVFR1BSG4aNiyZcvuyujrpHbJ+fTK2xoYNyXtMmwbHHQdHHrl7vVMkSbvPYEKSJO0Sg4n9Q20t3HUXfO97yWVJcw48EC67LOlNcfjhvfd+bW3JKR4jR6bTW6G2Nump8dxzSW+N2tqkphg7go26uo7TR7ZuTQKIjRuTMGd7ysqS0CbXc6W6Ojm1ZsqUZDtz625rS5bLDQ6am4YNS0KSIUOSHiJphRoxpteLJMZkP7/+ejI1NychUH4gVFVlLxdJ21do28IxJiRJkvYjlZUwd24y/eUvsGBB8tf+D30oOf2htxUVpTsIZmVl0qPhuON2/bX19UnPiPXrk7Ciurrjh/OgQUno8Mor8NRT8OSTye3tt3f0xMidYtLYmAQe2xNCsu6RI5PTZw45JDnt5JBDkt4cW7Z0npqaOnqB5G7Ly5N1VFUlt4MHJ4HQG2/Am28mt6tWJSFMc3PH1Nqa1Fha2jFVVMCECUlANXFiMo0dm+yLd95JpjVrklBm1CgYMwYOOii5HTIk2dbNm5NaN29Owof16ztOM1q/PrnCy+uv73i/QLIthx3Weaqs7LwNzc1Jj6H83kH19clzjY0dt83NyfqGDk2mYcOS2/yeMoMHJ8d227bO29DQkLzv4MHJVFmZLBdC96nrKUa5Y7u3ApY0wyWpr7DHhCRJAuwxoYGtqaljgNDcj/PNmztPuUFB33gjud+TQYOS3hrQ+cdoQ0PPPTxKS5NLuB5ySHJbWZkEUCUlybzi4uT0l6amZGpuTsKC115Lgqt163quo6wsWdeGDYXvg6qqjp4Qo0cn4ceECUkIM2FCEq7kAoxciPHmm8kpOcuXJ0HGzq4MU1SUBCuDBiXrKyvrmEpKkm3bsCEJSxobC699T1VUJNt56KHJtg4a1HFKUG5qauo4xSiE5H5LS1Jnbmpq6jhlKXcaU4xJIDRyZMc0YkTy+lyvndyyra0dr21pSd6npqbj1KVhw5LjBJ17FpWWJgFjbuDbYcOS9Tc3d96GzZuTwKquLplqazveJ7dtRUXJduSWy922tnY+zSqE5HNdW5sERbnlysq6j0WTC4tyt5WVHcc9F7jl7uc/X1KSbEPus5/bvz3Jry0XOuXWkfs+Qfd9nPs5nPu+5h/f/PW1tnYEbbljnFu+637ZXhiWySRTCN3Du1zQmvv3Z/36ZAyf3LGqr09um5o615n/70x+INrW1lFr7j3Kyjp6gtXUJLcNDZ3HP9qwoWP8o/zjkb8Nue0tK0u+xzua/tf/sseEJEkaYNavX88Pf/jDTs9NmjSJY489lubmZubNm9ftNdOnT2f69OnU1dUxf/78bvNnzpzJ5MmT2bx5M3fffXe3+e973/t4z3vew7p167jvvvu6zZ89ezaHHnooa9as4f777+82/9RTT2XcuHGsXLmShx56qNv8OXPmMGrUKF577TUeffTRbvM/9KEPMWLECJYtW8YTTzzRbf65557LkCFDWLp0KT2FNhdeeCEVFRUsWbKEJUuWdJt/ySWXUFJSwqJFi3jhhRe6zZ87dy4Ajz/+OH/5y186zSspKeGSSy4B4Pe//z2vv/56p/kVFRVceOGFADz44IOsWrWq0/zq6mrOO+88AO6//37WrFnTaf7w4cP5m7/5GwB+9atfsb7LqJ2jRo1izpw5APziF79gy5YtneaPHTuW0047DYB77plPXV1dp/mHHz6BE7PXfp03bx7Nzc3t85qaMlRXT+Koo2YwZAjcd98dlJc3k8l0/NGv62evpSXQ0FDSPs2adRinnTaZhoZd/+ydemry2Rs58j0sWrSJX/3qGQYPbmTIkHqqq+sZNKiZE0+czZgxh/L88+/yy18uYuPGCurrSxk0qIny8mZmz57G4YcfSG3tWyxZ8hDFxW2d3mNHn70Q4O/+rvNnr6UlsGHDYJqbM2QybZx55mkMH17N8uUv8eKLz1Ba2kom09b+Q2pnn73zzruEbdtKePzxP/PnP69o329NTRnKy5s5//zTqa6GV15ZzMqVy2lqKqaxsZjGxhJaWso4/vjTiBH+/OelrFnzDkSIbZE2MpSWljFr1vtpa4MnnniZ5ctbWLu2iueeq+LBBwfT0pLhgAMyDBsGIWyguHgbVVWtxJgUX1xcxqhRoykuhrVrV9HaWk9JSbJ9mUwbVVUVTJx4KJkM/OXlpWzYGNiytYL168tZsaKcuroKSkvLsiFAHdBGUVGkqCi5LS8vprp6MDHCW2/VUltbSkND4V2miovbKC8vYtu2gl/STQhQURHJZBooLW2hqCgSI8QYKC0tJ5MpoaSklZaWTZSVtVBW1kJpaQvNzUXU1o5g86ZyGjduoWH9JrY0V7GhcQht9HCJov1cCU2MZRWlNNFEafvUTAkZWvOeaaKYFhopo44K6hlEPYNoohQorMtMhhZGsI4DWMuoorWMKNtAKI4UFweqSwJDSgIlZcXUVFcTYhtbN2+kraWZNopoKBpEQyintayK8mFjKM5E6raspKx1CxWhngpqKd7STNnGSHnMEGvraN22haFFLYwvbaW0tIWysmaqhhZTOewAasNgXl9by9amSrY1VEKMFLe1UFmSoaq8lExrE+s3bKG5NUNzazHNrcXUtZSynsFsbalic1M5tVQWvJ8NJiRJkrTLyrds4cBly5I/6RUVMWrFCsK6dRACbZkMrSUlyZ/LhgyBsjKG/+lPFC9fTvnWrR1TUxPcfDNs3cr73nmHUFtLa3ExzYMG0VRRQVt1NWR/kE9dtYr31NZS1NpKaGsjxEhpSQncdBO0tfHed9/tFBwAlJaXww9/CCNGcOS777KlpITGykpaS0tpKS1l6Pr1ST/+GBn9zDOEdesoy9ZW0tDAkIqK5PUtLRy/fDmNZWVsHj06mUaN4qAjxzN1avJelZXJ+RtFzS3ETIbYw6AUxcWRwYObGFJWS2lJHaNLayhqauj4M+duGDoUZsxoYfXqN3ucX1YG48a1cdhha7vNmzHjSMaNg5Ur2zqFEqG1lZL6ejJvvglr1lD+8suMXbKE8m3b2vdP+datVN9xBzQ0cMj69YzasIHixkaKGxtpKy6msbKSwXePo3jECAaHwOht22jLZGgrLk5uMxlKXnoJMhlGv/02mXfeIcRIc1kZTRUVNFdUUFFZSXVNDdPeeJ4D3nmF4sZGSrLvUdzQwNQ7fwW1tYxesYL69espammhqLWVotZWMq2tjFg4GGpr+esNGwj19RQ3NhJipKm8nJbKSip+NwpqapjT0EB9UxNtmQzxgCLaRmUoLill3LixALz99ts0ZLu7BCLESBkljG4bCQ1tvFO7hubGxvbPZUlDA4Oamxn8x5aki0JdHW2ZDM3l5e1TGD2EmrFjoaqK19eto76khJayMmIIhLY2aoYM4eAxY6ClheXPPkumro7i+kYytY3Q0goHjGLwXx1O2wEH8syq1dSXDqaxtoim2iKaa6GMwVSUD6Z0UGDtxtUUl0NJeStlmSYqW7ZxcHUpoyuKCVs2sGblG7QVJZ/Z1qIi2ooyHDCshmFVFbTU1bFm5UqKWlrItLSQaW4m09xMZXExZTHS2tJGbX09NARi9s/3xY2NlLe0kMkNGJMVQ6C5tIymsnIy1UMpKi2jubmF+roGIhAJtBSV0lxUQumQ4VA2mLqWNuq3baE4tpKhhUxspbi1mSFlGUpamqBuK6F+G0VtbbSFDG1FRbSGDMXlg4il5dSFUra0ZWjMDKK+eBBtIUMmtjK8pooS2miq20ptYzPNxWU0FZfRWFxOc6aU8VVlVK1/k7I1y6na8g5F7P53tI3svskLJ0KAWFRMawi0ZusOMVLRuC3/hbCDcXR6VcPeW3WhZzF5KockSQI8laPfS/70mvS9h46+uLkf0Pl9pZuaOkaFzL/O5+rVyaU8fv97eOml3a+lsjIZYbKmpmPQhVx/74aG5Mfcpk0dgwiE0LmWXH/o/IECejqJv6UlOSdg3bpkfYW2e0tLk0ClpKRj2zOZjkt+5JSVJduQP1hCTu5SIrlta2zsOEehocuvgEymez/33P2KiqQPee64NDUlj3OXMMlNFRUdA0Zs3Ji815Yt3QdyaG3tfH3a3EAM+SOL7mhU0dx250YJranpqDk35W9rbsqd55A/5feBz30Od/beOcXFnc8PyJ1Dk+u7n+uDnr8vBw9O3jM3MEVuqq3t6Nef6/fe02clv19//mVj8rehqCh5r/xjU1WVbG9u/+bf5u/33EitXT/bgwd3HqAkk0k+0++8A+++u/1zG3akoqKjvtLS7tufP7BJ/v7MnX+Tu82NJJt/XknXz35FRfL5y9/ubdu67+O2tmQ/5X9mm5o6X2c4k0nqqajo+PxWVCTzup6jkTvPJHeuybZtybyu68uNsJs7V6KuLkn8xo/vmA45JNnm/HOqcrXl76fi4qT23LpyA6p03dYYu9cL3Ufhra7ufEmj3PJdP3etrZ3fs7Y21+Wl4/ue21f5+27QoO6Xdoox2Ybc+Tm5fdd1sJvcoEe545+rs8uAMuHCC3vvVI4Qwhzg/wIZ4Lsxxn/tMv9g4EdATXaZa2OMCwtZtyRJGnhsW+wFjY0dP1RyIyCuXg3LliUhwrJl7FG/7pyqKjjhhOQSISedlIzAmPtBkjtZPnc90PwfGBUVSUN7xIikMbyvtbZ2/BbKL2EAAB+4SURBVEjO/xGS+9GQ+yEwYsSOLzWxbl2yL19+OZm2bOn4UZz7sZYbCCL/x2dZWceojkOHJj/oW1o6/3Dqen/jxiRMKi7u+HGY+4G4aVMy2EXux3V9ffeRI0eO7DyYQ3l58iMmfwTKurrkuOV+ROb/oMydhJ6bhg1L9lHuB/7e0NLSERxs2pTsi4qK7uFHbiCPga6trSPEy/+MlJYmx6jrj99MpiN0k7ansjL5vu9DOw0mQggZ4CbgdGAVsCiEcG+M8cW8xf4XMD/GeEsI4ShgITB+L9QrSZL6uP2ubbF0adJdv7o6GT1u5MjkdswYOPjgvfcDrLY2+cFeU5P8WChUjMmPteeeg6ef7phefrlTt+l2Bx8MRxwBl1+e3I4fn/w4zQ8TYuz+l7DcX+Hy/5JaUwNTp3b/C1tfkMkkocOIEXu2ntw6jj++d+rqLf3l0g/FxR2jPGrniorcX+oXCvlfZRawPMb4GkAI4WfAOUB+4yEC1dn7Q4DVvVmkJEnqV/aftsW8eXDllR1DlndVUwMzZsDRRye3Eyd2dDnPqapKhvIv5Md6QwMsXAh33AH33ddx2YH8v3SXl3fvarx1a/du8TmjRsExx8B55yUhxIEHdp7S6J2gfa8/hBKSBqxCgokxwMq8x6uArlea/hrw2xDCZ4FK4LReqU6SJPVH6bctmprg859PBk6cPRvuvDP5i+PatR2nQqxYAUuWwDPPwP/7fzu+dmFpaXJKw1FHJdNBB3U/d/4Pf4AFC5Iu1yNHwic/CX/1V50Dh9w1CXM9FOrqktvBg5P15sKL4cNh0qQkLDnooF7dNZIk7WuFBBM9xa9dR4O5GPhhjPHfQwjvA34SQpgcY+zUnzCEcCVwJcDBBx+8O/VKkqS+r9faFrAb7YtVq+CCC+DJJ+ELX4BvfKPjfOsxY5Kpq+bm5FSJFSu6z9uwIRnD4cUXk1Mq7rqr54HzBg9OejVccgmcckrfPB1CkqS9oJD/EVcB4/Iej6V7d8qPA3MAYoxPhBDKgRHAu/kLxRhvA26DZNTs3axZkiT1bb3WtsjOL7x9cffdSU+F+vokQDj//MIqLimBKVOSaWfq65OrN+SP4dDWlvRs8LQKSZK66X6B5e4WARNDCBNCCKXARcC9XZZ5EzgVIIRwJFAOdL9YsiRJUhpti9Wr4SMfSXosHHQQLFpUeCixqwYNgrFjYdy45BJzEyYkp2wYSkiS1KOdBhMxxhbgM8ADwEskI2S/EEK4PoRwdnaxLwBXhBCeA34KzI2x0AtFS5KkgWSfti1ihNtvT8Zn+PWvk9M2Fi1Krk4hSZL2CwWd3Ji9bvjCLs99Ne/+i8B+ds0kSZK0v9onbYvNm+HDH4ZHHoETT0wCiokT92iVkiSp9znqkiRJ6p++8Q34/e/hP/8TPvGJ7pf5lCRJ+wWDCUmS1P+8/TbceCP87d/ClVemXY0kSdoB/3QgSZL6n3/5l+QSn1/7WtqVSJKknTCYkCRJ/cuKFXDbbfDxj8Nhh6VdjSRJ2gmDCUmS1L98/evJeBJf+UralUiSpAIYTEiSpP6joQF+/GP4zGdgzJi0q5EkSQUwmJAkSf3H6tVQUQHXXpt2JZIkqUAGE5Ikqf/YuBE+/3kYMSLtSiRJUoEMJiRJUv+RySTBhCRJ6jMMJiRJUv8xahQMGZJ2FZIkaRcYTEiSpP5j5Mi0K5AkSbvIYEKSJPUfRTZtJEnqa/zfW5IkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpaagYCKEMCeEsCyEsDyEcO12lrkwhPBiCOGFEMIdvVumJEnqT2xbSJKknOKdLRBCyAA3AacDq4BFIYR7Y4wv5i0zEfgycHyMcWMIYeTeKliSJPVtti0kSVK+QnpMzAKWxxhfizE2AT8DzumyzBXATTHGjQAxxnd7t0xJktSP2LaQJEntCgkmxgAr8x6vyj6X73Dg8BDCH0MIT4YQ5vS0ohDClSGExSGExWvXrt29iiVJUl/Xa20LsH0hSVJfV0gwEXp4LnZ5XAxMBE4CLga+G0Ko6faiGG+LMc6MMc484IADdrVWSZLUP/Ra2wJsX0iS1NcVEkysAsblPR4LrO5hmV/GGJtjjK8Dy0gaE5IkSV3ZtpAkSe0KCSYWARNDCBNCCKXARcC9XZa5BzgZIIQwgqT75Wu9WagkSeo3bFtIkqR2Ow0mYowtwGeAB4CXgPkxxhdCCNeHEM7OLvYAsD6E8CLwMHBNjHH93ipakiT1XbYtJElSvhBj11M6942ZM2fGxYsXp/LekiSpuxDC0zHGmWnXsSdsX0iStP8otG1RyKkckiRJkiRJe4XBhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJkiRJSo3BhCRJ2iMhhBNCCH+XvX9ACGFC2jVJkqS+w2BCkiTtthDCdcCXgC9nnyoB/iu9iiRJUl9jMCFJkvbEucDZQC1AjHE1UJVqRZIkqU8xmJAkSXuiKcYYgQgQQqhMuR5JktTHGExIkqQ9MT+E8J9ATQjhCuBB4PaUa5IkSX1IcdoFSJKkvivG+M0QwunAFuA9wFdjjL9LuSxJktSHGExIkqTdEkLIAA/EGE8DDCMkSdJu8VQOSZK0W2KMrUBdCGFI2rVIkqS+yx4TkiRpTzQAfw4h/I7slTkAYoyfS68kSZLUlxhMSJKkPfHr7CRJkrRbDCYkSdJuizH+KIRQChyefWpZjLE5zZokSVLfYjAhSZJ2WwjhJOBHwAogAONCCJfFGB9Nsy5JktR3GExIkqQ98e/AGTHGZQAhhMOBnwLHpFqVJEnqM7wqhyRJ2hMluVACIMb4F6AkxXokSVIfY48JSZK0JxaHEL4H/CT7+BLg6RTrkSRJfYzBhCRJ2hNXAX8PfI5kjIlHgZtTrUiSJPUpBhOSJGlPFAP/N8b4LYAQQgYoS7ckSZLUlzjGhCRJ2hMPAYPyHg8CHkypFkmS1AcZTEiSpD1RHmPclnuQvV+RYj2SJKmPMZiQJEl7ojaEMCP3IIQwE6hPsR5JktTHOMaEJEnaE/8DuCuEsBqIwEHAR9MtSZIk9SUGE5IkaU9MAI4GDgbOBd5LElBIkiQVxFM5JEnSnvhKjHELUAOcDtwG3JJuSZIkqS8xmJAkSXuiNXv7QeDWGOMvgdIU65EkSX2MwYQkSdoTb4UQ/hO4EFgYQijD9oUkSdoFNhwkSdKeuBB4AJgTY9wEDAOuSbckSZLUlzj4pSRJ2m0xxjrgF3mP3wbeTq8iSZLU19hjQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpcZgQpIkSZIkpaagYCKEMCeEsCyEsDyEcO0Oljs/hBBDCDN7r0RJktTf2LaQJEk5Ow0mQggZ4CbgTOAo4OIQwlE9LFcFfA54qreLlCRJ/YdtC0mSlK+QHhOzgOUxxtdijE3Az4Bzeljun4AbgIZerE+SJPU/ti0kSVK7QoKJMcDKvMerss+1CyEcDYyLMd63oxWFEK4MISwOISxeu3btLhcrSZL6hV5rW2SXtX0hSVIfVkgwEXp4LrbPDKEI+DbwhZ2tKMZ4W4xxZoxx5gEHHFB4lZIkqT/ptbYF2L6QJKmvKySYWAWMy3s8Flid97gKmAw8EkJYAbwXuNdBqiRJ0nbYtpAkSe0KCSYWARNDCBNCCKXARcC9uZkxxs0xxhExxvExxvHAk8DZMcbFe6ViSZLU19m2kCRJ7XYaTMQYW4DPAA8ALwHzY4wvhBCuDyGcvbcLlCRJ/YttC0mSlK+4kIVijAuBhV2e++p2lj1pz8uSJEn9mW0LSZKUU8ipHJIkSZIkSXuFwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUqNwYQkSZIkSUpNQcFECGFOCGFZCGF5COHaHuZ/PoTwYgjh+RDCQyGEQ3q/VEmS1F/YtpAkSTk7DSZCCBngJuBM4Cjg4hDCUV0WexaYGWOcCiwAbujtQiVJUv9g20KSJOUrpMfELGB5jPG1GGMT8DPgnPwFYowPxxjrsg+fBMb2bpmSJKkfsW0hSZLaFRJMjAFW5j1elX1uez4O/GZPipIkSf2abQtJktSuuIBlQg/PxR4XDOFSYCZw4nbmXwlcCXDwwQcXWKIkSepneq1tkV3G9oUkSX1YIT0mVgHj8h6PBVZ3XSiEcBrw/wFnxxgbe1pRjPG2GOPMGOPMAw44YHfqlSRJfV+vtS3A9oUkSX1dIcHEImBiCGFCCKEUuAi4N3+BEMLRwH+SNBze7f0yJUlSP2LbQpIktdtpMBFjbAE+AzwAvATMjzG+EEK4PoRwdnaxfwMGA3eFEJaEEO7dzuokSdIAZ9tCkiTlK2SMCWKMC4GFXZ77at7903q5LkmS1I/ZtpAkSTmFnMohSZIkSZK0VxhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BhMSJIkSZKk1BQUTIQQ5oQQloUQlocQru1hflkI4c7s/KdCCON7u1BJktR/2LaQJEk5Ow0mQggZ4CbgTOAo4OIQwlFdFvs4sDHGeBjwbeD/9HahkiSpf7BtIUmS8hXSY2IWsDzG+FqMsQn4GXBOl2XOAX6Uvb8AODWEEHqvTEmS1I/YtpAkSe0KCSbGACvzHq/KPtfjMjHGFmAzMLw3CpQkSf2ObQtJktSuuIBlevrrRNyNZQghXAlcmX3YGEJYWsD7a+8ZAaxLuwh5HPYDHoP9g8chfe/ZR+/Ta20L6Na+2BZCWLYHtfV1fo86c3905z7pzn3SnfukO/dJd4Xsk0MKWVEhwcQqYFze47HA6u0ssyqEUAwMATZ0XVGM8TbgNoAQwuIY48xCitTe4THYP3gc0ucx2D94HNIXQli8j96q19oW0Ll9MdD5PerM/dGd+6Q790l37pPu3Cfd9eY+KeRUjkXAxBDChBBCKXARcG+XZe4FLsvePx/47xhjj3/VkCRJA55tC0mS1G6nPSZijC0hhM8ADwAZ4PsxxhdCCNcDi2OM9wLfA34SQlhO8teMi/Zm0ZIkqe+ybSFJkvIVcioHMcaFwMIuz301734DcMEuvrddLtPnMdg/eBzS5zHYP3gc0rfPjsFealvI71FX7o/u3CfduU+6c5905z7prtf2SbBXpCRJkiRJSkshY0xIkiRJkiTtFXs9mAghzAkhLAshLA8hXNvD/LIQwp3Z+U+FEMbv7ZoGmgKOwedDCC+GEJ4PITwUQijoki7aNTs7DnnLnR9CiCEER/3tZYUcgxDChdnvwwshhDv2dY39XQH/Hh0cQng4hPBs9t+ks9Kosz8LIXw/hPDu9i7ZHRI3Zo/R8yGEGfu6RnXo6XiFEIaFEH4XQnglezs0+/x2j10I4bLs8q+EEC7r6b36ihDCuOy/Ey9l/63+H9nnB+R+CSGUhxD+FEJ4Lrs/vp59fkK2bf1Ktq1dmn1+u23vEMKXs88vCyH8dTpb1HtCCJns/yf3ZR8P6H0SQlgRQvhzCGFJyF6FaaB+b3JCCDUhhAUhhJez/6a8byDvkxDCe7Kfj9y0JYRw9T7ZJzHGvTaRDGj1KnAoUAo8BxzVZZlPA7dm718E3Lk3axpoU4HH4GSgInv/Ko9BOschu1wV8CjwJDAz7br701Tgd2Ei8CwwNPt4ZNp196epwGNwG3BV9v5RwIq06+5vEzAbmAEs3c78s4DfAAF4L/BU2jUP5Kmn4wXcAFybvX8t8H92dOyAYcBr2duh2ftD0962Pdgno4EZ2ftVwF+y/14MyP2S3a7B2fslwFPZ7ZwPXJR9/ta8f1t7bHtn9+FzQBkwIfvvdSbt7dvDffN54A7gvuzjAb1PgBXAiC7PDcjvTd72/wj4RPZ+KVAz0PdJ3r7JAGuAQ/bFPtnbPSZmActjjK/FGJuAnwHndFnmHJIPBMAC4NQQQtjLdQ0kOz0GMcaHY4x12YdPklxPXr2rkO8CwD+RfPEb9mVxA0Qhx+AK4KYY40aAGOO7+7jG/q6QYxCB6uz9IcDqfVjfgBBjfJTkKhfbcw7w45h4EqgJIYzeN9Wpq+0cr/y204+AD+c939Ox+2vgdzHGDdl/334HzNn71e8dMca3Y4zPZO9vBV4CxjBA90t2u7ZlH5ZkpwicQtK2hu77o6e29znAz2KMjTHG14HlJP9u90khhLHAB4HvZh8HBvg+2Y4B+b0BCCFUk4S/3wOIMTbFGDcxgPdJF6cCr8YY32Af7JO9HUyMAVbmPV6Vfa7HZWKMLcBmYPhermsgKeQY5Ps4Seql3rXT4xBCOBoYF2O8b18WNoAU8l04HDg8hPDHEMKTIYT+8J/K/qSQY/A14NIQwiqSKzZ8dt+Upjy7+v+G9r0DY4xvQ/IjHRiZfX57x67fHtNsl/ujSXoJDNj9kj1lYQnwLskPgFeBTdm2NXTetu21vfvN/sj6DvAPQFv28XDcJxH4bQjh6RDCldnnBuz3hqQH51rgB9lTfr4bQqhkYO+TfBcBP83e3+v7ZG8HEz31fOh6GZBCltHuK3j/hhAuBWYC/7ZXKxqYdngcQghFwLeBL+yzigaeQr4LxSSnc5wEXAx8N4RQs5frGkgKOQYXAz+MMY4l6R74k+z3Q/uO/y/3Xds7dv3ymIYQBgM/B66OMW7Z0aI9PNev9kuMsTXGOJ2k1+ss4MieFsve9vv9EUL4EPBujPHp/Kd7WHTA7JOs42OMM4Azgb8PIczewbL/f3v3H3p3Vcdx/PnS1Vw1NnP+5Y/WRDJdtUxCS2LQWMw/VpaJIRhtFFIkBAbJqJj9ExgShA2TyH8KaaXzS8Wy0kFJteb4urkkcs5qZMuGWhgsG+/++Jyrt+1+9/22vvd75/c+H3C5n3s+5/P5ns85u/ee+9455zMOdbKAbqrclqp6O/AC3TSFqYxDnQDQ1l9ZD2ydLuuAtJOqk2F39g4C5/W9Ppfjh+W+lCfJArqhuycaYqr/zUzagCRrgE3A+qo6MkdlGyfTtcNiYCWwI8lTdHO0JuICmLNppp9H91fVi23I5u/oAhWaHTNpg410c4Cpql8CZwDL5qR06pnR94ZG6lBvek177k07m6rt5l2bJnkVXVDi21V1b0se+3ppw9B30PUjlra+Nfz3tU3V955P9fFuYH3rU91DN4Xjq4x3nVBVf27PfwXuowtijfP75iBwsKp+3V5/jy5QMc510rMO2F1Vh9rrodfJsAMTvwEuTLcC7qvphoNMHJNnAuit0nkN8GC1FTM0K6ZtgzaF4E66oIRz6ofjhO1QVc9X1bKqWl5Vy+nW+lhfVbtGU9x5aSafR9voFoMlyTK6qR1Pzmkp57eZtMEf6eY0kuTNdIGJZ+a0lJoAbmgrbV8OPN8bvqlTRn/f6aPA/X3pg9rux8DaJGe2ldTXtrRXpDb3/5vA41V1e9+usayXJGf3RvclWQSsoVt34yG6vjUcXx+D+t4TwHXp7lDxRrrA/M65uYrZVVW3VNW5rU91Hd01Xs8Y10mS1yZZ3Num+/f+GGP6vgGoqr8Af0ryppb0XuC3jHGd9PkIL0/jgLmokxr+ap5X0a2WvB/Y1NJupfvRBV2ncyvdYjI7gRXDLtO4PWbQBj8FDgGT7TEx6jLPx8d07XBM3h14V445bwO6YWe3030p7aWt3O1jTtvgYuBhulXQJ4G1oy7zfHu0jsbTwIt0/6OxEbgRuLHtD3BHa6O9fhadku11FvAz4Pft+fXTtR2wofW1ngA+Nurr+j/r5Eq6IcF7+vouV41rvQBvpbuj1B66H5pfaOkrWt/6idbXXtjSp+x7042e3U83YnDdqK9tlupnNS/flWNs66Rd+6Ptsa/vO3gs3zd917IK2NXeP9vo7iAx7nXyGuAwsKQvbeh1knaQJEmSJEnSnHNBMUmSJEmSNDIGJiRJkiRJ0sgYmJAkSZIkSSNjYEKSJEmSJI2MgQlJkiRJkjQyBiYkSZIkHSfJ0iSfPMljf5Rk6TR5bk2y5uRKN6MyDPX8kmaPtwuVJEmSdJwky4EfVNXKAftOr6qjc14oSfOSIyYkSZIkDfJl4IIkk0luS7I6yUNJvgPsBUiyLckjSfYl+UTvwCRPJVmWZHmSx5Pc1fI8kGRRy3N3kmv68m9OsjvJ3iQXtfSzk/ykpd+Z5A9JlvUXMsnp7VyPtWM/03/+JJe1a5hs+6vtvyDJ9lb+n/f+pqS5Z2BCkiRJ0iCfA/ZX1aqq+mxLeyewqaoubq83VNU7gMuAm5KcNeA8FwJ3VNUlwHPAh6b4e3+rqkuBLcDNLe2LwIMt/T7g/AHHrQLOqaqVVfUW4Fv9O6tqV7uGVcB24Ctt1zeAT7fy3wx8feqqkDRMC0ZdAEmSJEmvGDur6kDf65uSXN22z6MLQhw+5pgDVTXZth8Blk9x7nv78nywbV8JXA1QVduTPDvguCeBFUm+BvwQeGDQyZNcC1wKrE3yOuBdwNYkvSwLpyiXpCEzMCFJkiRppl7obSRZDawBrqiqfybZAZwx4JgjfdtHgUVTnPtIX57e75RMkfclVfVskrcB7wM+BVwLbOjPk+QSYDPwnqo6muQ04Lk2ikLSiDmVQ5IkSdIg/wAWn2D/EuDZFpS4CLh8CGX4BV2ggSRrgTOPzdDWnDitqr4PfJ5uVET//iXAPcANVfUMQFX9HTiQ5MMtT1pwQ9IIGJiQJEmSdJyqOgw83BaVvG1Alu3AgiR7gC8BvxpCMTbTTb3YDawDnqYLmPQ7B9iRZBK4G7jlmP0fAN4A3NVbBLOlXw9sTPIosA94/xDKL2kGvF2oJEmSpFNSkoXA0ar6d5IrgC1Ov5DmH9eYkCRJknSqOh/4blsT4l/Ax0dcHklD4IgJSZIkSZI0Mq4xIUmSJEmSRsbAhCRJkiRJGhkDE5IkSZIkaWQMTEiSJEmSpJExMCFJkiRJkkbGwIQkSZIkSRqZ/wCqAD0noEAkzgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import learning_curve\n",
    "\n",
    "fig, ax = plt.subplots(1, 2, figsize=(16, 6))\n",
    "fig.subplots_adjust(left=0.0625, right=0.95, wspace=0.1)\n",
    "\n",
    "# for i, degree in enumerate([2, 9]):\n",
    "N, train_lc, val_lc = learning_curve(LogisticRegression(),\n",
    "                                     X_train, y_train, cv=5,\n",
    "                                     train_sizes=np.linspace(0.01,1,100))\n",
    "\n",
    "ax[i].plot(N, np.mean(train_lc, 1), color='blue', label='training score')\n",
    "ax[i].plot(N, np.mean(val_lc, 1), color='red', label='validation score')\n",
    "ax[i].hlines(np.mean([train_lc[-1], val_lc[-1]]), N[0], N[-1],\n",
    "             color='gray', linestyle='dashed')\n",
    "\n",
    "ax[i].set_ylim(0, 1)\n",
    "ax[i].set_xlim(N[0], N[-1])\n",
    "ax[i].set_xlabel('training size')\n",
    "ax[i].set_ylabel('score')\n",
    "# ax[i].set_title('degree = {0}'.format(degree), size=14)\n",
    "ax[i].legend(loc='best')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><I><B><center>...The End...</center></B></I></h3>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}