fastai-xente-project-v.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [],
   "source": [
    "#we are going to use fastai.\n",
    "#%pip install fastai==0.7.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fraud Detection Zindi Project\n",
    "\n",
    "By Atwine Mugume\n",
    "\n",
    "Some of the resources used here include these articles that educate on imbalanced data\n",
    "\n",
    "- [Dealing with Imbalanced Data](https://towardsdatascience.com/methods-for-dealing-with-imbalanced-data-5b761be45a18)\n",
    "- [Handling Imbalanced Datasets: Predicting Credit Card Fraud](https://medium.com/coinmonks/handling-imbalanced-datasets-predicting-credit-card-fraud-544f5e74e0fd)\n",
    "- [Tackling Class Imbalance](https://www.kaggle.com/shahules/tackling-class-imbalance)  thanks to Shahules786's kernel\n",
    "- [Great article on scaling data](https://www.kaggle.com/discdiver/guide-to-scaling-and-standardizing?source=post_page---------------------------)\n",
    "-[performance-metrics-for-classification-problems](https://medium.com/thalus-ai/performance-metrics-for-classification-problems-in-machine-learning-part-i-b085d432082b)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['training.csv', 'test.csv', '.ipynb_checkpoints', 'Xente_Variable_Definitions.csv', 'fastai-xente-project-v.ipynb']\n",
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 6.68 µs\n",
      "Importing fastai libraries\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"./\"))\n",
    "\n",
    "# Any results you write to the current directory are saved as output.\n",
    "\n",
    "%time\n",
    "print('Importing fastai libraries')\n",
    "from fastai.imports import *\n",
    "from fastai.structured import *\n",
    "\n",
    "from pandas_summary import DataFrameSummary\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from IPython.display import display\n",
    "\n",
    "from sklearn import metrics\n",
    "\n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%pip install --upgrade numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done Readind the data!\n"
     ]
    }
   ],
   "source": [
    "#let's take a quick look at the data\n",
    "#since we have a time variable we have to parse the dates\n",
    "train = pd.read_csv('./training.csv', low_memory=True, \n",
    "                     parse_dates=[\"TransactionStartTime\"])\n",
    "test = pd.read_csv('./test.csv',low_memory=True, \n",
    "                     parse_dates=[\"TransactionStartTime\"])\n",
    "\n",
    "print('Done Readind the data!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Column Name</th>\n",
       "      <th>Definition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TransactionId</td>\n",
       "      <td>Unique �transaction identifier on platform</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BatchId</td>\n",
       "      <td>Unique number assigned to a batch of transacti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AccountId</td>\n",
       "      <td>Unique number identifying the customer on plat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SubscriptionId</td>\n",
       "      <td>Unique number identifying the customer subscri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CustomerId</td>\n",
       "      <td>Unique identifier attached to Account</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CurrencyCode</td>\n",
       "      <td>Country currency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CountryCode</td>\n",
       "      <td>Numerical geographical code of country</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ProviderId</td>\n",
       "      <td>Source provider of Item �bought.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ProductId</td>\n",
       "      <td>Item name being bought.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ProductCategory</td>\n",
       "      <td>ProductIds are organized into these broader pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ChannelId</td>\n",
       "      <td>Identifies if customer used web,Android, IOS, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Amount</td>\n",
       "      <td>Value of the transaction. Positive for debits ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Value</td>\n",
       "      <td>Absolute value of the amount</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>TransactionStartTime</td>\n",
       "      <td>Transaction start time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>PricingStrategy</td>\n",
       "      <td>Category of Xente's pricing structure for merc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>FraudResult</td>\n",
       "      <td>Fraud status of transaction 1 -yes or 0-No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Column Name                                         Definition\n",
       "0          TransactionId         Unique �transaction identifier on platform\n",
       "1                BatchId  Unique number assigned to a batch of transacti...\n",
       "2              AccountId  Unique number identifying the customer on plat...\n",
       "3         SubscriptionId  Unique number identifying the customer subscri...\n",
       "4             CustomerId              Unique identifier attached to Account\n",
       "5           CurrencyCode                                   Country currency\n",
       "6            CountryCode             Numerical geographical code of country\n",
       "7             ProviderId                   Source provider of Item �bought.\n",
       "8              ProductId                            Item name being bought.\n",
       "9        ProductCategory  ProductIds are organized into these broader pr...\n",
       "10             ChannelId  Identifies if customer used web,Android, IOS, ...\n",
       "11                Amount  Value of the transaction. Positive for debits ...\n",
       "12                 Value                       Absolute value of the amount\n",
       "13  TransactionStartTime                             Transaction start time\n",
       "14       PricingStrategy  Category of Xente's pricing structure for merc...\n",
       "15           FraudResult         Fraud status of transaction 1 -yes or 0-No"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's have a description of the variables here to keep refering to them\n",
    "feature_desc = pd.read_csv('./Xente_Variable_Definitions.csv')\n",
    "feature_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 16), (45019, 15))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#what does the data look like\n",
    "train.shape,test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are datasets which we have to look at carefully because they have a time variable. We have to make sure the time is progressive.\n",
    "That is we are having the first rows from a time before those at the end of the dataset\n",
    "\n",
    "For example if the first row is on the date 9/Jan/2019 the last row should not be the same date...it should be maybe 25/Oct/2021 so we can have a progression. \n",
    "\n",
    "Therefore our model will be used to tell the future."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionId</th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>FraudResult</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TransactionId_76871</td>\n",
       "      <td>BatchId_36123</td>\n",
       "      <td>AccountId_3957</td>\n",
       "      <td>SubscriptionId_887</td>\n",
       "      <td>CustomerId_4406</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2018-11-15 02:18:49+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TransactionId_73770</td>\n",
       "      <td>BatchId_15642</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_4406</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-11-15 02:19:08+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TransactionId_26203</td>\n",
       "      <td>BatchId_53941</td>\n",
       "      <td>AccountId_4229</td>\n",
       "      <td>SubscriptionId_222</td>\n",
       "      <td>CustomerId_4683</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_1</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>500.0</td>\n",
       "      <td>500</td>\n",
       "      <td>2018-11-15 02:44:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TransactionId_380</td>\n",
       "      <td>BatchId_102363</td>\n",
       "      <td>AccountId_648</td>\n",
       "      <td>SubscriptionId_2185</td>\n",
       "      <td>CustomerId_988</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_1</td>\n",
       "      <td>ProductId_21</td>\n",
       "      <td>utility_bill</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>21800</td>\n",
       "      <td>2018-11-15 03:32:55+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TransactionId_28195</td>\n",
       "      <td>BatchId_38780</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_988</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-644.0</td>\n",
       "      <td>644</td>\n",
       "      <td>2018-11-15 03:34:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         TransactionId         BatchId       AccountId       SubscriptionId  \\\n",
       "0  TransactionId_76871   BatchId_36123  AccountId_3957   SubscriptionId_887   \n",
       "1  TransactionId_73770   BatchId_15642  AccountId_4841  SubscriptionId_3829   \n",
       "2  TransactionId_26203   BatchId_53941  AccountId_4229   SubscriptionId_222   \n",
       "3    TransactionId_380  BatchId_102363   AccountId_648  SubscriptionId_2185   \n",
       "4  TransactionId_28195   BatchId_38780  AccountId_4841  SubscriptionId_3829   \n",
       "\n",
       "        CustomerId CurrencyCode  CountryCode    ProviderId     ProductId  \\\n",
       "0  CustomerId_4406          UGX          256  ProviderId_6  ProductId_10   \n",
       "1  CustomerId_4406          UGX          256  ProviderId_4   ProductId_6   \n",
       "2  CustomerId_4683          UGX          256  ProviderId_6   ProductId_1   \n",
       "3   CustomerId_988          UGX          256  ProviderId_1  ProductId_21   \n",
       "4   CustomerId_988          UGX          256  ProviderId_4   ProductId_6   \n",
       "\n",
       "      ProductCategory    ChannelId   Amount  Value      TransactionStartTime  \\\n",
       "0             airtime  ChannelId_3   1000.0   1000 2018-11-15 02:18:49+00:00   \n",
       "1  financial_services  ChannelId_2    -20.0     20 2018-11-15 02:19:08+00:00   \n",
       "2             airtime  ChannelId_3    500.0    500 2018-11-15 02:44:21+00:00   \n",
       "3        utility_bill  ChannelId_3  20000.0  21800 2018-11-15 03:32:55+00:00   \n",
       "4  financial_services  ChannelId_2   -644.0    644 2018-11-15 03:34:21+00:00   \n",
       "\n",
       "   PricingStrategy  FraudResult  \n",
       "0                2            0  \n",
       "1                2            0  \n",
       "2                2            0  \n",
       "3                2            0  \n",
       "4                2            0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's have a snippet of the head of the two datasets\n",
    "train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionId</th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>FraudResult</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95657</th>\n",
       "      <td>TransactionId_89881</td>\n",
       "      <td>BatchId_96668</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_3078</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 09:54:09+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95658</th>\n",
       "      <td>TransactionId_91597</td>\n",
       "      <td>BatchId_3503</td>\n",
       "      <td>AccountId_3439</td>\n",
       "      <td>SubscriptionId_2643</td>\n",
       "      <td>CustomerId_3874</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 09:54:25+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95659</th>\n",
       "      <td>TransactionId_82501</td>\n",
       "      <td>BatchId_118602</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_3874</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2019-02-13 09:54:35+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95660</th>\n",
       "      <td>TransactionId_136354</td>\n",
       "      <td>BatchId_70924</td>\n",
       "      <td>AccountId_1346</td>\n",
       "      <td>SubscriptionId_652</td>\n",
       "      <td>CustomerId_1709</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_19</td>\n",
       "      <td>tv</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>3000</td>\n",
       "      <td>2019-02-13 10:01:10+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95661</th>\n",
       "      <td>TransactionId_35670</td>\n",
       "      <td>BatchId_29317</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_1709</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-60.0</td>\n",
       "      <td>60</td>\n",
       "      <td>2019-02-13 10:01:28+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              TransactionId         BatchId       AccountId  \\\n",
       "95657   TransactionId_89881   BatchId_96668  AccountId_4841   \n",
       "95658   TransactionId_91597    BatchId_3503  AccountId_3439   \n",
       "95659   TransactionId_82501  BatchId_118602  AccountId_4841   \n",
       "95660  TransactionId_136354   BatchId_70924  AccountId_1346   \n",
       "95661   TransactionId_35670   BatchId_29317  AccountId_4841   \n",
       "\n",
       "            SubscriptionId       CustomerId CurrencyCode  CountryCode  \\\n",
       "95657  SubscriptionId_3829  CustomerId_3078          UGX          256   \n",
       "95658  SubscriptionId_2643  CustomerId_3874          UGX          256   \n",
       "95659  SubscriptionId_3829  CustomerId_3874          UGX          256   \n",
       "95660   SubscriptionId_652  CustomerId_1709          UGX          256   \n",
       "95661  SubscriptionId_3829  CustomerId_1709          UGX          256   \n",
       "\n",
       "         ProviderId     ProductId     ProductCategory    ChannelId  Amount  \\\n",
       "95657  ProviderId_4   ProductId_6  financial_services  ChannelId_2 -1000.0   \n",
       "95658  ProviderId_6  ProductId_10             airtime  ChannelId_3  1000.0   \n",
       "95659  ProviderId_4   ProductId_6  financial_services  ChannelId_2   -20.0   \n",
       "95660  ProviderId_6  ProductId_19                  tv  ChannelId_3  3000.0   \n",
       "95661  ProviderId_4   ProductId_6  financial_services  ChannelId_2   -60.0   \n",
       "\n",
       "       Value      TransactionStartTime  PricingStrategy  FraudResult  \n",
       "95657   1000 2019-02-13 09:54:09+00:00                2            0  \n",
       "95658   1000 2019-02-13 09:54:25+00:00                2            0  \n",
       "95659     20 2019-02-13 09:54:35+00:00                2            0  \n",
       "95660   3000 2019-02-13 10:01:10+00:00                2            0  \n",
       "95661     60 2019-02-13 10:01:28+00:00                2            0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionId</th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TransactionId_50600</td>\n",
       "      <td>BatchId_35028</td>\n",
       "      <td>AccountId_2441</td>\n",
       "      <td>SubscriptionId_4426</td>\n",
       "      <td>CustomerId_2857</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_3</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 10:01:40+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TransactionId_95109</td>\n",
       "      <td>BatchId_45139</td>\n",
       "      <td>AccountId_3439</td>\n",
       "      <td>SubscriptionId_2643</td>\n",
       "      <td>CustomerId_3874</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_15</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019-02-13 10:02:12+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TransactionId_47357</td>\n",
       "      <td>BatchId_74887</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_2857</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-50.0</td>\n",
       "      <td>50</td>\n",
       "      <td>2019-02-13 10:02:30+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TransactionId_28185</td>\n",
       "      <td>BatchId_11025</td>\n",
       "      <td>AccountId_2685</td>\n",
       "      <td>SubscriptionId_4626</td>\n",
       "      <td>CustomerId_3105</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>3000</td>\n",
       "      <td>2019-02-13 10:02:38+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TransactionId_22140</td>\n",
       "      <td>BatchId_29804</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_3105</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-60.0</td>\n",
       "      <td>60</td>\n",
       "      <td>2019-02-13 10:02:58+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         TransactionId        BatchId       AccountId       SubscriptionId  \\\n",
       "0  TransactionId_50600  BatchId_35028  AccountId_2441  SubscriptionId_4426   \n",
       "1  TransactionId_95109  BatchId_45139  AccountId_3439  SubscriptionId_2643   \n",
       "2  TransactionId_47357  BatchId_74887  AccountId_4841  SubscriptionId_3829   \n",
       "3  TransactionId_28185  BatchId_11025  AccountId_2685  SubscriptionId_4626   \n",
       "4  TransactionId_22140  BatchId_29804  AccountId_4841  SubscriptionId_3829   \n",
       "\n",
       "        CustomerId CurrencyCode  CountryCode    ProviderId     ProductId  \\\n",
       "0  CustomerId_2857          UGX          256  ProviderId_5   ProductId_3   \n",
       "1  CustomerId_3874          UGX          256  ProviderId_5  ProductId_15   \n",
       "2  CustomerId_2857          UGX          256  ProviderId_4   ProductId_6   \n",
       "3  CustomerId_3105          UGX          256  ProviderId_5  ProductId_10   \n",
       "4  CustomerId_3105          UGX          256  ProviderId_4   ProductId_6   \n",
       "\n",
       "      ProductCategory    ChannelId  Amount  Value      TransactionStartTime  \\\n",
       "0             airtime  ChannelId_3  1000.0   1000 2019-02-13 10:01:40+00:00   \n",
       "1  financial_services  ChannelId_3  2000.0   2000 2019-02-13 10:02:12+00:00   \n",
       "2  financial_services  ChannelId_2   -50.0     50 2019-02-13 10:02:30+00:00   \n",
       "3             airtime  ChannelId_3  3000.0   3000 2019-02-13 10:02:38+00:00   \n",
       "4  financial_services  ChannelId_2   -60.0     60 2019-02-13 10:02:58+00:00   \n",
       "\n",
       "   PricingStrategy  \n",
       "0                4  \n",
       "1                2  \n",
       "2                2  \n",
       "3                4  \n",
       "4                2  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's get snippet of the test data\n",
    "test.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionId</th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>45014</th>\n",
       "      <td>TransactionId_116198</td>\n",
       "      <td>BatchId_117348</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_5225</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>-5000.0</td>\n",
       "      <td>5000</td>\n",
       "      <td>2019-03-14 23:56:28+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45015</th>\n",
       "      <td>TransactionId_24829</td>\n",
       "      <td>BatchId_53630</td>\n",
       "      <td>AccountId_4803</td>\n",
       "      <td>SubscriptionId_1738</td>\n",
       "      <td>CustomerId_5272</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_3</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>2019-03-14 23:57:38+00:00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45016</th>\n",
       "      <td>TransactionId_102920</td>\n",
       "      <td>BatchId_112209</td>\n",
       "      <td>AccountId_4817</td>\n",
       "      <td>SubscriptionId_4613</td>\n",
       "      <td>CustomerId_5286</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_3</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>2019-03-14 23:59:17+00:00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45017</th>\n",
       "      <td>TransactionId_128439</td>\n",
       "      <td>BatchId_58734</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_5286</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>-5000.0</td>\n",
       "      <td>5000</td>\n",
       "      <td>2019-03-14 23:59:39+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45018</th>\n",
       "      <td>TransactionId_45927</td>\n",
       "      <td>BatchId_98308</td>\n",
       "      <td>AccountId_4593</td>\n",
       "      <td>SubscriptionId_3866</td>\n",
       "      <td>CustomerId_5057</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_3</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>3500</td>\n",
       "      <td>2019-03-14 23:59:51+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              TransactionId         BatchId       AccountId  \\\n",
       "45014  TransactionId_116198  BatchId_117348  AccountId_4841   \n",
       "45015   TransactionId_24829   BatchId_53630  AccountId_4803   \n",
       "45016  TransactionId_102920  BatchId_112209  AccountId_4817   \n",
       "45017  TransactionId_128439   BatchId_58734  AccountId_4841   \n",
       "45018   TransactionId_45927   BatchId_98308  AccountId_4593   \n",
       "\n",
       "            SubscriptionId       CustomerId CurrencyCode  CountryCode  \\\n",
       "45014  SubscriptionId_3829  CustomerId_5225          UGX          256   \n",
       "45015  SubscriptionId_1738  CustomerId_5272          UGX          256   \n",
       "45016  SubscriptionId_4613  CustomerId_5286          UGX          256   \n",
       "45017  SubscriptionId_3829  CustomerId_5286          UGX          256   \n",
       "45018  SubscriptionId_3866  CustomerId_5057          UGX          256   \n",
       "\n",
       "         ProviderId     ProductId     ProductCategory    ChannelId   Amount  \\\n",
       "45014  ProviderId_4   ProductId_6  financial_services  ChannelId_3  -5000.0   \n",
       "45015  ProviderId_3  ProductId_10             airtime  ChannelId_3  10000.0   \n",
       "45016  ProviderId_3  ProductId_10             airtime  ChannelId_3  10000.0   \n",
       "45017  ProviderId_4   ProductId_6  financial_services  ChannelId_3  -5000.0   \n",
       "45018  ProviderId_5   ProductId_3             airtime  ChannelId_3   3500.0   \n",
       "\n",
       "       Value      TransactionStartTime  PricingStrategy  \n",
       "45014   5000 2019-03-14 23:56:28+00:00                2  \n",
       "45015  10000 2019-03-14 23:57:38+00:00                1  \n",
       "45016  10000 2019-03-14 23:59:17+00:00                1  \n",
       "45017   5000 2019-03-14 23:59:39+00:00                2  \n",
       "45018   3500 2019-03-14 23:59:51+00:00                4  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 95662 entries, 0 to 95661\n",
      "Data columns (total 16 columns):\n",
      "TransactionId           95662 non-null object\n",
      "BatchId                 95662 non-null object\n",
      "AccountId               95662 non-null object\n",
      "SubscriptionId          95662 non-null object\n",
      "CustomerId              95662 non-null object\n",
      "CurrencyCode            95662 non-null object\n",
      "CountryCode             95662 non-null int64\n",
      "ProviderId              95662 non-null object\n",
      "ProductId               95662 non-null object\n",
      "ProductCategory         95662 non-null object\n",
      "ChannelId               95662 non-null object\n",
      "Amount                  95662 non-null float64\n",
      "Value                   95662 non-null int64\n",
      "TransactionStartTime    95662 non-null datetime64[ns, UTC]\n",
      "PricingStrategy         95662 non-null int64\n",
      "FraudResult             95662 non-null int64\n",
      "dtypes: datetime64[ns, UTC](1), float64(1), int64(4), object(10)\n",
      "memory usage: 11.7+ MB\n"
     ]
    }
   ],
   "source": [
    "#Let's take a peek at the data into\n",
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 16), (45019, 15))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's look at the shapes of the data\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Data Analysis\n",
    "\n",
    "### Encoding\n",
    "\n",
    "I have only discovered that the way you encode the data you have can affect how well your model will function. I have found some great resources here in these [article](https://medium.com/data-design/visiting-categorical-features-and-encoding-in-decision-trees-53400fa65931)\n",
    "\n",
    "-This article is also a [good one](https://towardsdatascience.com/smarter-ways-to-encode-categorical-data-for-machine-learning-part-1-of-3-6dca2f71b159)\n",
    "\n",
    "What stands out the most is these two points:\n",
    "\n",
    "- Categorical features with large cardinalities/features (over 1000): `Binary Encoding` is ideal\n",
    "- Categorical features with small cardinalities/features (less than 1000): `Numeric Encoding` is ideal\n",
    "\n",
    "As an esample here is a code below from the library: binaryencoder\n",
    "\n",
    "```python \n",
    "from category_encoders import *\n",
    "import pandas as pd\n",
    "from sklearn.datasets import load_boston\n",
    "\n",
    "# prepare some data\n",
    "bunch = load_boston()\n",
    "y = bunch.target\n",
    "X = pd.DataFrame(bunch.data, columns=bunch.feature_names)\n",
    "\n",
    "# use binary encoding to encode two categorical features\n",
    "enc = BinaryEncoder(cols=['CHAS', 'RAD']).fit(X)\n",
    "\n",
    "# transform the dataset\n",
    "numeric_dataset = enc.transform(X)\n",
    "\n",
    "\n",
    "## A supervised example\n",
    "\n",
    "from category_encoders import *\n",
    "import pandas as pd\n",
    "from sklearn.datasets import load_boston\n",
    "\n",
    "# prepare some data\n",
    "bunch = load_boston()\n",
    "y_train = bunch.target[0:250]\n",
    "y_test = bunch.target[250:506]\n",
    "X_train = pd.DataFrame(bunch.data[0:250], columns=bunch.feature_names)\n",
    "X_test = pd.DataFrame(bunch.data[250:506], columns=bunch.feature_names)\n",
    "\n",
    "# use target encoding to encode two categorical features\n",
    "enc = TargetEncoder(cols=['CHAS', 'RAD']).fit(X_train, y_train)\n",
    "\n",
    "# transform the datasets\n",
    "training_numeric_dataset = enc.transform(X_train, y_train)\n",
    "testing_numeric_dataset = enc.transform(X_test)\n",
    "\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TransactionId has 95662\n",
      "BatchId has 94809\n",
      "AccountId has 3633\n",
      "SubscriptionId has 3627\n",
      "CustomerId has 3742\n",
      "CurrencyCode has 1\n",
      "CountryCode has 1\n",
      "ProviderId has 6\n",
      "ProductId has 23\n",
      "ProductCategory has 9\n",
      "ChannelId has 4\n",
      "Amount has 1676\n",
      "Value has 1517\n",
      "TransactionStartTime has 94556\n",
      "PricingStrategy has 4\n",
      "FraudResult has 2\n"
     ]
    }
   ],
   "source": [
    "#how unique is the batchId?\n",
    "#i need to know if I should leave it in the conversation\n",
    "for col in train.columns:\n",
    "    print(col,'has', train[col].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's install the package with the encoders that make sense for this kind of data with only 10+ features\n",
    "#%pip install category_encoders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lil Function\n",
    "\n",
    "One fo the things I have learnt is to make code reusable, so I am going to write a function that will help us encode without having to repeat the code so much.\n",
    "\n",
    "`Why is this function so important?`\n",
    "\n",
    "I have been having memory issues wen I use the pd.getdummies. \n",
    "I normally have memory limited to 13Gb and as a result this method of replacing the categorical values with simple codes keeps the memory usage under 200Mbs. Which is quite phenomenal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#little function to help us encode data\n",
    "def lil_encoder(df,col):\n",
    "    df[col] = train[col].astype('category')\n",
    "    \n",
    "    #let's create a new column of the categorical features\n",
    "    #df[col] = df[col].cat.codes\n",
    "    \n",
    "    return df.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we need to keep the ids of the test data\n",
    "fin_Id = test['TransactionId']\n",
    "y = train['FraudResult']\n",
    "\n",
    "#let's now drop them from their original datasets\n",
    "test = test.drop(columns=['TransactionId'])\n",
    "train = train.drop(columns=['FraudResult','TransactionId'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 14), (45019, 14))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#i want to see if the datasets are aligned, when they are not aligned one gets problems in fitting the data\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BatchId_35028</td>\n",
       "      <td>AccountId_2441</td>\n",
       "      <td>SubscriptionId_4426</td>\n",
       "      <td>CustomerId_2857</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_3</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 10:01:40+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BatchId_45139</td>\n",
       "      <td>AccountId_3439</td>\n",
       "      <td>SubscriptionId_2643</td>\n",
       "      <td>CustomerId_3874</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_15</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019-02-13 10:02:12+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BatchId_74887</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_2857</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-50.0</td>\n",
       "      <td>50</td>\n",
       "      <td>2019-02-13 10:02:30+00:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BatchId_11025</td>\n",
       "      <td>AccountId_2685</td>\n",
       "      <td>SubscriptionId_4626</td>\n",
       "      <td>CustomerId_3105</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_5</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>3000</td>\n",
       "      <td>2019-02-13 10:02:38+00:00</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         BatchId       AccountId       SubscriptionId       CustomerId  \\\n",
       "0  BatchId_35028  AccountId_2441  SubscriptionId_4426  CustomerId_2857   \n",
       "1  BatchId_45139  AccountId_3439  SubscriptionId_2643  CustomerId_3874   \n",
       "2  BatchId_74887  AccountId_4841  SubscriptionId_3829  CustomerId_2857   \n",
       "3  BatchId_11025  AccountId_2685  SubscriptionId_4626  CustomerId_3105   \n",
       "\n",
       "  CurrencyCode  CountryCode    ProviderId     ProductId     ProductCategory  \\\n",
       "0          UGX          256  ProviderId_5   ProductId_3             airtime   \n",
       "1          UGX          256  ProviderId_5  ProductId_15  financial_services   \n",
       "2          UGX          256  ProviderId_4   ProductId_6  financial_services   \n",
       "3          UGX          256  ProviderId_5  ProductId_10             airtime   \n",
       "\n",
       "     ChannelId  Amount  Value      TransactionStartTime  PricingStrategy  \n",
       "0  ChannelId_3  1000.0   1000 2019-02-13 10:01:40+00:00                4  \n",
       "1  ChannelId_3  2000.0   2000 2019-02-13 10:02:12+00:00                2  \n",
       "2  ChannelId_2   -50.0     50 2019-02-13 10:02:30+00:00                2  \n",
       "3  ChannelId_3  3000.0   3000 2019-02-13 10:02:38+00:00                4  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId</th>\n",
       "      <th>AccountId</th>\n",
       "      <th>SubscriptionId</th>\n",
       "      <th>CustomerId</th>\n",
       "      <th>CurrencyCode</th>\n",
       "      <th>CountryCode</th>\n",
       "      <th>ProviderId</th>\n",
       "      <th>ProductId</th>\n",
       "      <th>ProductCategory</th>\n",
       "      <th>ChannelId</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>Identifier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BatchId_36123</td>\n",
       "      <td>AccountId_3957</td>\n",
       "      <td>SubscriptionId_887</td>\n",
       "      <td>CustomerId_4406</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_10</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2018-11-15 02:18:49+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BatchId_15642</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_4406</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-11-15 02:19:08+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BatchId_53941</td>\n",
       "      <td>AccountId_4229</td>\n",
       "      <td>SubscriptionId_222</td>\n",
       "      <td>CustomerId_4683</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_6</td>\n",
       "      <td>ProductId_1</td>\n",
       "      <td>airtime</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>500.0</td>\n",
       "      <td>500</td>\n",
       "      <td>2018-11-15 02:44:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BatchId_102363</td>\n",
       "      <td>AccountId_648</td>\n",
       "      <td>SubscriptionId_2185</td>\n",
       "      <td>CustomerId_988</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_1</td>\n",
       "      <td>ProductId_21</td>\n",
       "      <td>utility_bill</td>\n",
       "      <td>ChannelId_3</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>21800</td>\n",
       "      <td>2018-11-15 03:32:55+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BatchId_38780</td>\n",
       "      <td>AccountId_4841</td>\n",
       "      <td>SubscriptionId_3829</td>\n",
       "      <td>CustomerId_988</td>\n",
       "      <td>UGX</td>\n",
       "      <td>256</td>\n",
       "      <td>ProviderId_4</td>\n",
       "      <td>ProductId_6</td>\n",
       "      <td>financial_services</td>\n",
       "      <td>ChannelId_2</td>\n",
       "      <td>-644.0</td>\n",
       "      <td>644</td>\n",
       "      <td>2018-11-15 03:34:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          BatchId       AccountId       SubscriptionId       CustomerId  \\\n",
       "0   BatchId_36123  AccountId_3957   SubscriptionId_887  CustomerId_4406   \n",
       "1   BatchId_15642  AccountId_4841  SubscriptionId_3829  CustomerId_4406   \n",
       "2   BatchId_53941  AccountId_4229   SubscriptionId_222  CustomerId_4683   \n",
       "3  BatchId_102363   AccountId_648  SubscriptionId_2185   CustomerId_988   \n",
       "4   BatchId_38780  AccountId_4841  SubscriptionId_3829   CustomerId_988   \n",
       "\n",
       "  CurrencyCode  CountryCode    ProviderId     ProductId     ProductCategory  \\\n",
       "0          UGX          256  ProviderId_6  ProductId_10             airtime   \n",
       "1          UGX          256  ProviderId_4   ProductId_6  financial_services   \n",
       "2          UGX          256  ProviderId_6   ProductId_1             airtime   \n",
       "3          UGX          256  ProviderId_1  ProductId_21        utility_bill   \n",
       "4          UGX          256  ProviderId_4   ProductId_6  financial_services   \n",
       "\n",
       "     ChannelId   Amount  Value      TransactionStartTime  PricingStrategy  \\\n",
       "0  ChannelId_3   1000.0   1000 2018-11-15 02:18:49+00:00                2   \n",
       "1  ChannelId_2    -20.0     20 2018-11-15 02:19:08+00:00                2   \n",
       "2  ChannelId_3    500.0    500 2018-11-15 02:44:21+00:00                2   \n",
       "3  ChannelId_3  20000.0  21800 2018-11-15 03:32:55+00:00                2   \n",
       "4  ChannelId_2   -644.0    644 2018-11-15 03:34:21+00:00                2   \n",
       "\n",
       "  Identifier  \n",
       "0      Train  \n",
       "1      Train  \n",
       "2      Train  \n",
       "3      Train  \n",
       "4      Train  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let me put the datasets together so that I can work on them at the same time\n",
    "train['Identifier']='Train'\n",
    "test['Identifier']='Test'\n",
    "\n",
    "ls = [train, test]\n",
    "data = pd.concat(ls)\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's encode all the data\n",
    "ls_ = ['AccountId','CustomerId','ProviderId','ProductId','ProductCategory','ChannelId','BatchId','SubscriptionId']\n",
    "\n",
    "#this is the for loop that calls the function and makes the lil_function work\n",
    "for col in ls_:\n",
    "    lil_encoder(data,col)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now have all the columns in the list above transformed from object to categorical, now we are going to encode them with a categorical encoder. I have chosen to update my encoding method because of some of the articles I have read on this topic, they insist that we shd use Binary encoders for small datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from category_encoders import *\n",
    "\n",
    "# use binary encoding to encode two categorical features\n",
    "enc = BinaryEncoder(cols=['AccountId','CustomerId','ProviderId','ProductId','ProductCategory','ChannelId','BatchId','SubscriptionId']).fit(data)\n",
    "\n",
    "data = enc.transform(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>ProductCategory_3</th>\n",
       "      <th>ProductCategory_4</th>\n",
       "      <th>ChannelId_0</th>\n",
       "      <th>ChannelId_1</th>\n",
       "      <th>ChannelId_2</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>Identifier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000</td>\n",
       "      <td>2018-11-15 02:18:49+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-20.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-11-15 02:19:08+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>500.0</td>\n",
       "      <td>500</td>\n",
       "      <td>2018-11-15 02:44:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>21800</td>\n",
       "      <td>2018-11-15 03:32:55+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-644.0</td>\n",
       "      <td>644</td>\n",
       "      <td>2018-11-15 03:34:21+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "0          0          0          1          0          1          1   \n",
       "1          0          0          0          1          1          1   \n",
       "2          0          0          1          1          1          0   \n",
       "3          0          0          0          0          0          0   \n",
       "4          0          0          1          0          1          1   \n",
       "\n",
       "   BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  ProductCategory_3  \\\n",
       "0          0          1          1          1  ...                  0   \n",
       "1          1          1          0          0  ...                  1   \n",
       "2          1          0          1          1  ...                  0   \n",
       "3          0          1          1          1  ...                  0   \n",
       "4          1          1          1          1  ...                  1   \n",
       "\n",
       "   ProductCategory_4  ChannelId_0  ChannelId_1  ChannelId_2   Amount  Value  \\\n",
       "0                  1            0            1            1   1000.0   1000   \n",
       "1                  1            0            1            0    -20.0     20   \n",
       "2                  1            0            1            1    500.0    500   \n",
       "3                  1            0            1            1  20000.0  21800   \n",
       "4                  1            0            1            0   -644.0    644   \n",
       "\n",
       "       TransactionStartTime  PricingStrategy  Identifier  \n",
       "0 2018-11-15 02:18:49+00:00                2       Train  \n",
       "1 2018-11-15 02:19:08+00:00                2       Train  \n",
       "2 2018-11-15 02:44:21+00:00                2       Train  \n",
       "3 2018-11-15 03:32:55+00:00                2       Train  \n",
       "4 2018-11-15 03:34:21+00:00                2       Train  \n",
       "\n",
       "[5 rows x 82 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 79), (45019, 79))"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#dropping some features: there are some features we can see that will not contribute to the ML process\n",
    "\n",
    "#let's split the data\n",
    "train = data[data['Identifier']=='Train']\n",
    "test = data[data['Identifier']=='Test']\n",
    "\n",
    "#now let's drop the useless columns from the datasets\n",
    "train = train.drop(columns=['CurrencyCode','CountryCode','Amount'])\n",
    "test = test.drop(columns=['CurrencyCode','CountryCode','Amount'])\n",
    "\n",
    "#let's see the new shapes\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>ProductCategory_2</th>\n",
       "      <th>ProductCategory_3</th>\n",
       "      <th>ProductCategory_4</th>\n",
       "      <th>ChannelId_0</th>\n",
       "      <th>ChannelId_1</th>\n",
       "      <th>ChannelId_2</th>\n",
       "      <th>Value</th>\n",
       "      <th>TransactionStartTime</th>\n",
       "      <th>PricingStrategy</th>\n",
       "      <th>Identifier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1000</td>\n",
       "      <td>2019-02-13 10:01:40+00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>2019-02-13 10:02:12+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>2019-02-13 10:02:30+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>Test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "0          0          0          1          0          1          1   \n",
       "1          0          0          0          1          1          1   \n",
       "2          0          0          1          1          1          0   \n",
       "\n",
       "   BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  ProductCategory_2  \\\n",
       "0          0          1          1          1  ...                  0   \n",
       "1          1          1          0          0  ...                  0   \n",
       "2          1          0          1          1  ...                  0   \n",
       "\n",
       "   ProductCategory_3  ProductCategory_4  ChannelId_0  ChannelId_1  \\\n",
       "0                  0                  1            0            1   \n",
       "1                  1                  1            0            1   \n",
       "2                  0                  1            0            1   \n",
       "\n",
       "   ChannelId_2  Value      TransactionStartTime  PricingStrategy  Identifier  \n",
       "0            1   1000 2019-02-13 10:01:40+00:00                4        Test  \n",
       "1            0   2000 2019-02-13 10:02:12+00:00                2        Test  \n",
       "2            1     50 2019-02-13 10:02:30+00:00                2        Test  \n",
       "\n",
       "[3 rows x 79 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I have tried to load the date time function from fastai but it has failed so i am going to copy it manually and use it here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_datepart(df, fldname, drop=True, time=False):\n",
    "    \"\"\"add_datepart converts a column of df from a datetime64 to many columns containing\n",
    "    the information from the date. This applies changes inplace.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: A pandas data frame. df gain several new columns.\n",
    "    fldname: A string that is the name of the date column you wish to expand.\n",
    "        If it is not a datetime64 series, it will be converted to one with pd.to_datetime.\n",
    "    drop: If true then the original date column will be removed.\n",
    "    time: If true time features: Hour, Minute, Second will be added.\n",
    "\n",
    "    Examples:\n",
    "    ---------\n",
    "\n",
    "    >>> df = pd.DataFrame({ 'A' : pd.to_datetime(['3/11/2000', '3/12/2000', '3/13/2000'], infer_datetime_format=False) })\n",
    "    >>> df\n",
    "\n",
    "        A\n",
    "    0   2000-03-11\n",
    "    1   2000-03-12\n",
    "    2   2000-03-13\n",
    "\n",
    "    >>> add_datepart(df, 'A')\n",
    "    >>> df\n",
    "\n",
    "        AYear AMonth AWeek ADay ADayofweek ADayofyear AIs_month_end AIs_month_start AIs_quarter_end AIs_quarter_start AIs_year_end AIs_year_start AElapsed\n",
    "    0   2000  3      10    11   5          71         False         False           False           False             False        False          952732800\n",
    "    1   2000  3      10    12   6          72         False         False           False           False             False        False          952819200\n",
    "    2   2000  3      11    13   0          73         False         False           False           False             False        False          952905600\n",
    "    \"\"\"\n",
    "    fld = df[fldname]\n",
    "    fld_dtype = fld.dtype\n",
    "    if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):\n",
    "        fld_dtype = np.datetime64\n",
    "\n",
    "    if not np.issubdtype(fld_dtype, np.datetime64):\n",
    "        df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)\n",
    "    targ_pre = re.sub('[Dd]ate$', '', fldname)\n",
    "    attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',\n",
    "            'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']\n",
    "    if time: attr = attr + ['Hour', 'Minute', 'Second']\n",
    "    for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())\n",
    "    df[targ_pre + 'Elapsed'] = fld.astype(np.int64) // 10 ** 9\n",
    "    if drop: df.drop(fldname, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's add the datetime variables\n",
    "#this is a function in the fastai lib that deals with all variables of time, ie it converts the datatime column into many others\n",
    "add_datepart(test,'TransactionStartTime')\n",
    "add_datepart(train,'TransactionStartTime')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>TransactionStartTimeDay</th>\n",
       "      <th>TransactionStartTimeDayofweek</th>\n",
       "      <th>TransactionStartTimeDayofyear</th>\n",
       "      <th>TransactionStartTimeIs_month_end</th>\n",
       "      <th>TransactionStartTimeIs_month_start</th>\n",
       "      <th>TransactionStartTimeIs_quarter_end</th>\n",
       "      <th>TransactionStartTimeIs_quarter_start</th>\n",
       "      <th>TransactionStartTimeIs_year_end</th>\n",
       "      <th>TransactionStartTimeIs_year_start</th>\n",
       "      <th>TransactionStartTimeElapsed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542248329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542248348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542249861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542252775</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "0          0          0          1          0          1          1   \n",
       "1          0          0          0          1          1          1   \n",
       "2          0          0          1          1          1          0   \n",
       "3          0          0          0          0          0          0   \n",
       "\n",
       "   BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  TransactionStartTimeDay  \\\n",
       "0          0          1          1          1  ...                       15   \n",
       "1          1          1          0          0  ...                       15   \n",
       "2          1          0          1          1  ...                       15   \n",
       "3          0          1          1          1  ...                       15   \n",
       "\n",
       "   TransactionStartTimeDayofweek  TransactionStartTimeDayofyear  \\\n",
       "0                              3                            319   \n",
       "1                              3                            319   \n",
       "2                              3                            319   \n",
       "3                              3                            319   \n",
       "\n",
       "   TransactionStartTimeIs_month_end  TransactionStartTimeIs_month_start  \\\n",
       "0                             False                               False   \n",
       "1                             False                               False   \n",
       "2                             False                               False   \n",
       "3                             False                               False   \n",
       "\n",
       "   TransactionStartTimeIs_quarter_end  TransactionStartTimeIs_quarter_start  \\\n",
       "0                               False                                 False   \n",
       "1                               False                                 False   \n",
       "2                               False                                 False   \n",
       "3                               False                                 False   \n",
       "\n",
       "   TransactionStartTimeIs_year_end  TransactionStartTimeIs_year_start  \\\n",
       "0                            False                              False   \n",
       "1                            False                              False   \n",
       "2                            False                              False   \n",
       "3                            False                              False   \n",
       "\n",
       "   TransactionStartTimeElapsed  \n",
       "0                   1542248329  \n",
       "1                   1542248348  \n",
       "2                   1542249861  \n",
       "3                   1542252775  \n",
       "\n",
       "[4 rows x 91 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's have a look at the dataframe now and see how its looks like after we add the datetime variables\n",
    "train.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's remove the identifier columns so that we have only the features we need to use otherwise we will have problems.\n",
    "train = train.drop(columns='Identifier')\n",
    "test = test.drop(columns='Identifier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>TransactionStartTimeDay</th>\n",
       "      <th>TransactionStartTimeDayofweek</th>\n",
       "      <th>TransactionStartTimeDayofyear</th>\n",
       "      <th>TransactionStartTimeIs_month_end</th>\n",
       "      <th>TransactionStartTimeIs_month_start</th>\n",
       "      <th>TransactionStartTimeIs_quarter_end</th>\n",
       "      <th>TransactionStartTimeIs_quarter_start</th>\n",
       "      <th>TransactionStartTimeIs_year_end</th>\n",
       "      <th>TransactionStartTimeIs_year_start</th>\n",
       "      <th>TransactionStartTimeElapsed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542248329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542248348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542249861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1542252775</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 90 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "0          0          0          1          0          1          1   \n",
       "1          0          0          0          1          1          1   \n",
       "2          0          0          1          1          1          0   \n",
       "3          0          0          0          0          0          0   \n",
       "\n",
       "   BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  TransactionStartTimeDay  \\\n",
       "0          0          1          1          1  ...                       15   \n",
       "1          1          1          0          0  ...                       15   \n",
       "2          1          0          1          1  ...                       15   \n",
       "3          0          1          1          1  ...                       15   \n",
       "\n",
       "   TransactionStartTimeDayofweek  TransactionStartTimeDayofyear  \\\n",
       "0                              3                            319   \n",
       "1                              3                            319   \n",
       "2                              3                            319   \n",
       "3                              3                            319   \n",
       "\n",
       "   TransactionStartTimeIs_month_end  TransactionStartTimeIs_month_start  \\\n",
       "0                             False                               False   \n",
       "1                             False                               False   \n",
       "2                             False                               False   \n",
       "3                             False                               False   \n",
       "\n",
       "   TransactionStartTimeIs_quarter_end  TransactionStartTimeIs_quarter_start  \\\n",
       "0                               False                                 False   \n",
       "1                               False                                 False   \n",
       "2                               False                                 False   \n",
       "3                               False                                 False   \n",
       "\n",
       "   TransactionStartTimeIs_year_end  TransactionStartTimeIs_year_start  \\\n",
       "0                            False                              False   \n",
       "1                            False                              False   \n",
       "2                            False                              False   \n",
       "3                            False                              False   \n",
       "\n",
       "   TransactionStartTimeElapsed  \n",
       "0                   1542248329  \n",
       "1                   1542248348  \n",
       "2                   1542249861  \n",
       "3                   1542252775  \n",
       "\n",
       "[4 rows x 90 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plot graph of feature importances for better visualization\n",
    "#plt.figure(figsize=(7,7))\n",
    "#feat_importances = pd.Series(rm_clf.feature_importances_, index=train.columns)\n",
    "#feat_importances.nlargest(10).plot(kind='bar')\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((95662, 90), (45019, 90))"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's see the shapes of the data now\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Positive Correlations:\n",
      " SubscriptionId_2     0.033930\n",
      "ChannelId_2          0.034272\n",
      "ProductId_3          0.035686\n",
      "ProductId_5          0.045169\n",
      "ProductId_4          0.049997\n",
      "ProviderId_3         0.074962\n",
      "Value                0.566739\n",
      "FraudResult          1.000000\n",
      "BatchId_0                 NaN\n",
      "AccountId_0               NaN\n",
      "SubscriptionId_0          NaN\n",
      "CustomerId_0              NaN\n",
      "ProviderId_0              NaN\n",
      "ProductId_0               NaN\n",
      "ProductCategory_0         NaN\n",
      "Name: FraudResult, dtype: float64\n",
      "\n",
      "Most Negative Correlations:\n",
      " ProviderId_1                    -0.083548\n",
      "ProductId_1                     -0.049321\n",
      "PricingStrategy                 -0.033821\n",
      "SubscriptionId_1                -0.029144\n",
      "SubscriptionId_4                -0.028527\n",
      "AccountId_2                     -0.023884\n",
      "AccountId_10                    -0.021586\n",
      "CustomerId_7                    -0.014871\n",
      "SubscriptionId_11               -0.010745\n",
      "CustomerId_10                   -0.010288\n",
      "CustomerId_2                    -0.010247\n",
      "SubscriptionId_8                -0.009600\n",
      "SubscriptionId_7                -0.009546\n",
      "TransactionStartTimeDayofyear   -0.009241\n",
      "TransactionStartTimeMonth       -0.008887\n",
      "Name: FraudResult, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#let's attach the y column to find correlations and after remove\n",
    "train['FraudResult']= y\n",
    "# Find correlations with the target and sort\n",
    "correlations = train.corr()['FraudResult'].sort_values()\n",
    "\n",
    "# Display correlations\n",
    "print('Most Positive Correlations:\\n', correlations.tail(15))\n",
    "print('\\nMost Negative Correlations:\\n', correlations.head(15))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I really don't have much issue with correlation\n",
    "y = train['FraudResult']\n",
    "train = train.drop(columns='FraudResult')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a Validation set\n",
    "\n",
    "We are going to take the last 20,000 values and keep them for validation. We want to use this information to make sure we are checkin properly if our models are doing well on out of sample data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lets first mix the train and lable\n",
    "train['FraudResult']= y\n",
    "\n",
    "#choose the last 20k rows to be kept aside\n",
    "valid = train.tail(20000)\n",
    "\n",
    "#bread the validation set into X and y\n",
    "y_valid = valid['FraudResult']\n",
    "X_valid = valid.drop(columns='FraudResult')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BatchId_0</th>\n",
       "      <th>BatchId_1</th>\n",
       "      <th>BatchId_2</th>\n",
       "      <th>BatchId_3</th>\n",
       "      <th>BatchId_4</th>\n",
       "      <th>BatchId_5</th>\n",
       "      <th>BatchId_6</th>\n",
       "      <th>BatchId_7</th>\n",
       "      <th>BatchId_8</th>\n",
       "      <th>BatchId_9</th>\n",
       "      <th>...</th>\n",
       "      <th>TransactionStartTimeDayofweek</th>\n",
       "      <th>TransactionStartTimeDayofyear</th>\n",
       "      <th>TransactionStartTimeIs_month_end</th>\n",
       "      <th>TransactionStartTimeIs_month_start</th>\n",
       "      <th>TransactionStartTimeIs_quarter_end</th>\n",
       "      <th>TransactionStartTimeIs_quarter_start</th>\n",
       "      <th>TransactionStartTimeIs_year_end</th>\n",
       "      <th>TransactionStartTimeIs_year_start</th>\n",
       "      <th>TransactionStartTimeElapsed</th>\n",
       "      <th>FraudResult</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>75662</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1548656268</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75663</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1548656287</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75664</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1548656326</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75665</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1548656337</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       BatchId_0  BatchId_1  BatchId_2  BatchId_3  BatchId_4  BatchId_5  \\\n",
       "75662          0          0          0          0          0          1   \n",
       "75663          0          0          0          0          0          1   \n",
       "75664          0          0          0          1          0          1   \n",
       "75665          0          0          0          1          0          1   \n",
       "\n",
       "       BatchId_6  BatchId_7  BatchId_8  BatchId_9  ...  \\\n",
       "75662          1          0          0          1  ...   \n",
       "75663          0          0          0          1  ...   \n",
       "75664          1          0          1          1  ...   \n",
       "75665          0          0          1          0  ...   \n",
       "\n",
       "       TransactionStartTimeDayofweek  TransactionStartTimeDayofyear  \\\n",
       "75662                              0                             28   \n",
       "75663                              0                             28   \n",
       "75664                              0                             28   \n",
       "75665                              0                             28   \n",
       "\n",
       "       TransactionStartTimeIs_month_end  TransactionStartTimeIs_month_start  \\\n",
       "75662                             False                               False   \n",
       "75663                             False                               False   \n",
       "75664                             False                               False   \n",
       "75665                             False                               False   \n",
       "\n",
       "       TransactionStartTimeIs_quarter_end  \\\n",
       "75662                               False   \n",
       "75663                               False   \n",
       "75664                               False   \n",
       "75665                               False   \n",
       "\n",
       "       TransactionStartTimeIs_quarter_start  TransactionStartTimeIs_year_end  \\\n",
       "75662                                 False                            False   \n",
       "75663                                 False                            False   \n",
       "75664                                 False                            False   \n",
       "75665                                 False                            False   \n",
       "\n",
       "       TransactionStartTimeIs_year_start  TransactionStartTimeElapsed  \\\n",
       "75662                              False                   1548656268   \n",
       "75663                              False                   1548656287   \n",
       "75664                              False                   1548656326   \n",
       "75665                              False                   1548656337   \n",
       "\n",
       "       FraudResult  \n",
       "75662            0  \n",
       "75663            0  \n",
       "75664            0  \n",
       "75665            0  \n",
       "\n",
       "[4 rows x 91 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 91)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'FraudResult' in X_valid.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lets now take the ones for validation out of the major dataset\n",
    "\n",
    "train  = train.iloc[0:75662, ]\n",
    "\n",
    "y = train['FraudResult']\n",
    "train = train.drop(columns='FraudResult')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((75662, 90), (20000, 90))"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, X_valid.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imbalance in data:\n",
    "There is a problem we have with the target variable, we have more of one result than the other, that is we have less fraud cases than the where there was actually fraud.\n",
    "\n",
    "We have to handle this data in a different way."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Target variable count')"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEICAYAAAC0+DhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAF3FJREFUeJzt3X/Y3XV93/HnSyJKFSRImmKCwmouvBCnQoRYndtkhkBbg5fVQu2SeUXSDqi129UNnTNXQVpd3dBcs7hMUpKuimjLyBw0S1Hr3BrkRhgISHOLZUkkcGsgDFERfO+P8wk93N/7zn3ukOTckOfjus51vt/35/P9fj/fc905r/P9cU5SVUiS1O85wx6AJGnmMRwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEj7UJIvJfnVAfvuSPLGSdqWJBndt6OTBmc4aJ9K8kjf46dJftg3/64DPJbnJ6kk8w/UNqvqzVX1uQO1vZkiyVVJPjjscWjfmTXsAejZpapeuHs6yd8C76mqv9ybdSWZVVWP76ux7U9JngNQVT8d9likfcEjBx1QSd6Q5MYkDyX5bpLLksxqbbs/6f/zJN8Gvtnqv5hkS1vm40k2J/n1vnX+RpK7k+xM8t+TzGtNX23Pd7cjl7PHjeVnWv3lfbV57WhndpI5Sa5PMtbWfW2SY/r6bk5ycZIbgUeBl/SPLckrknylLTuWZF2Sw8e9JL+Q5Futz5okz5vkdTu2bf97Se5J8pt7eI1fkGR1kq1JdiX5q77X+O1J7myv5V8mWTDutZ/ft54njwZ2n+ZK8oG2L9t3HwkmeS/wduDfttfz85ONTc8choMOtJ8AFwIvBv4B8MvAe8b1+SXgFOC17c34c8DvAHOA77Y2ANr5/fe19cwFbgH+S2t+U3s+oapeWFX/tX8jVfUosAE4t698DrCxqh6k9+/jU8BLgeNb+2XjxvrrwDLgcGDHBPt7MfBzwKuAE4B/M679XODNre21wO+OX0GSQ4DrgP8NvARYAnwgyT+cYHsAq4FXAK8DjgI+CFSSVwFXAucDPwv8FXDt7uAYwMuAtDFcCHwqyQurajXwZ8Al7XV+x4Dr0wxmOOiAqqqvV9VNVfVEVX0b+DQw/k3u0qp6qKp+SO9N/6aq+mJV/QT4GPBgX9/fBD5cVX/T2n8PeGOSuQMO6TM8NRx+rdWoqvur6tqq+mFV7QL+YIKxfrqq7q6qn4w/BVZV36qqL1XVY1W1A/j4BMt/oqq+W1Vjbf3n0vVG4PlV9dG2rr8B/phekD1FkufSC6vfqqod7XX+n1X1ROt/TVV9paoeA36fXuAunPplAnpHR3/Q9vUaoICXT7GMnqG85qADKsmJwL8HTgYOo/c3+L/GddvaN/2S/vmq+mmS7X3tL6P3CfaTfbXHgfnArgGGtBFYl+TVwI+ABcB/a2M9HPgE8E+AI1v/w/Yw1qdI8pK2/C/QO7J4DnDfHpa/l97+jvcy4LgkD/XVDgEmupZzDL3X9NsTtL2kbQOAqnqivZbzgFsn248+Y+OuqTwKvHCyznpm88hBB9p/Br4B/HxVHUHvtEvG9en/qeD76L3RA09e+J3X174V+GdVdWTf47CqunnceibUjja+QO8T+6/R+2T9w9Z8Udv269pYF08x1vH+EPgBcFJb/j0TLH9s3/RL6Z02G28r8K1x+3h4Vb1tgr730QvHn5+g7bv0ggZ48nTVPGA78Bi9U34/09f/5/awb+P5887PMoaDDrTDgV1V9UiSVwLnTdF/A3BakrPaufF/Aczua/8U8MEkJwC0C8lvB6iqH9M7evh7U2zjM/ROuZzbpvvH+ijwUJKj6Z27n47DgUeAh5O8tI19vPcmOaat/yJ611fG+xpAkve1C8ezkvz9JCeP79jCbj3wiSRzkxyS5I0tCD4HvC3Jm9rpp4uA7wMj7YjgduBdbZm3Aq+fxr7ez9Svs55BDAcdaL8DvCfJI8AnmfjN8ElVdR+9N+3VwPfofZK/Hfhxa/8s8B+BP0/yML3TI2/pW8WHgM+3u3PeOslmvkrvNM2LeOqpmo8BR9N7A/0avYvC0/EhetcLdgHX0LtoO95VwJeBLW2//t34Du0N/yx6p6fuBcaAy5n8lM576Z1WuqWN/RIgVXUbsAL4T20dpwNL+66VXAj8Kr1rOmcDX5zGvq4BXtde56umsZxmqPif/eiZpB097AB+uar+etjjkZ6tPHLQjJfkzCQvSvJ8YBW9Uz03D3lY0rOa4aBngjcB3wEeoHcq5G3tVkxJ+4mnlSRJHR45SJI6nrFfgjv66KPruOOOG/YwJOkZ4+abb/5eVc0ZpO8zNhyOO+44RkZGhj0MSXrGSHLv1L16PK0kSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqeMZ+Q/rpOuV31w97CJqBbv7DZcMegjQjeOQgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpY8pwSHJCklv7Hg8neV+So5JsSrKlPc9u/ZNkdZLRJLclOblvXctb/y1JlvfVT0lye1tmdZLsn92VJA1iynCoqrur6jVV9RrgFOBR4BrgIuCGqloA3NDmAc4EFrTHSuBygCRHAauA04BTgVW7A6X1Oa9vuSX7ZO8kSXtluqeVTge+XVX3AkuBda2+Dji7TS8F1lfPZuDIJMcAZwCbqmpnVT0IbAKWtLYjqmpzVRWwvm9dkqQhmG44nAN8tk3Prar72vQOYG6bngds7VtmW6vtqb5tgnpHkpVJRpKMjI2NTXPokqRBDRwOSQ4F3gp8fnxb+8Rf+3BcE6qqNVW1sKoWzpkzZ39vTpIOWtM5cjgT+EZV3d/m72+nhGjPD7T6duDYvuXmt9qe6vMnqEuShmQ64XAuf3dKCWADsPuOo+XAtX31Ze2upUXArnb6aSOwOMnsdiF6MbCxtT2cZFG7S2lZ37okSUMw0H8TmuQFwFuA3+grfwS4OskK4F7gna1+HXAWMErvzqZ3A1TVziSXADe1fhdX1c42fT5wJXAYcH17SJKGZKBwqKofAC8eV/s+vbuXxvct4IJJ1rMWWDtBfQQ4aZCxSJL2P78hLUnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHQOFQ5Ijk3whybeS3JXk9UmOSrIpyZb2PLv1TZLVSUaT3Jbk5L71LG/9tyRZ3lc/JcntbZnVSbLvd1WSNKhBjxw+AfxFVb0CeDVwF3ARcENVLQBuaPMAZwIL2mMlcDlAkqOAVcBpwKnAqt2B0vqc17fckqe3W5Kkp2PKcEjyIuBNwBUAVfVYVT0ELAXWtW7rgLPb9FJgffVsBo5McgxwBrCpqnZW1YPAJmBJazuiqjZXVQHr+9YlSRqCQY4cjgfGgD9OckuSTyd5ATC3qu5rfXYAc9v0PGBr3/LbWm1P9W0T1DuSrEwykmRkbGxsgKFLkvbGIOEwCzgZuLyqXgv8gL87hQRA+8Rf+354T1VVa6pqYVUtnDNnzv7enCQdtAYJh23Atqq6sc1/gV5Y3N9OCdGeH2jt24Fj+5af32p7qs+foC5JGpIpw6GqdgBbk5zQSqcDdwIbgN13HC0Hrm3TG4Bl7a6lRcCudvppI7A4yex2IXoxsLG1PZxkUbtLaVnfuiRJQzBrwH6/BfxpkkOBe4B30wuWq5OsAO4F3tn6XgecBYwCj7a+VNXOJJcAN7V+F1fVzjZ9PnAlcBhwfXtIkoZkoHCoqluBhRM0nT5B3wIumGQ9a4G1E9RHgJMGGYskaf/zG9KSpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgYKhyR/m+T2JLcmGWm1o5JsSrKlPc9u9SRZnWQ0yW1JTu5bz/LWf0uS5X31U9r6R9uy2dc7Kkka3HSOHP5xVb2mqha2+YuAG6pqAXBDmwc4E1jQHiuBy6EXJsAq4DTgVGDV7kBpfc7rW27JXu+RJOlpezqnlZYC69r0OuDsvvr66tkMHJnkGOAMYFNV7ayqB4FNwJLWdkRVba6qAtb3rUuSNASDhkMB/yPJzUlWttrcqrqvTe8A5rbpecDWvmW3tdqe6tsmqHckWZlkJMnI2NjYgEOXJE3XrAH7vbGqtif5WWBTkm/1N1ZVJal9P7ynqqo1wBqAhQsX7vftSdLBaqAjh6ra3p4fAK6hd83g/nZKiPb8QOu+HTi2b/H5rban+vwJ6pKkIZkyHJK8IMnhu6eBxcA3gQ3A7juOlgPXtukNwLJ219IiYFc7/bQRWJxkdrsQvRjY2NoeTrKo3aW0rG9dkqQhGOS00lzgmnZ36SzgM1X1F0luAq5OsgK4F3hn638dcBYwCjwKvBugqnYmuQS4qfW7uKp2tunzgSuBw4Dr20OSNCRThkNV3QO8eoL694HTJ6gXcMEk61oLrJ2gPgKcNMB4JUkHgN+QliR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkjoHDIckhSW5J8sU2f3ySG5OMJvlckkNb/XltfrS1H9e3jve3+t1JzuirL2m10SQX7bvdkyTtjekcOfw2cFff/EeBy6rq5cCDwIpWXwE82OqXtX4kORE4B3glsAT4oxY4hwCfBM4ETgTObX0lSUMyUDgkmQ/8IvDpNh/gzcAXWpd1wNltemmbp7Wf3vovBa6qqh9X1XeAUeDU9hitqnuq6jHgqtZXkjQkgx45fBz4V8BP2/yLgYeq6vE2vw2Y16bnAVsBWvuu1v/J+rhlJqt3JFmZZCTJyNjY2IBDlyRN15ThkOSXgAeq6uYDMJ49qqo1VbWwqhbOmTNn2MORpGetWQP0eQPw1iRnAc8HjgA+ARyZZFY7OpgPbG/9twPHAtuSzAJeBHy/r75b/zKT1SVJQzDlkUNVvb+q5lfVcfQuKH+pqt4FfBn4ldZtOXBtm97Q5mntX6qqavVz2t1MxwMLgK8DNwEL2t1Ph7ZtbNgneydJ2iuDHDlM5l8DVyX5MHALcEWrXwH8SZJRYCe9N3uq6o4kVwN3Ao8DF1TVEwBJLgQ2AocAa6vqjqcxLknS0zStcKiqrwBfadP30LvTaHyfHwHvmGT5S4FLJ6hfB1w3nbFIkvYfvyEtSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdU4ZDkucn+XqS/5PkjiS/1+rHJ7kxyWiSzyU5tNWf1+ZHW/txfet6f6vfneSMvvqSVhtNctG+301J0nQMcuTwY+DNVfVq4DXAkiSLgI8Cl1XVy4EHgRWt/wrgwVa/rPUjyYnAOcArgSXAHyU5JMkhwCeBM4ETgXNbX0nSkEwZDtXzSJt9bnsU8GbgC62+Dji7TS9t87T205Ok1a+qqh9X1XeAUeDU9hitqnuq6jHgqtZXkjQkA11zaJ/wbwUeADYB3wYeqqrHW5dtwLw2PQ/YCtDadwEv7q+PW2ayuiRpSAYKh6p6oqpeA8yn90n/Fft1VJNIsjLJSJKRsbGxYQxBkg4K07pbqaoeAr4MvB44Msms1jQf2N6mtwPHArT2FwHf76+PW2ay+kTbX1NVC6tq4Zw5c6YzdEnSNAxyt9KcJEe26cOAtwB30QuJX2ndlgPXtukNbZ7W/qWqqlY/p93NdDywAPg6cBOwoN39dCi9i9Yb9sXOSZL2zqypu3AMsK7dVfQc4Oqq+mKSO4GrknwYuAW4ovW/AviTJKPATnpv9lTVHUmuBu4EHgcuqKonAJJcCGwEDgHWVtUd+2wPJUnTNmU4VNVtwGsnqN9D7/rD+PqPgHdMsq5LgUsnqF8HXDfAeCVJB4DfkJYkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUMWU4JDk2yZeT3JnkjiS/3epHJdmUZEt7nt3qSbI6yWiS25Kc3Leu5a3/liTL++qnJLm9LbM6SfbHzkqSBjPIkcPjwL+sqhOBRcAFSU4ELgJuqKoFwA1tHuBMYEF7rAQuh16YAKuA04BTgVW7A6X1Oa9vuSVPf9ckSXtrynCoqvuq6htt+v8BdwHzgKXAutZtHXB2m14KrK+ezcCRSY4BzgA2VdXOqnoQ2AQsaW1HVNXmqipgfd+6JElDMK1rDkmOA14L3AjMrar7WtMOYG6bngds7VtsW6vtqb5tgvpE21+ZZCTJyNjY2HSGLkmahoHDIckLgT8D3ldVD/e3tU/8tY/H1lFVa6pqYVUtnDNnzv7enCQdtAYKhyTPpRcMf1pVf97K97dTQrTnB1p9O3Bs3+LzW21P9fkT1CVJQzLI3UoBrgDuqqr/0Ne0Adh9x9Fy4Nq++rJ219IiYFc7/bQRWJxkdrsQvRjY2NoeTrKobWtZ37okSUMwa4A+bwD+KXB7kltb7QPAR4Crk6wA7gXe2dquA84CRoFHgXcDVNXOJJcAN7V+F1fVzjZ9PnAlcBhwfXtIkoZkynCoqq8Bk33v4PQJ+hdwwSTrWgusnaA+Apw01VgkSQeG35CWJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqSOKcMhydokDyT5Zl/tqCSbkmxpz7NbPUlWJxlNcluSk/uWWd76b0myvK9+SpLb2zKrk0z2X5JKkg6QQY4crgSWjKtdBNxQVQuAG9o8wJnAgvZYCVwOvTABVgGnAacCq3YHSutzXt9y47clSTrApgyHqvoqsHNceSmwrk2vA87uq6+vns3AkUmOAc4ANlXVzqp6ENgELGltR1TV5qoqYH3fuiRJQ7K31xzmVtV9bXoHMLdNzwO29vXb1mp7qm+boD6hJCuTjCQZGRsb28uhS5Km8rQvSLdP/LUPxjLIttZU1cKqWjhnzpwDsUlJOijtbTjc304J0Z4faPXtwLF9/ea32p7q8yeoS5KGaG/DYQOw+46j5cC1ffVl7a6lRcCudvppI7A4yex2IXoxsLG1PZxkUbtLaVnfuiRJQzJrqg5JPgv8I+DoJNvo3XX0EeDqJCuAe4F3tu7XAWcBo8CjwLsBqmpnkkuAm1q/i6tq90Xu8+ndEXUYcH17SJKGaMpwqKpzJ2k6fYK+BVwwyXrWAmsnqI8AJ001DknSgeM3pCVJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1zJhwSLIkyd1JRpNcNOzxSNLBbEaEQ5JDgE8CZwInAucmOXG4o5Kkg9esYQ+gORUYrap7AJJcBSwF7hzqqKQh+b8Xv2rYQ9AM9NIP3X7AtjVTwmEesLVvfhtw2vhOSVYCK9vsI0nuPgBjOxgcDXxv2IOYCfKx5cMegrr8+9xtVZ7uGl42aMeZEg4Dqao1wJphj+PZJslIVS0c9jikifj3ORwz4poDsB04tm9+fqtJkoZgpoTDTcCCJMcnORQ4B9gw5DFJ0kFrRpxWqqrHk1wIbAQOAdZW1R1DHtbBxFN1msn8+xyCVNWwxyBJmmFmymklSdIMYjhIkjoMh4OcP1uimSrJ2iQPJPnmsMdyMDIcDmL+bIlmuCuBJcMexMHKcDi4PfmzJVX1GLD7Z0ukoauqrwI7hz2Og5XhcHCb6GdL5g1pLJJmEMNBktRhOBzc/NkSSRMyHA5u/myJpAkZDgexqnoc2P2zJXcBV/uzJZopknwW+GvghCTbkqwY9pgOJv58hiSpwyMHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLU8f8Bh4cyt2WtBBIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#i want to visualize the imbalance in the target variable\n",
    "x=y.value_counts().values\n",
    "sns.barplot([0,1],x)\n",
    "plt.title('Target variable count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Smote: Oversampling data:\n",
    "\n",
    "`SMOTE (Synthetic Minority Oversampling TEchnique)` consists of synthesizing elements for the minority class, based on those that already exist. It works randomly picking a point from the minority class and computing the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60529, 90), (15133, 90), (60529,), (15133,))"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#First we are going to separate the target variable from the rest of the observations and split the data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x = train\n",
    "\n",
    "#split the data to prevent overfitting\n",
    "X_train,X_test,y_train,y_test=train_test_split(x,y,test_size=.2,random_state=1)\n",
    "\n",
    "#let's see the shape of the data\n",
    "X_train.shape,X_test.shape,y_train.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%pip install imblearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#first we are going to filter out the different results from the label in the main label\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "smote = SMOTE(random_state=27,ratio='minority')\n",
    "X_train, y_train = smote.fit_sample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((120846, 90), (120846,))"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's see the shape\n",
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After OverSampling, the shape of train_X: (120846, 90)\n",
      "After OverSampling, the shape of train_y: (120846,) \n",
      "\n",
      "After OverSampling, counts of label '1': 60423\n",
      "After OverSampling, counts of label '0': 60423\n"
     ]
    }
   ],
   "source": [
    "#let's check and see the number of os and ones\n",
    "print('After OverSampling, the shape of train_X: {}'.format(X_train.shape))\n",
    "print('After OverSampling, the shape of train_y: {} \\n'.format(y_train.shape))\n",
    "\n",
    "print(\"After OverSampling, counts of label '1': {}\".format(sum(y_train==1)))\n",
    "print(\"After OverSampling, counts of label '0': {}\".format(sum(y_train==0)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scaling the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "scaler = StandardScaler()\n",
    "maxscaler = MinMaxScaler()\n",
    "\n",
    "#let's scale the data with Standard Scaler\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test  = scaler.fit_transform(X_test)\n",
    "\n",
    "#let's make all the data be in the range of 0-1 so the scales are balanced.\n",
    "X_train = maxscaler.fit_transform(X_train)\n",
    "X_test = maxscaler.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.5/site-packages/sklearn/ensemble/forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9999751750161362"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#now we have something to put in the model\n",
    "m = RandomForestClassifier(n_jobs=-1)\n",
    "m.fit(X_train, y_train)\n",
    "m.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Engineering\n",
    "\n",
    "### Feature Importances and Selection\n",
    "\n",
    "We want to know the best features in our model and how to harness them, this is important, and to find our more please read this [Article](https://towardsdatascience.com/feature-selection-techniques-in-machine-learning-with-python-f24e7da3f36e)\n",
    "\n",
    "- What we want to do is use those features instead of the many useless, because I have noticed I get a good score on my F1 here but on the leader board it goes low, so I am overfitting\n",
    "\n",
    "- [Great Article Here on Feature Selection and Importance methods](https://www.kaggle.com/kanncaa1/feature-selection-and-data-visualization)\n",
    "- [Will Koerhsen on Feature Importances e.t.c](https://www.kaggle.com/willkoehrsen/introduction-to-feature-selection)\n",
    "- [Manual Feature Engineering](https://www.kaggle.com/willkoehrsen/introduction-to-manual-feature-engineering)\n",
    "- [Part Two of Feature Engineering](https://www.kaggle.com/willkoehrsen/introduction-to-manual-feature-engineering-p2)\n",
    "\n",
    "### Univariate feature selection and random forest classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from sklearn.feature_selection import SelectKBest\n",
    "#from sklearn.feature_selection import chi2\n",
    "# find best scored 10 features\n",
    "#select_feature = SelectKBest(chi2, k=10).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's print the selected features\n",
    "#print('Score list:', select_feature.scores_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's take those features and transform the data we have\n",
    "#x_train = select_feature.transform(X_train)\n",
    "#x_test = select_feature.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Score Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I want to write a function that I will call to evaluate my model, I am going to evaluate based on ROC_AUC and score.\n",
    "\n",
    "#when you build a model you want to know how it is functioning. And there are functions that are pre_built in scikit learn \n",
    "#to help you evaluate the model, so I built these in the functions below\n",
    "\n",
    "def print_score(m,md):\n",
    "    \n",
    "    # Used to ignore warnings generated from StackingCVClassifier\n",
    "    import warnings\n",
    "    warnings.simplefilter('ignore')\n",
    "\n",
    "    #Call the classification metrics we want to use\n",
    "    from sklearn.metrics import roc_auc_score,auc,f1_score,roc_curve\n",
    "    \n",
    "    \n",
    "    false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, m)\n",
    "    roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "    \n",
    "    f1 = f1_score(y_test,m)\n",
    "    roc = roc_auc_score(y_test,m)\n",
    "    #oob = md.oob_score_ #I added the oob score\n",
    "    \n",
    "    #let's see how the model does on validation data\n",
    "    pred = md.predict(X_valid)\n",
    "    f2 = f1_score(y_valid,pred)\n",
    "    \n",
    "    pred_3 = md.predict(X_train)\n",
    "    f3 = f1_score(y_train,pred_3)\n",
    "    \n",
    "    res = [\"roc_auc-->\",roc_auc,\"F1_score-->\",f1,\"roc_auc_score-->\",roc]\n",
    "    print(res)\n",
    "    print('\\n')\n",
    "    \n",
    "    print(\"Model Performance on out of sample data\")\n",
    "    \n",
    "    valid = ['Valid_data_F1_score-->',f2]\n",
    "    \n",
    "    print(valid)\n",
    "    \n",
    "    print('\\n')\n",
    "    \n",
    "    print(\"Model Performance on train data\")\n",
    "    \n",
    "    train_ = ['Train_data_F1_score-->',f3]\n",
    "    \n",
    "    print(train_)\n",
    "    \n",
    "    \n",
    "    print('\\n')\n",
    "    print('--'*10)\n",
    "    print('Scores on the Validation Data according to the Confusion_matrix')\n",
    "    \n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    tn, fp, fn, tp = confusion_matrix(y_valid,pred).ravel()\n",
    "    \n",
    "    # Error rate : \n",
    "    err_rate = (fp + fn) / (tp + tn + fn + fp)\n",
    "    print(\"Error rate  : \", err_rate)\n",
    "\n",
    "    # Accuracy : \n",
    "    acc_ = (tp + tn) / (tp + tn + fn + fp)\n",
    "    print(\"Accuracy  : \", acc_)\n",
    "    \n",
    "    \n",
    "    #precision\n",
    "    preci_ = tp / (tp + fp)\n",
    "    print(\"Precision  : \", preci_)\n",
    "\n",
    "    # Sensitivity : \n",
    "    sens_ = tp / (tp + fn)\n",
    "    print(\"Recall  : \", sens_)\n",
    "\n",
    "    # Specificity \n",
    "    sp_ = tn / (tn + fp)\n",
    "    print(\"Specificity  : \", sens_)\n",
    "\n",
    "    # False positive rate (FPR)\n",
    "    FPR = fp / (tn + fp)\n",
    "    print(\"False positive rate  : \", FPR)\n",
    "    \n",
    "    # False positive rate (FPR)\n",
    "    TPR = tp / (tp + fn)\n",
    "    print(\"True positive rate  : \", TPR)\n",
    "    \n",
    "    print('--'*10)\n",
    "    \n",
    "    \n",
    "    print('\\n')\n",
    "    print('Drawing the Confusion Matrix on Out_of_sample_data')\n",
    "    #from sklearn.metrics import confusion_matrix\n",
    "    \n",
    "    plt.figure(figsize=(7,7))\n",
    "    cm_2 = confusion_matrix(y_valid,pred)\n",
    "    sns.heatmap(cm_2,annot=True,fmt=\"d\")\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 3s, sys: 12 ms, total: 2min 3s\n",
      "Wall time: 17 s\n",
      "['roc_auc-->', 0.9277773576344248, 'F1_score-->', 0.5714285714285714, 'roc_auc_score-->', 0.9277773576344248]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.003026951510565225]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 0.9998924277404033]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.85635\n",
      "Accuracy  :  0.14365\n",
      "Precision  :  0.0015193128031321219\n",
      "Recall  :  0.3939393939393939\n",
      "Specificity  :  0.3939393939393939\n",
      "False positive rate  :  0.8571786896759306\n",
      "True positive rate  :  0.3939393939393939\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Out_of_sample_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAG9RJREFUeJzt3Xu4VmWZ+PHvzUaw0AIBUcFMkzQ76JSRmSlKClqKU9alTeqYDZbajL8OHms8pKW/Mid/qc1WKc0Dko6BphKiguYBaEBFxZFsTEhFRJw8JMF+5o+9wJ0/cAu97Pdmr++Ha1287/OutfbzXhcX977vda9nRSkFSZKarUezJyBJEhiQJElJGJAkSSkYkCRJKRiQJEkpGJAkSSkYkCRJKRiQJEkpGJAkSSn0XNc/YPiQT7gUhLrM5NmtzZ6CamaDAdtEo871l0WPN+z/y0bOq6uYIUmSUljnGZIk6U1qW97sGTSVGZIkKQUzJEnKorQ1ewZNZUCSpCza6h2QLNlJklIwQ5KkJIolO0lSCpbsJElqPjMkScrCkp0kKQVvjJUkqfnMkCQpC0t2kqQU7LKTJKn5zJAkKQlvjJUk5WDJTpKk5jNDkqQsLNlJklLwxlhJkprPDEmSsrBkJ0lKwS47SZKazwxJkrKwZCdJSsGSnSRJzWeGJElJlFLv+5AMSJKURc2vIVmykySlYIYkSVnUvKnBgCRJWdS8ZGdAkqQsXFxVkqTmM0OSpCws2UmSUqh5U4MlO0lSCmZIkpSFJTtJUgqW7CRJaj4zJEnKouYZkgFJkpKo+2rfluwkqYYiYmxELIyIOR3GTouIBRExu9r26/DZSRExLyIejYiRHcZHVWPzIuLEDuNbR8R91fg1EdGrszkZkCQpi7a2xm2d+xkwahXj55VSdqq2mwAiYgfgYOC91TEXRkRLRLQAFwD7AjsAh1T7ApxTnWtb4HngyM4mZECSpCxKW+O2zn5UKdOAxW9yZqOBcaWUV0spvwfmAcOqbV4p5fFSylJgHDA6IgLYC7i2Ov4y4MDOfogBSZLU0bER8UBV0utXjQ0Gnuywz/xqbHXj/YElpZRlrxt/QwYkScqigSW7iBgTETM7bGPexAwuAt4F7AQ8BZy7Tr/v69hlJ0lZNHClhlJKK9C6hsc8s+J1RFwM3Fi9XQBs2WHXIdUYqxl/DugbET2rLKnj/qtlhiRJAiAiNu/w9u+BFR14E4GDI6J3RGwNDAWmAzOAoVVHXS/aGx8mllIKcDtwUHX84cCEzn6+GZIkZdGFN8ZGxNXAcGBARMwHTgWGR8ROQAH+GzgKoJTyUESMBx4GlgHHlOqmqYg4FpgEtABjSykPVT/iBGBcRJwJzAIu7WxOBiRJyqILF1ctpRyyiuHVBo1SylnAWasYvwm4aRXjj9PehfemWbKTJKVghiRJWbiWnSQphZoHJEt2kqQUzJAkKQufGCtJSsGSnSRJzWeGJElZWLKTJKVgyU6SpOYzQ5KkLCzZSZJSsGQnSVLzmSFJUhY1z5AMSJKURSnNnkFTWbKTJKVghiRJWViykySlUPOAZMlOkpSCGZIkZeGNsZKkFCzZSZLUfGZIkpRFze9DMiBJUhaW7CRJaj4zJEnKouYZkgFJkrKoedu3JTtJUgpmSJKURGmzy06SlEHNryFZspMkpWCGJElZ1LypwYAkSVnU/BqSJTtJUgpmSJKURc2bGgxIkpSFAUmSlELNV/v2GpIkKQUzJEnKwpKdusLAzQdy8o9OoN+AfpRSuPGqX3Hdpdez7Q7v4mtnH0ev3huwfNlyzjvlfObOfnTlcdvtuB0XTjifM445k6m/upOddt2RY0/9ysrP3/Gud3DGMWdy16S7m/G1lMy3vvtDpv1mOpv068svr/gJAF//9vf47z/MB+BPL77IxhttxHWXXQDAxZdfw3/cOImWHj046f98hY995EMAXD7ueq674RYigqHveidnnvw1evfuxWFf+QYvvfwKAIufX8L7d9iO88/+1yZ8026q5m3fBqQusnz5ci484yc8Nmceb+nzFlpvvoiZ037LUaf8Ez8773Km3z6Dj+w1jC+fMobjPvt1AHr06MFRJ3+JGdNmrjzP7Lvv50sjvwzAxn035sq7LmPG1N825TspnwP325vPf+YATv7OD1aOnfudk1a+/v7/u5iN+rwVgN/9/glunjKVCVf8hIWLFvOlfzmJX427hEWLn+fKaycw4cp/Z8Pevfn6t7/LzbdO5cBP7s3lF7123uNOPpM9P75L1305dXudXkOKiO0j4oSIOL/aToiI93TF5LqTxQsX89iceQC88tIrPPHYHxiw2QBKgT4b9QGgz8Z9WPTMcyuP+fQRBzLtpjtZsmjJKs+5xyd3577bZ/Dqn19d919A64Wdd3o/b3/bxqv8rJTCLbdNY7+9hwNw2533su+IPejVqxdDttiMdwzZggcf+S8Ali1fzquvLmXZsuW88udXGThgk78614svvcT0/7yfEbt/dJ1+n9opbY3b1kNvGJAi4gRgHBDA9GoL4OqIOHHdT6972mzIIIa+b1semTWXH592IV/+1hjGT7+Kr3z7KC7+3iUADNisP7vt+zEmXH7Das+z1wHDue2Xt3XVtLWe++39c+jfrx9bbTkYgIXPPsdmgwau/HzQpgNY+OwiBg0cwD8e8hk+8enD2HP059m4z1tXlvJWmDLtHj7yoR3ZqE+fLv0O3V5bady2HuosQzoS+HAp5exSyhXVdjYwrPpslSJiTETMjIiZf3xpQSPnu957y1s35PTWU/nxaRfy8osvM/qw/bng9Iv43LDPc8FpF3H8D74BwLGnHU3rdy+hrKYNdJNNN2Gb7bdm+tSZq/xcer2bJt/Bfnvv0el+L/zPn7j9znuZ9IufctuEK3nlz69yw6S//sXn5lunst8nhq+jmaquOgtIbcAWqxjfvPpslUopraWUnUspO2/RZ/DfMr9upaVnC6e3nsat10/hzpvvAmDkQfsw7aY7Abjjxqlsv9N2AGz3gXfzrxecwrh7rmCPT+7OcWf9M7uN3HXlufbcfw/uvOU3LF+2vOu/iNY7y5Yt59apdzNqxO4rxzYd2J+nn3l25ftnFi5i04EDuHfmbAZvMYhN+vVlg549GbHHrsx+8OGV+z2/5AUefPhRdt91WJd+hzoobW0N29ZHnTU1HAdMiYjHgCersXcA2wLHrsuJdUfH/+Ab/GHeE/zi4utWjj33zCJ2+uiOzL7nfj74sb9j/u/bM8pDdj105T4n/vCb3DPl3r/qpBsxei9az76k6yav9dq9M2exzVZD2GzT10p0e+62C8effg6HH/z3LFy0mD/M/yPvf8+76RHBA3Pm8sqf/8yGvXtz38zZvHf7oSuP+/Xtd7HHrsPo3btXM75K97aeltoa5Q0DUinlloh4N+0luhWpzgJgRinFX83XwPs//D5GHrQ3v3vkcS6Z1N6Oe/E5Y/nB8edx7OlH09KzhaWvLuXcE87r9FybDRnEwC0Gcv89D6zraWs9881Tz2bGrAdYsuR/GHHgFzj6yEP5zP4jufnWqez7uhLbtttsxci9Ps4B/3AUPVtaOOVrR9PS0sIH3rs9e++5G5874qu0tLSw/bvfxWdH77vyuJunTOVLX/hcF38z1UGs7hpFowwf8ol6h3x1qcmzW5s9BdXMBgO2iUad66Uzv9Cw/y/7fOuKhs2rq3gfkiRlUfOSnWvZSZJSMEOSpCzW0+64RjEgSVIWluwkSWo+MyRJymI9XYOuUQxIkpSFJTtJkprPDEmSklhf16BrFAOSJGVhyU6SVDcRMTYiFkbEnA5j34+IuRHxQERcHxF9O3x2UkTMi4hHI2Jkh/FR1di8js/Ji4itI+K+avyaiOh0NV4DkiRl0bUP6PsZMOp1Y5OB95VSPgD8F3ASQETsABwMvLc65sKIaImIFuACYF9gB+CQal+Ac4DzSinbAs/zBs/QW8GAJElZdOEjzEsp04DFrxv7dSllWfX2XmBI9Xo0MK6U8mop5ffAPNqfAjEMmFdKebyUspT2J4yPjogA9gKurY6/DDiwszkZkCRJq/JF4Obq9WBeeyYewPxqbHXj/YElHYLbivE3ZFODJGXRwKaGiBgDjOkw1FpKeVPPZ4mIU4BlwJUNm9CbYECSpCRKAwNSFXzW+AFhEfGPwKeAEeW1B+YtALbssNuQaozVjD8H9I2InlWW1HH/1bJkJ0kC2jvmgOOBA0opL3f4aCJwcET0joitgaHAdGAGMLTqqOtFe+PDxCqQ3Q4cVB1/ODChs59vhiRJWXThfUgRcTUwHBgQEfOBU2nvqusNTG7vS+DeUsqXSykPRcR44GHaS3nHlFKWV+c5FpgEtABjSykPVT/iBGBcRJwJzAIu7WxOBiRJyqILV2oopRyyiuHVBo1SylnAWasYvwm4aRXjj9PehfemWbKTJKVghiRJWdR86SADkiRlUfOAZMlOkpSCGZIkJfHabT/1ZECSpCws2UmS1HxmSJKURc0zJAOSJCXRyLXs1keW7CRJKZghSVIWNc+QDEiSlEXXLWWXkiU7SVIKZkiSlETdmxoMSJKURc0DkiU7SVIKZkiSlEXNmxoMSJKURN2vIVmykySlYIYkSVlYspMkZWDJTpKkBMyQJCkLS3aSpAyKAUmSlELNA5LXkCRJKZghSVISluwkSTnUPCBZspMkpWCGJElJWLKTJKVQ94BkyU6SlIIZkiQlUfcMyYAkSVmUaPYMmsqSnSQpBTMkSUrCkp0kKYXSZslOkqSmM0OSpCQs2UmSUih22UmS1HxmSJKUhCU7SVIKdtlJkpSAGZIkJVFKs2fQXAYkSUrCkp0kSQmYIUlSEnXPkAxIkpRE3a8hWbKTJKVghiRJSViykySl4Fp2kiQlYIYkSUm4lp0kKYU2S3aSJDWfGZIkJVH3pgYDkiQlUfe2b0t2klRTEfEvETEnIh6KiOOqsU0iYnJEPFb93a8aj4g4PyLmRcQDEfHBDuc5vNr/sYg4fG3nY0CSpCRKadzWmYh4H/BPwDBgR+BTEbEtcCIwpZQyFJhSvQfYFxhabWOAi6rzbAKcCnykOtepK4LYmjIgSVISpS0atr0J7wHuK6W8XEpZBkwFPg2MBi6r9rkMOLB6PRq4vLS7F+gbEZsDI4HJpZTFpZTngcnAqLX5/gYkSaqnOcDHI6J/RLwV2A/YEhhUSnmq2udpYFD1ejDwZIfj51djqxtfYzY1SFISjbwPKSLG0F5aW6G1lNK64k0p5ZGIOAf4NfASMBtY3vEcpZQSEV22BrkBSZKSaGTbdxV8WjvZ51LgUoCI+C7t2c0zEbF5KeWpqiS3sNp9Ae0Z1ApDqrEFwPDXjd+xNnO2ZCdJNRURm1Z/v4P260dXAROBFZ1yhwMTqtcTgcOqbrtdgBeq0t4kYJ+I6Fc1M+xTja0xMyRJSqIJD+i7LiL6A38BjimlLImIs4HxEXEk8ATwuWrfm2i/zjQPeBk4on3OZXFEfAeYUe13Rill8dpMxoAkSUl09Vp2pZSPr2LsOWDEKsYLcMxqzjMWGPu3zseSnSQpBTMkSUrCtewkSSk04RpSKpbsJEkprPMM6a6Fj6zrHyGt9JYt/r9rtNI6tWzpgoadq+4P6LNkJ0lJ1P0akiU7SVIKZkiSlIQlO0lSCjVvsjMgSVIWdc+QvIYkSUrBDEmSkqh7l50BSZKSaGv2BJrMkp0kKQUzJElKomDJTpKUQFvN+74t2UmSUjBDkqQk2izZSZIyqPs1JEt2kqQUzJAkKYm634dkQJKkJCzZSZKUgBmSJCVhyU6SlELdA5IlO0lSCmZIkpRE3ZsaDEiSlERbveORJTtJUg5mSJKUhGvZSZJSqPnTJyzZSZJyMEOSpCTqfh+SAUmSkmiLel9DsmQnSUrBDEmSkqh7U4MBSZKSqPs1JEt2kqQUzJAkKYm6Lx1kQJKkJOq+UoMlO0lSCmZIkpSEXXaSpBTqfg3Jkp0kKQUzJElKou73IRmQJCmJul9DsmQnSUrBDEmSkqh7U4MBSZKSqPs1JEt2kqQUzJAkKYm6Z0gGJElKotT8GpIlO0lSCmZIkpSEJTtJUgp1D0iW7CRJKZghSVISdV86yIAkSUnUfaUGS3aSVFMR0Tciro2IuRHxSER8NCI2iYjJEfFY9Xe/at+IiPMjYl5EPBARH+xwnsOr/R+LiMPXdj4GJElKoq2B25v0I+CWUsr2wI7AI8CJwJRSylBgSvUeYF9gaLWNAS4CiIhNgFOBjwDDgFNXBLE1ZUCSpCS6MiBFxNuB3YFLAUopS0spS4DRwGXVbpcBB1avRwOXl3b3An0jYnNgJDC5lLK4lPI8MBkYtTbf34AkSfW0NfAs8NOImBURl0REH2BQKeWpap+ngUHV68HAkx2On1+NrW58jRmQJCmJ0sAtIsZExMwO25jX/biewAeBi0opfwe8xGvlufb5lLLidF3CLjtJSqKRXXallFag9Q12mQ/ML6XcV72/lvaA9ExEbF5KeaoqyS2sPl8AbNnh+CHV2AJg+OvG71ibOZshSVISXXkNqZTyNPBkRGxXDY0AHgYmAis65Q4HJlSvJwKHVd12uwAvVKW9ScA+EdGvambYpxpbY2ZIklRfXwWujIhewOPAEbQnKuMj4kjgCeBz1b43AfsB84CXq30ppSyOiO8AM6r9ziilLF6byRiQJCmJrl6poZQyG9h5FR+NWMW+BThmNecZC4z9W+djQJKkJNpqvniQ15AkSSmYIUlSEnV//IQBSZKSqHfBzpKdJCkJMyRJSsKSnSQpBZ+HJElSAmZIkpRE3e9DMiBJUhL1DkeW7CRJSZghSVISdtlJklKo+zUkS3aSpBTMkCQpiXrnRwYkSUqj7teQLNlJklIwQ5KkJOre1GBAkqQk6h2OLNlJkpIwQ5KkJOre1GBAkqQkSs2LdpbsJEkpmCFJUhKW7CRJKdS97duSnSQpBTMkSUqi3vmRAUmS0rBkpzR69OjBjOmTmHD9ZQC8851bcvddNzD34bu46sqL2GCDDZo8Q3UHQ4Zswa2//gUP3H8798++ja8ee+TKz445+gjmPDiV+2ffxtnfO6WJs1QdmSEl8s9f/RJz5z7G2zbeGIDvffcU/u38ixk/fiIX/PhsvnjEIfx76+VNnqXWd8uWLeObx5/OrNlz2GijPky/7xZunTKNQZsO5ID9R/LBD+3N0qVLGTiwf7OnWjt177IzQ0pi8ODN2W/fEYwde/XKsT2Hf4zrrvsVAD//+S8YfcDIZk1P3cjTTy9k1uw5ALz44kvMnfsYg7fYjKOOOoz/+/0LWLp0KQDPPvtcM6dZS6WBf9ZHax2QIuKIRk6k7n547umceNKZtLW1/47Uv38/lix5geXLlwMwf8FTbDF4s2ZOUd3QVlsNYacd38d902cxdOg27LbbMO6+6wZuu/Vadv7Qjs2enmrmb8mQTm/YLGruk/t9goULF/Gfsx5s9lRUI336vJXx11zM175xKn/604v07NlCv3592XW3/TnhxDO5+qqfNHuKtdPWwG199IbXkCLigdV9BAx6g+PGAGMAouXt9OjRZ60nWAe77roz+39qH/YdtRcbbtibt71tY8774Rn07ft2WlpaWL58OUMGb84fFzzd7Kmqm+jZsye/uOZirr76en75y5sBWDD/qZWvZ8ycTVtbGwMGbMKiRYubOdVaWV9LbY3SWYY0CDgM2H8V22oLzKWU1lLKzqWUnQ1GnTvlW2fzzm12Ztt378I/fOFobr/9Nxx2+Fe5Y+rdfOYznwTg0EM/y8Qbft3kmaq7uLj1XB6ZO49/+1HryrEJEycxfPiuAAwdug29evUyGKlLddZldyOwUSll9us/iIg71smMtNJJJ5/FVVdcyBmnHc/s+x9i7E+v7vwgqRMf2/XDHPqFg3jgwYeZOaP9l5xvf/tsfvqzcVxy8bnMnjWFpUv/whePPK7JM62f9bXU1ihRyrpNEXv2GlzvHFRSt7Zs6YJo1LkO3erTDfv/8udP/EfD5tVVbPuWJKXgjbGSlETdy0kGJElKwrXsJElKwAxJkpKo+31IBiRJSqLubd+W7CRJKZghSVISdW9qMCBJUhJ1v4ZkyU6SlIIZkiQlUfemBgOSJCWxrtcWzc6SnSQpBTMkSUrCLjtJUgpeQ5IkpWDbtyRJCZghSVISXkOSJKVg27ckSQmYIUlSEnXvsjNDkqQkSgP/dCYiNoyI6RFxf0Q8FBGnV+NbR8R9ETEvIq6JiF7VeO/q/bzq83d2ONdJ1fijETFybb+/AUmS6ulVYK9Syo7ATsCoiNgFOAc4r5SyLfA8cGS1/5HA89X4edV+RMQOwMHAe4FRwIUR0bI2EzIgSVISbZSGbZ0p7V6s3m5QbQXYC7i2Gr8MOLB6Pbp6T/X5iIiIanxcKeXVUsrvgXnAsLX5/gYkSUqilNKwLSLGRMTMDtuY1/+8iGiJiNnAQmAy8DtgSSllWbXLfGBw9Xow8GQ1z2XAC0D/juOrOGaN2NQgSd1QKaUVaO1kn+XAThHRF7ge2L4r5rY6BiRJSqJZN8aWUpZExO3AR4G+EdGzyoKGAAuq3RYAWwLzI6In8HbguQ7jK3Q8Zo1YspOkJLq4y25glRkREW8B9gYeAW4HDqp2OxyYUL2eWL2n+vy20n4n70Tg4KoLb2tgKDB9bb6/GZIk1dPmwGVVR1wPYHwp5caIeBgYFxFnArOAS6v9LwV+HhHzgMW0d9ZRSnkoIsYDDwPLgGOqUuAai3W9VEXPXoPrvRaGpG5t2dIF0ahz7T54RMP+v5y2YErD5tVVzJAkKYm6//buNSRJUgpmSJKUhI+fkCSlUPeAZMlOkpSCGZIkJVH3B/QZkCQpCUt2kiQlYIYkSUm8mSV/ujMDkiQlUfdrSJbsJEkpmCFJUhJ1b2owIElSEpbsJElKwAxJkpKwZCdJSqHubd+W7CRJKZghSVISbTVvajAgSVISluwkSUrADEmSkrBkJ0lKwZKdJEkJmCFJUhKW7CRJKViykyQpATMkSUrCkp0kKQVLdpIkJWCGJElJlNLW7Ck0lQFJkpKo+/OQLNlJklIwQ5KkJIpddpKkDCzZSZKUgBmSJCVhyU6SlELdV2qwZCdJSsEMSZKSqPvSQQYkSUrCa0iSpBRs+5YkKQEzJElKwpKdJCkF274lSUrADEmSkrBkJ0lKwS47SZISMEOSpCQs2UmSUrDLTpKkBMyQJCkJF1eVJKVgyU6SpATMkCQpCbvsJEkp1P0akiU7SVIKZkiSlIQlO0lSCnUPSJbsJEkpmCFJUhL1zo8g6p4iZhURY0oprc2eh+rDf3NqNkt2eY1p9gRUO/6bU1MZkCRJKRiQJEkpGJDyspavrua/OTWVTQ2SpBTMkCRJKRiQEoqIURHxaETMi4gTmz0fdV8RMTYiFkbEnGbPRTIgJRMRLcAFwL7ADsAhEbFDc2elbuxnwKhmT0ICA1JGw4B5pZTHSylLgXHA6CbPSd1UKWUasLjZ85DAgJTRYODJDu/nV2OS1K0ZkCRJKRiQ8lkAbNnh/ZBqTJK6NQNSPjOAoRGxdUT0Ag4GJjZ5TpK0zhmQkimlLAOOBSYBjwDjSykPNXdW6q4i4mrgHmC7iJgfEUc2e06qL1dqkCSlYIYkSUrBgCRJSsGAJElKwYAkSUrBgCRJSsGAJElKwYAkSUrBgCRJSuF/AcZDb4aSxi37AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ml = RandomForestClassifier(n_jobs=-1, n_estimators=60, min_samples_leaf=3, max_features=0.5,bootstrap=True,oob_score =True)\n",
    "%time ml.fit(X_train, y_train)\n",
    "pred_d = ml.predict(X_test)\n",
    "print_score(pred_d,ml)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Importances\n",
    "\n",
    "```Python\n",
    "#The difference between standard Pearson's correlation is that this one first transforms variables into ranks and only then runs Pearson's correlation on the ranks.\n",
    "\n",
    "from rfpimp import plot_corr_heatmap\n",
    "viz = plot_corr_heatmap(X_train, figsize=(15,10))\n",
    "viz.view()\n",
    "\n",
    "#use this to overcome the bias in feature importances for all \n",
    "\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "from rfpimp import permutation_importances\n",
    "\n",
    "def r2(rf, X_train, y_train):\n",
    "    return r2_score(y_train, rf.predict(X_train))\n",
    "\n",
    "perm_imp_rfpimp = permutation_importances(rf, X_train, y_train, r2)\n",
    "perm_imp_rfpimp.reset_index(drop = False, inplace = True)\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGfCAYAAABFpjj0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xu03VV97/33p+EiKITDxT6B8pgqEeUiqQQRpVAQLzQtl4oiQ4fA0TJovfWxesxRnw58lHOCnlZq8YZFKJQCRUVUFMRyLYK4AwkJKFI1oEitCESByM3v88eakeXOTvY1e+XHfr/GYOy15pq/+fvOtYB8mHOuTaoKSZKkLvqdQRcgSZI0UQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWZsMugBp++23r7lz5w66DEnSRmTJkiX3VtUOo/UzyGjg5s6dy9DQ0KDLkCRtRJLcOZZ+bi1JkqTOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMsho4JbfvYq5iy4ZdBmSpA4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM6a1iCTZLskS9tf/5nk7r7nm01nLeuo78+SPK/v+clJDprgWHOSfDXJsiS3JflSa392ktdNYLxtk5zYHs/ve9/uS/LD9viyJDsnuWAiNY9wzw/1fUZ3JPl8//sjSdKgbTKdN6uqnwPzAZKcBDxYVf+nv0+SAKmqX09nbc2fAb8GvgtQVe+bxFgfAi6pqo8DJHlBa3828Drg/LEOlGQTYFvgROBTVbWUJ9/HfwY+V1Vf7Lvk6EnUPdxHqurUdq9jgCuT7NE+S0mSBmqj2FpKsktbtTgXuBWYk+T0JENJbk3yN319f5zkpCQ3J7klyXNb+8Ft9WNpkpuSPD3J1kmuaM9vSfInfeMc39qWJTkzyR8Cfwx8tI0xN8k/Jzmi9X9Fa1+e5DNrVpDWVQ8wB/jxmvtV1S3t4WLgoDbW25M8J8m17folSfZt4x6S5KokXwGWt+t2bdctHuW9XNoevznJF5J8I8mdSf4iybvbvb6ZZJvWb15bzVmS5Jq+OfyWqjoPuJJeECPJB5J8O8mKJJ9Kz65Jvt1Xz/OT3DjK3wKSJE3IRhFkmucBH62q3arqbmBRVS0A9gJenmS3vr4/rao/AP4ReGdrezdwQlXNBw4AfgWsBo6oqhcChwAfBUiyF/Ae4I+qai/gr6vqWuCrwP9TVfOrauWamyXZEvgs8Oqq2hPYEjhhlHpOA/6pBan3JpnT2hcBV7Z7fAy4B3h5u/71wMf6xl0A/GVVPb9dd3u7btE43tfdgcOBFwGnAPe3ey0B3tD6nN7uszfwP1vt63ITvc8K4O+rah9gT2A28Kqquh1YnWSP1ud44MzhgyQ5oQXVoSceXjWO6UiS9KSNKch8v6qG+p4fk+Qmen9wPh/oDzJfaD+XAHPb4+uAv0/yNmDrqnoCCLA4yS3A14Gdk2wPHAxcUFX3Aaz5uR7PB75XVd9vz8+mF5bWWU9VfRV4DnBGq/3mJNuNMPbmwBlJVtDbbuqf5/VVddcotY3miqp6qKp+CjwIfLm1LwfmtlWZFwOfbys5Hwd2XM946Xv8srbasgw4kF5ogt6cj29bYq8Bzhs+SFWdXlULqmrBrC1nT2J6kqSZbFrPyIzioTUPkswD3gG8qKoeaOdAntbX95H28wnaHKrqQ+kdqF0I3JDkZfT+cJ0NvLCqHk/y42HjTJW16mk1/Rw4Fzg3yaXA/vTNs/lr4Ef0Vkc2pRc21hjedzK1Qe/8zyN9jzehF0zubStZY/EHwL+3VarT6L23dyf5EE++txcC76UXLq+vqgcmOQdJkka0Ma3I9Nsa+CXwi7Yl88rRLkjynKq6par+N71VnF3phZj/aiHm5cBOrfsVwNFJtm3XbtvafwlsNcLw3wHmJXl2e/4G4OpR6nlZki3a462B3wfuGuEes4F7qqqAY/ntFY9+66ptUqrqfuCeJEe2Wn+nbb2tJclrgYOAC4At6IWhe5NsBby6b8yH6b3HpzHCtpIkSVNlYw0yNwG30fv20Nn0/st+NO9qh05vobeq8XXgHOAlSZbTO6B6B0BVLQM+DFzTtlM+0sY4D3jvmsO+awZufzC/CfhCG+sR4DOj1LMPcFOr55vAJ6vqZuBmYFY7ZPx2en/YvznJMnph55GRBmtbQ0vaYeN1HvadoNcBJ7YabgX+pO+1d7f3447W76Cq+nlbbfonep/T14BvDRvzXOAx4N+muFZJkn4jvYUAaWolWQRsXlUfGK3v5nPm1ZxjT2Xl4oXTUJkkqQuSLGlf+lmvjemMjJ4iknwZ2JneoWpJkjYYg4ymXFX96aBrkCTNDBvrGRlJkqRRGWQkSVJnGWQ0cHvuNNuDvpKkCTHISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLISJKkzjLIaOCW372KuYsuGXQZkqQOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMshIkqTOMsgMQJInkixNsiLJhUm2nKJxv7mO9rOSHDXOsU5K8q4R2ucmWbGe616eZEmS5e3nweO5ryRJ42GQGYzVVTW/qvYAHgVO7H8xPeP+bKrqJVNRXJJNJnH5vcCfVtWewLHAOVNRkyRJIzHIDN61wC5tpeP2JGcDK4CdkxzTVjZWJDkFIMmJST6y5uIkxyU5rT1+sP1MktPaeN8AntnXf+8kV7fVksuSzGntVyU5NckQ8I7+Ats1y5IsA96yvslU1c1V9ZP29FZgiySbD++X5IQkQ0mGnnh41XjfM0mSAIPMQLWVj0OB5a1pHvCJqtodeAw4BTgYmA/sk+QI4PPAkX3DHA2cP2zoI4Fdgd2ANwIvaffbFPgH4Kiq2hv4LHBy33WbVdWCqvrbYeOdCbytqvYa5xRfDdxUVY8Mf6GqTm/3WjBry9njHFaSpJ7JbCFo4rZIsrQ9vhY4A9gRuLOqbmjt+wBXVdXPAJKcCxxQVV9M8oMkLwbuAJ4HXDds/AOA86rqCeAnSa5o7bsCewCXJwGYBdzTd90FwwtNsg2wTVVd05rOoRe+1ivJ7vSC2CtG6ytJ0kQZZAZjdVXN729oweKhMV5/PvBa4LvARVVVY7wuwK1Vtd86Xh/r/dd/k+T3gIuAN1bV96diTEmSRuLW0sbrRuDAJNsnmQUcA1zdXrsIOLy1Dd9WArgGODrJrHYG5qDWfjuwQ5L9oLfV1FZO1qmqHgAeSLJ/a3r9+vq3FZxLgEVVNXylSJKkKWWQ2UhV1T3AIuBKYBmwpKoubq/dD3wHeFZV3TjC5RfR23a6DTgbuL5d9yhwFHBKO7i7lHZ+ZhTHAx9v22EZpe9bgV2Av2lfMV+a5JmjXCNJ0oRk7LsS0oax+Zx5NefYU1m5eOGgS5EkbSSSLKmqBaP1c0VGkiR1lod9NSFJXknvW0n9flhVR47UX5KkDcEgowmpqsuAywZdhyRpZnNrSQO3506zPR8jSZoQg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeosg4wkSeqsTQZdgLT87lXMXXTJb56vXLxwgNVIkrrEFRlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZBhlJktRZMy7IJHkiydIkK5JcmGTLSYx1XJLTJnHtjn3PN02yOMkdSW5Kcn2SQ0cZ468mU/+GkOTwJLe093goyf6DrkmS9NQ144IMsLqq5lfVHsCjwIn9L6ZnOt6X44Ad+55/EJgD7FFVLwSOALYaZYy/AjZokEky3t819G/AXlU1H/jvwD9OfVWSJPXMxCDT71pglyRzk9ye5GxgBbBzkmOSLG8rN6esuSDJ8Um+l+RG4KV97WclOarv+YN9j9/TxlrWVl2OAhYA57aVi6cDfw68raoeAaiqn1bVv7brP9lWN25N8oHW9nZ6QejKJFe2tle0lZyb2mrTM1r7Hyf5bpIlST6W5CutfdskX2wrKDckeUFrPynJOUmuA85Jck2S+X3z+fcke430hlbVg1VV7enTgRqpX5IT2pyGnnh41agflCRJI5mxQaatNBwKLG9N84BPVNXuwGPAKcDBwHxgnyRHJJkDfIBegNkf2G0M9zkUOBzYt6r2Aj5cVZ8DhoDXt5WL5wB3VdUv1jHM+6pqAfAC4MAkL6iqjwE/AQ6qqoOSbA+8HzikregMAe9M8jTg08ChVbU3sEPfuB8Abq6qFwDvBc7ue223NtYxwBn0VpBI8lzgaVW1bD1zPjLJd4FL6K3KrKWqTq+qBVW1YNaWs9c1lCRJ6zUTg8wWSZbS+4P+Lnp/SAPcWVU3tMf7AFdV1c+q6nHgXOAAYN++9keBC8Zwv0OAM6vqYYCqum8CNb82yU3AzcDujBygXtzar2vzOxZ4FvA84AdV9cPW77y+a/YHzml1XQFsl2Tr9tqXqmp1e3wh8CdJNqUXTM5aX7FVdVFVPY/e9tgHxzNRSZLGYyb+v5ZWt1WQ30gC8NAkx32cFgzbGZvNxnHtfwD/d5Kth6/KJPl94F3APlV1f5KzgKeNMEaAy9sKSv/180foOxa/eT+q6uEkl9NbWXotsPdYBqiqa5I8O8n2VXXvBOuQJGmdZuKKzFjcSG8LZ/sks4BjgKuBb7X27drqxGv6rlnJk3/AHwZs2h5fDhy/5ttFSbZt7b+kHeZtqzVnAH+fZLPWb4ckrwG2phcqViX5XXrbYQwfA7gBeGmSXdr1T2/bQLcDz04yt/U7uu/6a4HXt/5/BNy7nu2tfwQ+Bny7qu5fRx+S7JKWDJO8ENgc+Pm6+kuSNBkzcUVmVFV1T5JFwJX0VjouqaqLoXcQFrgeeABY2nfZZ4CLkywDLqWtaFTVpW1VZCjJo8BX6Z1HOQv4VJLVwH70zrd8CLgtya/a9X9TVcuS3Ax8F/gRcF3fPU8HLk3yk3ZO5jjgvCSbt9ffX1XfS/KXrd9DwLf7rj8J+GySW4CH6W1Hres9WZLkF8CZo7x9rwbemOQxYDVwdN/hX0mSplT8M+apL8kzqurBtlLyceCOqvroOMfYEbgKeF5V/Xoq69t8zryac+ypv3m+cvHCqRxektRBSZa0L7qsl1tLM8OftwPAtwKz6X2LacySvJHettr7pjrESJI0GW4tzQBt9WVcKzDDrj+b3/5qNkmOB94xrOt1VfWWid5HkqTxMshoQqrqTEY/LyNJ0gZlkNHA7bnTbIY8FyNJmgDPyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM4yyEiSpM7aZNAFSMvvXsXcRZes1b5y8cIBVCNJ6hJXZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZCRJUmcZZAYsyRNJliZZkeTCJFtOYqzjkpw2iWt37Ht+VZIFE71HklcnqZHGkCRpqhhkBm91Vc2vqj2AR4ET+19Mz3R8TscBO47WaSySbAW8A/jWVIwnSdK6GGQ2LtcCuySZm+T2JGcDK4CdkxyTZHlbuTllzQVJjk/yvSQ3Ai/taz8ryVF9zx/se/yeNtayJItbvwXAuW11aIv+otZ1j/X4IHAK8Kt1dUhyQpKhJENPPLxqDENKkrQ2g8xGIskmwKHA8tY0D/hEVe0OPEYvGBwMzAf2SXJEkjnAB+iFi/2B3cZwn0OBw4F9q2ov4MNV9TlgCHh9Wx1a3dd/XPdI8kJg56pa+1f19qmq06tqQVUtmLXl7NHKliRpRP4vCgZviyRL2+NrgTPobfHcWVU3tPZ9gKuq6mcASc4FDmiv9bdfADx3lPsdApxZVQ8DVNV9o/Tfd6z3aFtgf0dvm0qSpA3OIDN4q6tqfn9DEoCHJjnu47QVtxYwNpvkeGOxFbAHcFWbw/8FfCnJYVU1NA33lyTNMG4tdcONwIFJtk8yCzgGuJreYdoDk2yXZFPgNX3XrAT2bo8PAzZtjy8Hjl/z7agk27b2X9ILIsOt7x6/papWVdX2VTW3quYCNwCGGEnSBuOKTAdU1T1JFgFXAgEuqaqLAZKcBFwPPAAs7bvsM8DFSZYBl9JWeKrq0iTzgaEkjwJfBd4LnAV8KslqYL9h917XPSRJGqhU1aBr0Ay3+Zx5NefYU9dqX7l44QCqkSRtDJIsqapRfxeZW0uSJKmz3FrShCR5H2ufl7mwqk4eRD2SpJnJIKMJaYHF0CJJGiiDjAZuz51mM+R5GEnSBHhGRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkdZZBRpIkddYmgy5AWn73KuYuumTE11YuXjjN1UiSusQVGUmS1FkGGUmS1FkGGUmS1FkGGUmS1FkGGUmS1FkGGUmS1FkGmQFL8kSSpUlWJLkwyZaTGOu4JKdN4tod+55flWTBeO+R5MQky9uc/j3JbhOpR5KksTDIDN7qqppfVXsAjwIn9r+Ynun4nI4Ddhyt0xj8S1XtWVXzgQ8DfzcFY0qSNCKDzMblWmCXJHOT3J7kbGAFsHOSY9pKx4okp6y5IMnxSb6X5EbgpX3tZyU5qu/5g32P39PGWpZkceu3ADi3raRs0V/Uuu4xkqr6Rd/TpwM1oXdCkqQx8Df7biSSbAIcClzamuYBx1bVDW3L5xRgb+B+4OtJjgC+BXygta8CrgRuHuU+hwKHA/tW1cNJtq2q+5K8FXhXVQ21fmv6z5nAPd4CvBPYDDh4HX1OAE4AmLX1DusbTpKkdXJFZvC2SLIUGALuAs5o7XdW1Q3t8T7AVVX1s6p6HDgXOADYt6/9UeCCMdzvEODMqnoYoKruG6X/uO9RVR+vqucA7wHev44+p1fVgqpaMGvL2WMoW5KktbkiM3ir23mS32irIQ9NctzHaUG1nbHZbJLjTcT5wCcHcF9J0gzhikw33AgcmGT7JLOAY4Cr6W0tHZhkuySbAq/pu2Ylve0ggMOATdvjy4Hj13w7Ksm2rf2XwFYj3Ht991hLknl9TxcCd4xtipIkjZ8rMh1QVfckWUTvfEqAS6rqYoAkJwHXAw8AS/su+wxwcZJl9M7dPNTGujTJfGAoyaPAV4H3AmcBn0qyGthv2L3XdY+RvDXJIcBj9M7zHDvxmUuStH6p8kslGqzN58yrOceeOuJrKxcvnOZqJEkbgyRLqmqt32c2nFtLkiSps9xa0oQkeR9rn5e5sKpOHkQ9kqSZySCjCWmBxdAiSRoog4wGbs+dZjPkWRhJ0gR4RkaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHXWJoMuQFp+9yrmLrpkna+vXLxwGquRJHWJKzKSJKmzDDKSJKmzDDKSJKmzDDKSJKmzDDKSJKmz1htkkmyXZGn76z+T3N33fLPpKnI99f1Zkuf1PT85yUETHGtOkq8mWZbktiRfau3PTvK6CYy3bZIT2+P5fe/bfUl+2B5flmTnJBdMpOYR7vmhYZ/R0iRbJTkkyRen4h7jrGeXJEun+76SpJljvV+/rqqfA/MBkpwEPFhV/6e/T5IAqapfb6gi1+PPgF8D3wWoqvdNYqwPAZdU1ccBkrygtT8beB1w/lgHSrIJsC1wIvCpqlrKk+/jPwOfq6r+YHH0JOoe7iNVdeqweqZweEmSNh4T2lpq/6V9W5JzgVuBOUlOTzKU5NYkf9PX98dJTkpyc5Jbkjy3tR/cVj+WJrkpydOTbJ3kivb8liR/0jfO8a1tWZIzk/wh8MfAR9sYc5P8c5IjWv9XtPblST6zZgVpXfUAc4Afr7lfVd3SHi4GDmpjvT3Jc5Jc265fkmTfNu4hSa5K8hVgebtu13bd4lHey6Xt8ZuTfCHJN5LcmeQvkry73eubSbZp/ea11ZwlSa7pm8NYPrsXJ7m+jXldknl9974oydVJ7kjy/ta+VZKvtfd9RZKjWvs+re+S9vrv9rXf0uZ04ljrkiRpIibzC/GeB7yxqoYAkiyqqvvaasSVST5XVbe1vj+tqj9I8nbgnfT+gHs3cEJVfSvJM4Bf0QtWR1TVL5I8E7gO+EqSvYD3AC9p99i2/fwqfasba1YekmwJfBY4sKq+3wLXCcBp66nnNOBfktwEfAM4s6ruARYBb62qI/rGfnlV/Sq9ba1/AvZt4y4Adququ5LsAuxSVfPH+b7uDrwQeAZwB/DOVus/AG9odZ4OvLnN7aWt7RXt+ncnOa49vreqDhk2/neAP6yqx5O8it5K1JoVoRcBewCPAt9uoWxXYGVVHdrmPzvJ5sDfA4dV1b1JXg98sL3HZ9H7XK9L8tFxzl2SpHGZTJD5/poQ0xyT5E1tzB2B3YA1QeYL7ecSeqso0Aspf99Cxuer6sEks4DFSfant2W0c5LtgYOBC6rqPoA1P9fj+cD3qur77fnZwJt4MsisVU9VfTXJc4BXAYcCNyfZfYSxNwdOa+HqceA5fa9dX1V3jVLbaK6oqoeAh5I8CHy5tS8HnttWZV4MfD5Pbhn1f45rbS0Nsw1wdpvrcJdV1f0A6Z2p2R/4N3qfyWLgyy2gzKcXuL7RapgF/Lh9VltU1XVtvHOAEc8sJTmBXvBh1tY7rKdcSZLWbTJB5qE1D9r2xDuAF1XVA+mdA3laX99H2s8n1tyzqj6U3oHahcANSV4GHAjMBl7YVgx+PGycqbJWPa2mnwPnAucmuZTeH+QPDbv2r4Ef0Vsd2RR4sO+14X0nUxv0wtwjfY83AUJvpWW8Kz1rnEwvsHyirRpd2vdaDetbVfWdJAvoBb7FSb4GfA24par+sL9zCzJjUlWn01tZYvM584bfV5KkMZmqr19vDfwS+EWSOcArR7sgyXOq6paq+t/ATfS2MGYD/9VCzMuBnVr3K4Cjk2zbrt22tf8S2GqE4b8DzEvy7Pb8DcDVo9TzsiRbtMdbA78P3DXCPWYD91RVAcfSCxYjWVdtk9JWTO5JcmSr9Xfa6tBYzQbubo+PG/baK5Js07bPDgeuS7ITvUPe5wB/S2/b6zZgpyQvajVslmT3qroXWJ1kvzbe6ycwRUmSxmyqgsxN9P5w+y69bZzr1t8dgHe1w6O30FvV+Dq9rYiXJFlO75tCdwBU1TLgw8A17RDpR9oY5wHvbQdq564ZuKoepreV9IU21iPAZ0apZx/gplbPN4FPVtXNwM3ArHbY9e30tqfenGQZvbDzyEiDVdVPgSXpHTZe52HfCXodcGKr4VbgT/pee3d+++vXOw+79hTgI+0s0PAQ9m3gYmAZcF77ttVe9M7LLAXeC/yvqnoEOAr4u/Z+3cyT54SOBz7d+g/im2ySpBkkvYUFzXRJ3gzsUVV/Nd333nzOvJpz7LqP9fh/v5akmSfJkqpaMFo/f7OvJEnqrMkc9tVTSFX946BrkCRpvFyRkSRJnWWQkSRJneXWkgZuz51mM+SBXknSBLgiI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOssgI0mSOmuTQRcgLb97FXMXXTJl461cvHDKxpIkbdxckZEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkJEkSZ1lkBmwJE8kWZpkRZILk2w5ibGOS3LaJK7dse/5VUkWjPceSQ5IclOSx5McNZFaJEkaK4PM4K2uqvlVtQfwKHBi/4vpmY7P6Thgx9E6jcFdbax/mYKxJElaL4PMxuVaYJckc5PcnuRsYAWwc5JjkixvKzenrLkgyfFJvpfkRuClfe1n9a+IJHmw7/F72ljLkixu/RYA57bVoS36i1rXPUZSVSur6hbg15N7KyRJGp2/2XcjkWQT4FDg0tY0Dzi2qm5oWz6nAHsD9wNfT3IE8C3gA619FXAlcPMo9zkUOBzYt6oeTrJtVd2X5K3Au6pqqPVb03/OeO8xxvmeAJwAMGvrHSY7nCRphnJFZvC2SLIUGKK3LXNGa7+zqm5oj/cBrqqqn1XV48C5wAHAvn3tjwIXjOF+hwBnVtXDAFV13yj9J3KPUVXV6VW1oKoWzNpy9lQMKUmagVyRGbzVVTW/v6Gthjw0yXEfpwXVdsZms0mOJ0nSRscVmW64ETgwyfZJZgHHAFfT21o6MMl2STYFXtN3zUp620EAhwGbtseXA8ev+XZUkm1b+y+BrUa49/ruIUnSQBlkOqCq7gEW0TufsgxYUlUXt/aTgOuB64Dv9F32GXoBZBmwH22Fp6ouBb4EDLUtrXe1/mcBnxp+2HeUe6wlyT5Jfkwv8Hw6ya0Tn7kkSeuXqhp0DZrhNp8zr+Yce+qUjbdy8cIpG0uSNBhJllTVWr/PbDhXZCRJUmd52FcTkuR9rH1e5sKqOnkQ9UiSZiaDjCakBRZDiyRpoAwyGrg9d5rNkOdaJEkT4BkZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWQYZSZLUWZsMugBp+d2rmLvokkGX8ZS1cvHCQZcgSRuMKzKSJKmzDDKSJKmzDDKSJKlmrwevAAANM0lEQVSzDDKSJKmzDDKSJKmzDDKSJKmzDDIDkOSJJEuTrEhyYZItp2jcb66j/awkR41zrJOSvGuE9rlJVqznuhe1uS1NsizJkeO5ryRJ42GQGYzVVTW/qvYAHgVO7H8xPeP+bKrqJVNRXJLJ/H6hFcCCqpoPvAr49CTHkyRpnQwyg3ctsEtb6bg9ydn0wsDOSY5Jsryt3JwCkOTEJB9Zc3GS45Kc1h4/2H4myWltvG8Az+zrv3eSq5MsSXJZkjmt/aokpyYZAt7RX2C7ZlmSZcBb1jeZqnq4qh5vT58G1OTeHkmS1s0gM0BtpeJQYHlrmgd8oqp2Bx4DTgEOBuYD+yQ5Avg80L9dczRw/rChjwR2BXYD3gi8pN1vU+AfgKOqam/gs8DJfddtVlULqupvh413JvC2qtprjPPaN8mtbV4n9gWb/j4nJBlKMvTEw6vGMqwkSWsxyAzGFkmWAkPAXcAZrf3OqrqhPd4HuKqqftaCwLnAAVX1M+AHSV6cZDvgecB1w8Y/ADivqp6oqp8AV7T2XYE9gMvb/d8P/F7fdRcMLzTJNsA2VXVNazpntMlV1bdaGNsH+J9JnjZCn9NbaFowa8vZow0pSdKIPLswGKvbGZLfSALw0BivPx94LfBd4KKqGuv2TYBbq2q/dbw+1vuPSVV9p2137UEvtEmSNKVckdl43QgcmGT7JLOAY4Cr22sXAYe3tuHbSgDXAEcnmdXOwBzU2m8HdkiyH/S2mpLsvr4iquoB4IEk+7em16+vf5LfX3O4N8mz6K0YrVzvTCVJmiBXZDZSVXVPkkXAlfRWUi6pqovba/cn+Q6wW1XdOMLlF9E7W3Mbva2r69t1j7avYX8syWx6n/+pwK2jlHM88NkkBXx9lL77A4uSPAb8GvjLqrp39BlLkjR+GfuuhLRhbD5nXs059tRBl/GUtXLxwkGXIEnjlmRJVS0YrZ9bS5IkqbPcWtKEJHklva+H9/thVfmbfCVJ08YgowmpqsuAywZdhyRpZjPIaOD23Gk2Q57jkCRNgGdkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZ20y6AKk5XevYu6iSwZdhvqsXLxw0CVI0pi4IiNJkjrLICNJkjrLICNJkjrLICNJkjrLICNJkjrLICNJkjrLIDMASZ5IsjTJiiQXJtlyisb95jraz0py1DjHOinJu0Zon5tkxXqu2y7JlUkeTHLaeO4pSdJ4GWQGY3VVza+qPYBHgRP7X0zPuD+bqnrJVBSXZDK/X+hXwP8LrBWCJEmaagaZwbsW2KWtdNye5GxgBbBzkmOSLG8rN6cAJDkxyUfWXJzkuDUrH0kebD+T5LQ23jeAZ/b13zvJ1UmWJLksyZzWflWSU5MMAe/oL7BdsyzJMuAt65tMVT1UVf9OL9BIkrRBGWQGqK18HAosb03zgE9U1e7AY8ApwMHAfGCfJEcAnweO7BvmaOD8YUMfCewK7Aa8EXhJu9+mwD8AR1XV3sBngZP7rtusqhZU1d8OG+9M4G1VtdckpvtbkpyQZCjJ0BMPr5qqYSVJM4xBZjC2SLIUGALuAs5o7XdW1Q3t8T7AVVX1s6p6HDgXOKCqfgb8IMmLk2wHPA+4btj4BwDnVdUTVfUT4IrWviuwB3B5u//7gd/ru+6C4YUm2QbYpqquaU3nTHzaT6qq01toWjBry9lTMaQkaQby/7U0GKuran5/QxKAh8Z4/fnAa4HvAhdVVY3xugC3VtV+63h9rPeXJGmj4IrMxutG4MAk2yeZBRwDXN1euwg4vLUN31YCuAY4OsmsdgbmoNZ+O7BDkv2gt9WUZPf1FVFVDwAPJNm/Nb1+MpOSJGkquSKzkaqqe5IsAq6kt5JySVVd3F67P8l3gN2q6sYRLr+I3tma2+htXV3frnu0fQ37Y0lm0/v8TwVuHaWc44HPJing66PVnmQlsDWwWTvX84qqum206yRJGq+MfVdC2jA2nzOv5hx76qDLUJ+VixcOugRJM1ySJVW1YLR+bi1JkqTOcmtJE5LklfS+Ht7vh1V15Ej9JUnaEAwympCqugy4bNB1SJJmNoOMBm7PnWYz5JkMSdIEeEZGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR1lkFGkiR11iaDLkBafvcq5i66ZNBlSJImaeXihdN+T1dkJElSZxlkJElSZxlkJElSZxlkJElSZxlkJElSZxlkZqgkVyZ55bC2v0ryyfVc8+CGr0ySpLEzyMxc5wGvG9b2utYuSVInGGRmrs8BC5NsBpBkLrAjcHOSf0tyU5LlSQ4ffmGSP0rylb7npyU5rj3eO8nVSZYkuSzJnOmYjCRpZjLIzFBVdR9wI3Boa3od8K/AauDIqnohcBDwt0kyljGTbAr8A3BUVe0NfBY4eaprlyRpDX+z78y2Znvp4vbzTUCA/5XkAODXwE7A7wL/OYbxdgX2AC5v2WcWcM9IHZOcAJwAMGvrHSY1CUnSzGWQmdkuBj6a5IXAllW1pG0R7QDsXVWPJVkJPG3YdY/z26t5a14PcGtV7TfajavqdOB0gM3nzKtJzUKSNGO5tTSDVdWDwJX0toDWHPKdDfxXCzEHAc8a4dI7gd2SbJ5kG+Blrf12YIck+0FvqynJ7ht0EpKkGc0VGZ0HXMST32A6F/hykuXAEPDd4RdU1Y+S/CuwAvghcHNrfzTJUcDHksym9/fXqcCtG3wWkqQZySAzw1XVF+ltCa15fi8w4tZQVT2j7/H/AP7HCH2WAgdMfaWSJK3NrSVJktRZBhlJktRZBhlJktRZBhlJktRZHvbVwO2502yGFi8cdBmSpA5yRUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHWWQUaSJHVWqmrQNWiGS/JL4PZB1zEg2wP3DrqIAXHuM5Nzn5kmMvdnVdUOo3Xyf1GgjcHtVbVg0EUMQpIh5z7zOHfnPtNsyLm7tSRJkjrLICNJkjrLIKONwemDLmCAnPvM5NxnJue+AXjYV5IkdZYrMpIkqbMMMpo2SV6V5PYk/5Fk0Qivb57kgvb6t5LMnf4qN4wxzP2AJDcleTzJUYOocUMZw9zfmeS2JLck+bckzxpEnRvCGOZ+YpLlSZYm+fckuw2izg1htLn39Xt1kkrylPk2zxg+9+OS/Kx97kuTvHkQdW4IY/nck7y2/TN/a5J/mfRNq8q//GuD/wXMAr4PPBvYDFgG7Dasz18Cn2qPXwdcMOi6p3Huc4EXAGcDRw265mme+0HAlu3xX8ywz33rvseHAZcOuu7pmnvrtxVwDXADsGDQdU/j534ccNqgax3Q3OcBNwP/rT1/5mTv64qMpsuLgP+oqh9U1aPA+cDhw/ocDvxTe/w54GVJMo01biijzr2qVlbVLcCvB1HgBjSWuV9ZVQ+3pzcAvzfNNW4oY5n7L/qePh14qhxaHMs/7wAfBE4BfjWdxW1gY537U9FY5v7nwMer6n6Aqvqvyd7UIKPpshPwo77nP25tI/apqseBVcB201LdhjWWuT9VjXfubwK+tkErmj5jmnuStyT5PvBh4O3TVNuGNurck7wQ2LmqLpnOwqbBWP+ef3XbTv1ckp2np7QNbixzfy7w3CTXJbkhyasme1ODjKSNQpI3AAuAjwy6lulUVR+vqucA7wHeP+h6pkOS3wH+DvjrQdcyIF8G5lbVC4DLeXIleibYhN720h8BxwCfSbLNZAY0yGi63A30/1fH77W2Efsk2QSYDfx8WqrbsMYy96eqMc09ySHA+4DDquqRaaptQxvv534+cMQGrWj6jDb3rYA9gKuSrAReDHzpKXLgd9TPvap+3vf3+T8Ce09TbRvaWP6e/zHwpap6rKp+CHyPXrCZMIOMpsu3gXlJfj/JZvQO835pWJ8vAce2x0cBV1Q7DdZxY5n7U9Woc0/yB8Cn6YWYSe+Xb0TGMvf+f4EvBO6Yxvo2pPXOvapWVdX2VTW3qubSOxt1WFUNDabcKTWWz31O39PDgO9MY30b0lj+XfdFeqsxJNme3lbTDyZzU/+nkZoWVfV4krcCl9E72f7Zqro1yf8HDFXVl4AzgHOS/AdwH71/CDpvLHNPsg9wEfDfgD9N8oGq2n2AZU+JMX7uHwGeAVzYznbfVVWHDazoKTLGub+1rUY9BtzPk0G+08Y496ekMc797UkOAx6n9++64wZW8BQa49wvA16R5DbgCeDdVTWplXd/s68kSeost5YkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJnGWQkSVJn/f+RtD7qYiQ8aAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#plot graph of feature importances for better visualization\n",
    "plt.figure(figsize=(7,7))\n",
    "feat_importances = pd.Series(ml.feature_importances_, index=train.columns)\n",
    "feat_importances.nlargest(10).plot(kind='barh')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Importances Study\n",
    "\n",
    "I have been doing some study and I have found that using the default clasiffier feature importance maybe biased. There are other ways, I want to try them and see if I get the same results as the ones I get above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_train_bi.shape, y_train_bi.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I need to fit the new data and predict from that..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tuning Different Classifiers\n",
    "\n",
    "I have got a high score by stacking up classifiers, these classifiers were not really tuned well enough to better their functionality.\n",
    "\n",
    "My goal is to make better these classifiers and then stack them up again. The process involved takes a lot of time and computational space, to the extent that I can tune only about three models a day.\n",
    "\n",
    "- [A great article on overfitting](https://medium.com/machine-learning-intuition/overfitting-what-they-are-regularization-e950c2d66d50)\n",
    "- [Another article](https://www.gormanalysis.com/blog/magic-behind-constructing-a-decision-tree/)\n",
    "- [A cook notebook to help](https://github.com/erykml/medium_articles/blob/master/feature_importance.ipynb)\n",
    "- [OOB_Evaluation](http://cican17.com/machine-learning-with-tree-based-models-ii/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#classifiers\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import GaussianNB \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "#stack\n",
    "#from mlxtend.classifier import StackingClassifier\n",
    "\n",
    "#i will use this after the stack and see what happens\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "# Used to ignore warnings generated from StackingCVClassifier\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.metrics import roc_curve, auc\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest Classifier Tunning\n",
    "\n",
    "When you tune a model you need to know how the model is doig and in order to find that out you have to evaluate the model, this can be based on which model you have because different models have different evaluations.\n",
    "- [This article explain it well](https://towardsdatascience.com/common-classification-model-evaluation-metrics-2ba0a7a7436e)\n",
    "- [Article 2](https://towardsdatascience.com/metrics-to-evaluate-your-machine-learning-algorithm-f10ba6e38234)\n",
    "-[Ada Boost Article](https://www.datacamp.com/community/tutorials/adaboost-classifier-python)\n",
    "\n",
    "-This article on Cross Validation is great, take some time to read it: It mentions the different types of Cross Validation for better understanding.[Cross Validation](https://towardsdatascience.com/cross-validation-in-machine-learning-72924a69872f)\n",
    "\n",
    "Articles on RForestClassifier\n",
    "-[Article One] (https://chrisalbon.com/machine_learning/trees_and_forests/random_forest_classifier_example/)\n",
    "-[Article Two]()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\'rm_clf = RandomForestClassifier()\\n\\n\\n# Define parameter grid \\nparams = {\\n    #\\'C\\' :list(range(2,15))\\n    #\\'kernel\\':[\\'linear\\', \\'poly\\', \\'rbf\\', \\'sigmoid\\', \\'precomputed\\'],\\n    #\\'degree\\' : list(range(2,20)),\\n    #\\'gamma\\' : [\\'rbf\\', \\'poly\\' ,\\'sigmoid\\',\\'auto\\'],\\n    #\\'probability\\' : [False,True]\\n    \\n   \\'n_estimators\\':[147],\\n    \\'criterion\\':[\\'entropy\\'],\\n    #\\'max_features\\':[\\'auto\\',\\'sqrt\\',\\'log2\\',\\'None\\'],\\n    #\\'min_samples_leaf\\': list(range(10,200))\\n}\\n\\n\\n# Initialize GridSearchCV\\ngrid = GridSearchCV(estimator = rm_clf, \\n                    param_grid = params, \\n                    cv = 3,\\n                    scoring = \"roc_auc\",\\n                    verbose = 1,\\n                    n_jobs = -1)\\n\\n# Fit GridSearchCV\\n%time\\ngrid.fit(X_train, y_train)\\n\\n\\ny_pred = grid.predict(X_test)\\n\\nfalse_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)\\nroc_auc = auc(false_positive_rate, true_positive_rate)\\nf1 = f1_score(y_test,y_pred)\\n\\n# Print results\\nprint(f\"The ROC of the tuned classifier is {roc_auc:.3f}\")\\nprint(f\"The F1 of the tuned classifier is {f1:.3f}\")\\n\\nprint(grid.best_params_)'"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''''rm_clf = RandomForestClassifier()\n",
    "\n",
    "\n",
    "# Define parameter grid \n",
    "params = {\n",
    "    #'C' :list(range(2,15))\n",
    "    #'kernel':['linear', 'poly', 'rbf', 'sigmoid', 'precomputed'],\n",
    "    #'degree' : list(range(2,20)),\n",
    "    #'gamma' : ['rbf', 'poly' ,'sigmoid','auto'],\n",
    "    #'probability' : [False,True]\n",
    "    \n",
    "   'n_estimators':[147],\n",
    "    'criterion':['entropy'],\n",
    "    #'max_features':['auto','sqrt','log2','None'],\n",
    "    #'min_samples_leaf': list(range(10,200))\n",
    "}\n",
    "\n",
    "\n",
    "# Initialize GridSearchCV\n",
    "grid = GridSearchCV(estimator = rm_clf, \n",
    "                    param_grid = params, \n",
    "                    cv = 3,\n",
    "                    scoring = \"roc_auc\",\n",
    "                    verbose = 1,\n",
    "                    n_jobs = -1)\n",
    "\n",
    "# Fit GridSearchCV\n",
    "%time\n",
    "grid.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "y_pred = grid.predict(X_test)\n",
    "\n",
    "false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)\n",
    "roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "f1 = f1_score(y_test,y_pred)\n",
    "\n",
    "# Print results\n",
    "print(f\"The ROC of the tuned classifier is {roc_auc:.3f}\")\n",
    "print(f\"The F1 of the tuned classifier is {f1:.3f}\")\n",
    "\n",
    "print(grid.best_params_)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#grid.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Randomized Search\n",
    "\n",
    "I want to compare the two search algorithms and see the one that does the best.\n",
    "\n",
    "- I  have read that the randomize search can do the same work that the GridSearch can but in a shorter period of time and time is one thing I would like to save.\n",
    "\n",
    "I have been having some hard time implementing the grid search algo, I have borrowed some ideas from Koerhsen on handing this in a good way. Let's implement them here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Frome here ***Research\n",
    "\n",
    "To implement this I have decided to try using the LGBClassifier first.\n",
    "\n",
    "Here below is a function that is borrowed from fastai that is going to help us samplea number of rows so we can do rapid prototyping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sample(df,n):\n",
    "    \"\"\" Gets a random sample of n rows from df, without replacement.\n",
    "\n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: A pandas data frame, that you wish to sample from.\n",
    "    n: The number of rows you wish to sample.\n",
    "\n",
    "    Returns:\n",
    "    --------\n",
    "    return value: A random sample of n rows of df.\n",
    "\n",
    "    Examples:\n",
    "    ---------\n",
    "    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n",
    "    >>> df\n",
    "       col1 col2\n",
    "    0     1    a\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "\n",
    "    >>> get_sample(df, 2)\n",
    "       col1 col2\n",
    "    1     2    b\n",
    "    2     3    a\n",
    "    \"\"\"\n",
    "    idxs = sorted(np.random.permutation(len(df))[:n])\n",
    "    return df.iloc[idxs].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#we now take samples of the whole dataset without replacing\n",
    "X_train_sample = get_sample(pd.DataFrame(X_train), 15000)\n",
    "y_train_sample = get_sample(pd.DataFrame(y_train), 15000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%pip install setuptools wheel numpy scipy scikit-learn -U\n",
    "#%pip install lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's have a look at the hyper parameters\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "# Modeling\n",
    "import lightgbm as lgb\n",
    "\n",
    "# Evaluation of the model\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from timeit import default_timer as timer\n",
    "\n",
    "model = lgb.LGBMClassifier()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
       "               importance_type='split', learning_rate=0.1, max_depth=-1,\n",
       "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
       "               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
       "               random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
       "               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#list of \n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Hyperparameter grid\n",
    "param_grid = {\n",
    "    'class_weight': [None, 'balanced'],\n",
    "    'boosting_type': ['gbdt', 'goss', 'dart'],\n",
    "    'num_leaves': list(range(30, 150)),\n",
    "    'learning_rate': list(np.logspace(np.log(0.005), np.log(0.2), base = np.exp(1), num = 1000)),\n",
    "    'subsample_for_bin': list(range(20000, 300000, 20000)),\n",
    "    'min_child_samples': list(range(20, 500, 5)),\n",
    "    'reg_alpha': list(np.linspace(0, 1)),\n",
    "    'reg_lambda': list(np.linspace(0, 1)),\n",
    "    'colsample_bytree': list(np.linspace(0.6, 1, 10))\n",
    "}\n",
    "\n",
    "# Subsampling (only applicable with 'goss')\n",
    "subsample_dist = list(np.linspace(0.5, 1, 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'boosting_type': 'gbdt',\n",
       " 'class_weight': 'balanced',\n",
       " 'colsample_bytree': 0.7333333333333333,\n",
       " 'learning_rate': 0.01814047354852268,\n",
       " 'min_child_samples': 70,\n",
       " 'num_leaves': 104,\n",
       " 'reg_alpha': 0.4081632653061224,\n",
       " 'reg_lambda': 0.7959183673469387,\n",
       " 'subsample_for_bin': 240000}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#since we will have many spaces to sample from let's subset the hyper parameters and work with a few at a time\n",
    "import random\n",
    "\n",
    "params = {key: random.sample(value, 1)[0] for key, value in param_grid.items()}\n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'boosting_type': 'gbdt',\n",
       " 'class_weight': 'balanced',\n",
       " 'colsample_bytree': 0.7333333333333333,\n",
       " 'learning_rate': 0.01814047354852268,\n",
       " 'min_child_samples': 70,\n",
       " 'num_leaves': 104,\n",
       " 'reg_alpha': 0.4081632653061224,\n",
       " 'reg_lambda': 0.7959183673469387,\n",
       " 'subsample': 0.5151515151515151,\n",
       " 'subsample_for_bin': 240000}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params['subsample'] = random.sample(subsample_dist, 1)[0] if params['boosting_type'] != 'goss' else 1.0\n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a lgb dataset\n",
    "train_set = lgb.Dataset(X_train_sample, label = y_train_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximium ROC AUC on the validation set was 0.99803 with std of 0.00120.\n",
      "The ideal number of iterations was 28.\n"
     ]
    }
   ],
   "source": [
    "# Perform cross validation with 10 folds\n",
    "r = lgb.cv(params, train_set, num_boost_round = 10000, nfold = 10, metrics = 'auc', \n",
    "           early_stopping_rounds = 100, verbose_eval = False, seed = 50)\n",
    "\n",
    "# Highest score\n",
    "r_best = np.max(r['auc-mean'])\n",
    "\n",
    "# Standard deviation of best score\n",
    "r_best_std = r['auc-stdv'][np.argmax(r['auc-mean'])]\n",
    "\n",
    "print('The maximium ROC AUC on the validation set was {:.5f} with std of {:.5f}.'.format(r_best, r_best_std))\n",
    "print('The ideal number of iterations was {}.'.format(np.argmax(r['auc-mean']) + 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_EVALS = 10\n",
    "N_FOLDS = 5\n",
    "\n",
    "# Dataframe to hold cv results\n",
    "random_results = pd.DataFrame(columns = ['loss', 'params', 'iteration', 'estimators', 'time'],\n",
    "                       index = list(range(MAX_EVALS)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_objective(params, iteration, n_folds = N_FOLDS):\n",
    "    \"\"\"Random search objective function. Takes in hyperparameters\n",
    "       and returns a list of results to be saved.\"\"\"\n",
    "\n",
    "    start = timer()\n",
    "    \n",
    "    # Perform n_folds cross validation\n",
    "    cv_results = lgb.cv(params, train_set, num_boost_round = 10000, nfold = n_folds, \n",
    "                        early_stopping_rounds = 100, metrics = 'auc', seed = 50)\n",
    "    end = timer()\n",
    "    best_score = np.max(cv_results['auc-mean'])\n",
    "    \n",
    "    # Loss must be minimized\n",
    "    loss = 1 - best_score\n",
    "    \n",
    "    # Boosting rounds that returned the highest cv score\n",
    "    n_estimators = int(np.argmax(cv_results['auc-mean']) + 1)\n",
    "    \n",
    "    # Return list of results\n",
    "    return [loss, params, iteration, n_estimators, end - start]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "random.seed(50)\n",
    "\n",
    "# Iterate through the specified number of evaluations\n",
    "for i in range(MAX_EVALS):\n",
    "    \n",
    "    # Randomly sample parameters for gbm\n",
    "    params = {key: random.sample(value, 1)[0] for key, value in param_grid.items()}\n",
    "    \n",
    "    print(params)\n",
    "    \n",
    "    if params['boosting_type'] == 'goss':\n",
    "        # Cannot subsample with goss\n",
    "        params['subsample'] = 1.0\n",
    "    else:\n",
    "        # Subsample supported for gdbt and dart\n",
    "        params['subsample'] = random.sample(subsample_dist, 1)[0]\n",
    "        \n",
    "        \n",
    "    results_list = random_objective(params, i)\n",
    "    \n",
    "    # Add results to next row in dataframe\n",
    "    random_results.loc[i, :] = results_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss</th>\n",
       "      <th>params</th>\n",
       "      <th>iteration</th>\n",
       "      <th>estimators</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00127925</td>\n",
       "      <td>{'reg_lambda': 0.3469387755102041, 'subsample_...</td>\n",
       "      <td>5</td>\n",
       "      <td>465</td>\n",
       "      <td>713.923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.00135814</td>\n",
       "      <td>{'reg_lambda': 0.8163265306122448, 'subsample_...</td>\n",
       "      <td>6</td>\n",
       "      <td>1573</td>\n",
       "      <td>711.426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0013786</td>\n",
       "      <td>{'reg_lambda': 0.3469387755102041, 'subsample_...</td>\n",
       "      <td>0</td>\n",
       "      <td>4894</td>\n",
       "      <td>637.141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.00144347</td>\n",
       "      <td>{'reg_lambda': 0.8979591836734693, 'subsample_...</td>\n",
       "      <td>4</td>\n",
       "      <td>2880</td>\n",
       "      <td>655.919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.00150414</td>\n",
       "      <td>{'reg_lambda': 0.6938775510204082, 'subsample_...</td>\n",
       "      <td>9</td>\n",
       "      <td>314</td>\n",
       "      <td>6.60128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         loss                                             params iteration  \\\n",
       "0  0.00127925  {'reg_lambda': 0.3469387755102041, 'subsample_...         5   \n",
       "1  0.00135814  {'reg_lambda': 0.8163265306122448, 'subsample_...         6   \n",
       "2   0.0013786  {'reg_lambda': 0.3469387755102041, 'subsample_...         0   \n",
       "3  0.00144347  {'reg_lambda': 0.8979591836734693, 'subsample_...         4   \n",
       "4  0.00150414  {'reg_lambda': 0.6938775510204082, 'subsample_...         9   \n",
       "\n",
       "  estimators     time  \n",
       "0        465  713.923  \n",
       "1       1573  711.426  \n",
       "2       4894  637.141  \n",
       "3       2880  655.919  \n",
       "4        314  6.60128  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sort results by best validation score\n",
    "random_results.sort_values('loss', ascending = True, inplace = True)\n",
    "random_results.reset_index(inplace = True, drop = True)\n",
    "random_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'boosting_type': 'dart',\n",
       " 'class_weight': None,\n",
       " 'colsample_bytree': 0.7333333333333333,\n",
       " 'learning_rate': 0.14451322534564692,\n",
       " 'min_child_samples': 280,\n",
       " 'num_leaves': 117,\n",
       " 'reg_alpha': 0.16326530612244897,\n",
       " 'reg_lambda': 0.3469387755102041,\n",
       " 'subsample': 0.9494949494949496,\n",
       " 'subsample_for_bin': 200000}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_results.loc[0, 'params']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random and grid search are uninformed methods that do not use the past history, but we still need the history so we can find out which hyperparameters worked the best!\n",
    "\n",
    "A dataframe is a useful data structure to hold the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_EVALS = 100\n",
    "N_FOLDS = 5\n",
    "\n",
    "# Dataframes for random and grid search\n",
    "random_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],\n",
    "                              index = list(range(MAX_EVALS)))\n",
    "\n",
    "grid_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],\n",
    "                              index = list(range(MAX_EVALS)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "def grid_search(param_grid, max_evals = MAX_EVALS):\n",
    "    \"\"\"Grid search algorithm (with limit on max evals)\"\"\"\n",
    "    \n",
    "    # Dataframe to store results\n",
    "    results = pd.DataFrame(columns = ['score', 'params', 'iteration'],\n",
    "                              index = list(range(MAX_EVALS)))\n",
    "    \n",
    "    # https://codereview.stackexchange.com/questions/171173/list-all-possible-permutations-from-a-python-dictionary-of-lists\n",
    "    keys, values = zip(*param_grid.items())\n",
    "    \n",
    "    i = 0\n",
    "    \n",
    "    # Iterate through every possible combination of hyperparameters\n",
    "    for v in itertools.product(*values):\n",
    "        \n",
    "        # Create a hyperparameter dictionary\n",
    "        hyperparameters = dict(zip(keys, v))\n",
    "        \n",
    "        # Evalute the hyperparameters\n",
    "        eval_results = objective(hyperparameters, i)\n",
    "        \n",
    "        results.loc[i, :] = eval_results\n",
    "        \n",
    "        i += 1\n",
    "        \n",
    "        # Normally would not limit iterations\n",
    "        if i > MAX_EVALS:\n",
    "            break\n",
    "       \n",
    "    # Sort with best score on top\n",
    "    results.sort_values('score', ascending = False, inplace = True)\n",
    "    results.reset_index(inplace = True)\n",
    "    \n",
    "    return results    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(hyperparameters, iteration):\n",
    "    \"\"\"Objective function for grid and random search. Returns\n",
    "       the cross validation score from a set of hyperparameters.\"\"\"\n",
    "    \n",
    "    \n",
    "    # Number of estimators will be found using early stopping\n",
    "    if 'n_estimators' in hyperparameters.keys():\n",
    "        del hyperparameters['n_estimators']\n",
    "    \n",
    "     # Perform n_folds cross validation\n",
    "    cv_results = cross_val_score(rm_clf, X_train,y_train, cv=5)\n",
    "    \n",
    "    # results to retun\n",
    "    score = [cv_results]\n",
    "    estimators = len(cv_results)\n",
    "    hyperparameters['n_estimators'] = estimators \n",
    "    \n",
    "    return [score, hyperparameters, iteration]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-101-0858923369cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mgrid_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrid_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'The best validation score was {:.5f}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid_results\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'score'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-79-79ddd004e55f>\u001b[0m in \u001b[0;36mgrid_search\u001b[0;34m(param_grid, max_evals)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;31m# Sort with best score on top\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m     \u001b[0mresults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'score'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mascending\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m     \u001b[0mresults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minplace\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36msort_values\u001b[0;34m(self, by, axis, ascending, inplace, kind, na_position)\u001b[0m\n\u001b[1;32m   4992\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4993\u001b[0m             indexer = nargsort(\n\u001b[0;32m-> 4994\u001b[0;31m                 \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkind\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mascending\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mna_position\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mna_position\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4995\u001b[0m             )\n\u001b[1;32m   4996\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/core/sorting.py\u001b[0m in \u001b[0;36mnargsort\u001b[0;34m(items, kind, ascending, na_position)\u001b[0m\n\u001b[1;32m    258\u001b[0m         \u001b[0mnon_nans\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnon_nans\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0mnon_nan_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnon_nan_idx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 260\u001b[0;31m     \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnon_nan_idx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnon_nans\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margsort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkind\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    261\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mascending\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    262\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "\n",
    "\n",
    "grid_results = grid_search(param_grid)\n",
    "\n",
    "print('The best validation score was {:.5f}'.format(grid_results.loc[0, 'score']))\n",
    "print('\\nThe best hyperparameters were:')\n",
    "\n",
    "import pprint\n",
    "pprint.pprint(grid_results.loc[0, 'params'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## To here **Research\n",
    "I want to rewrite the functions above by example so that I can implement cross validation well on the classifiers I want to use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"from sklearn.model_selection import GridSearchCV , RandomSearchCV\\nrm_clf = RandomForestClassifier()\\n\\n\\n# Define parameter grid \\nparams = {\\n    #\\'C\\' : [1,5,9,10,11,15,18,20,25,28,30,35,36,40,47,48,55,60,63,65,70,75,80,81,84,85,90,95,100],\\n    #\\'kernel\\':[\\'linear\\', \\'poly\\', \\'rbf\\', \\'sigmoid\\', \\'precomputed\\'],\\n    #\\'degree\\' : [2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],\\n    #\\'gamma\\' : [\\'rbf\\', \\'poly\\' ,\\'sigmoid\\',\\'auto\\'],\\n    #\\'probability\\' : [False,True]\\n    \\n   \\'n_estimators\\':list(range(300, 800)),\\n    #\\'max_depth\\' : [25, 30,35,40,45,50,60,100],\\n    #\\'min_samples_split\\' : [2, 3, 4, 5, 6, 7,8, 10],\\n    #\\'criterion\\':[\\'entropy\\'],\\n    #\\'max_features\\':[\\'auto\\'],\\n    #\\'min_samples_leaf\\': list(range(9,16)),\\n    #\\'bootstrap\\':[True],\\n    #\\'oob_score\\':[True]\\n}\\n\\n\\n# Initialize GridSearchCV\\ngrid = RandomSearchCV(estimator = rm_clf, \\n                    param_grid = params, \\n                    cv = 3,\\n                    scoring = \"f1\",\\n                    verbose = 2,\\n                    n_iter=4,\\n                    n_jobs = -1)\\n\\n# Fit GridSearchCV\\n%time\\ngrid.fit(X_train, y_train)\\n\\n\\ny_pred = grid.predict(X_test)\\n\\nfalse_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)\\nroc_auc = auc(false_positive_rate, true_positive_rate)\\n\\n\\n# Print results\\nprint(\"The ROC of the tuned classifier is\",roc_auc)\\n\\n\\nprint(grid.best_params_)'"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\"from sklearn.model_selection import GridSearchCV , RandomSearchCV\n",
    "rm_clf = RandomForestClassifier()\n",
    "\n",
    "\n",
    "# Define parameter grid \n",
    "params = {\n",
    "    #'C' : [1,5,9,10,11,15,18,20,25,28,30,35,36,40,47,48,55,60,63,65,70,75,80,81,84,85,90,95,100],\n",
    "    #'kernel':['linear', 'poly', 'rbf', 'sigmoid', 'precomputed'],\n",
    "    #'degree' : [2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],\n",
    "    #'gamma' : ['rbf', 'poly' ,'sigmoid','auto'],\n",
    "    #'probability' : [False,True]\n",
    "    \n",
    "   'n_estimators':list(range(300, 800)),\n",
    "    #'max_depth' : [25, 30,35,40,45,50,60,100],\n",
    "    #'min_samples_split' : [2, 3, 4, 5, 6, 7,8, 10],\n",
    "    #'criterion':['entropy'],\n",
    "    #'max_features':['auto'],\n",
    "    #'min_samples_leaf': list(range(9,16)),\n",
    "    #'bootstrap':[True],\n",
    "    #'oob_score':[True]\n",
    "}\n",
    "\n",
    "\n",
    "# Initialize GridSearchCV\n",
    "grid = RandomSearchCV(estimator = rm_clf, \n",
    "                    param_grid = params, \n",
    "                    cv = 3,\n",
    "                    scoring = \"f1\",\n",
    "                    verbose = 2,\n",
    "                    n_iter=4,\n",
    "                    n_jobs = -1)\n",
    "\n",
    "# Fit GridSearchCV\n",
    "%time\n",
    "grid.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "y_pred = grid.predict(X_test)\n",
    "\n",
    "false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)\n",
    "roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "\n",
    "\n",
    "# Print results\n",
    "print(\"The ROC of the tuned classifier is\",roc_auc)\n",
    "\n",
    "\n",
    "print(grid.best_params_)\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['roc_auc-->', 0.809259119211475, 'F1_score-->', 0.6190476190476191, 'roc_auc_score-->', 0.809259119211475]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.0]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 1.0]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.00355\n",
      "Accuracy  :  0.99645\n",
      "Precision  :  0.0\n",
      "Recall  :  0.0\n",
      "Specificity  :  0.0\n",
      "False positive rate  :  0.0002508277315139962\n",
      "True positive rate  :  0.0\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Out_of_sample_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGO5JREFUeJzt3XuYXXV56PHvm5kEuYMBkVwKQaIVtKBAoBSOCCJBK8GWajheOBifUIUW8VQuRyztKUVQS/CGEARBbBM4HISoIF64iQgJbThIwsWBKMyEcA8WAkwm8zt/zCJuI2FCumfWm1nfD896nr1+e629fzvk2W/ed737t6KUgiRJdRtV9wQkSQIDkiQpCQOSJCkFA5IkKQUDkiQpBQOSJCkFA5IkKQUDkiQpBQOSJCmFzqF+g5VPPOhSEBo2G4/bv+4pqGH6enuiXa/Vzu/L0dvs1LZ5DRczJElSCkOeIUmS1lH/qrpnUCszJElSCmZIkpRF6a97BrUyIElSFv3NDkiW7CRJKZghSVISxZKdJCkFS3aSJNXPDEmSsrBkJ0lKwR/GSpJUPzMkScrCkp0kKQW77CRJqp8ZkiQl4Q9jJUk5WLKTJKl+ZkiSlIUlO0lSCv4wVpKk+pkhSVIWluwkSSnYZSdJUv3MkCQpC0t2kqQULNlJklQ/MyRJSqKUZv8OyYAkSVk0/BqSJTtJUgpmSJKURcObGgxIkpRFw0t2BiRJysLFVSVJqp8ZkiRlYclOkpRCw5saLNlJklIwQ5KkLCzZSZJSsGQnSVL9zJAkKYuGZ0gGJElKoumrfVuykySlYIYkSVlYspMkpdDwtm9LdpKkFMyQJCkLS3aSpBQs2UmSVD8zJEnKwpKdJCkFS3aSJNXPDEmSsrBkJ0lKoeEByZKdJCkFMyRJyqLhTQ0GJEnKwpKdJEn1M0OSpCws2UmSUrBkJ0lS/cyQJCkLS3aSpBQs2UmSVD8zJEnKouEZkgFJkrIope4Z1MqSnSQpBTMkScrCkp0kKYWGByRLdpKkFMyQJCmLhv8w1gxJkrLo72/fNoiIuCgiHouIu9cY/5uIuDciFkXEF1rGT4mIroi4LyIOaRmfWo11RcTJLeOTIuL2avyyiBgz2JwMSJLUTBcDU1sHIuKdwDRgt1LKrsCXqvFdgOnArtU550ZER0R0AF8HDgV2AY6sjgU4C5hVStkZeBqYMdiEDEiSlEUp7dsGfatyM/DUGsOfAM4spbxYHfNYNT4NmFtKebGUsgToAqZUW1cp5cFSSi8wF5gWEQEcCFxRnX8JcPhgczIgSVIWbSzZRcTMiLijZZu5DjN4I7B/VWq7KSL2qsbHAw+3HNddja1tfCywvJTSt8b4K7KpQZJGoFLKbGD2qzytE3gtsA+wF3B5ROzU7rm90ptLkjKo/3dI3cCVpZQCzI+IfmAboAeY2HLchGqMtYw/CWwVEZ1VltR6/FpZspOkLEp/+7b1cxXwToCIeCMwBngCmAdMj4iNImISMBmYDywAJlcddWMYaHyYVwW0G4Ajqtc9Crh6sDc3Q5KkBoqIOcABwDYR0Q2cBlwEXFS1gvcCR1XBZVFEXA4sBvqAY0spq6rXOQ64DugALiqlLKre4iRgbkScDiwELhx0TmWIV5dd+cSDzV6+VsNq43H71z0FNUxfb0+067VWzD6hbd+Xm8yc1bZ5DRczJEnKov5rSLXyGpIkKQUzJEnKouFr2RmQJCmL/mZfcrdkJ0lKwQxJkrJoeFODAUmSsjAgSZJSGOLfhWbnNSRJUgpmSJKUhSU7DaVTzzibm38+n9duvRVXfec8AO791YP80xe/yornX2Dc9q/jrNNOZLNNN2XlypX84xe+yqJ7f0WMCk4+/q+Z8vY/4fkXXuDTp55Bd88jjBo1igP225sTPvExAJYue5TPnTGLp5Y/w5ZbbM6Zf/8ZXv+6bev8yNrAdN1/G//57LOsWtVPX18f+/zpe+qeUnPZ9q2hdPh7Dua8s0//vbHTzjyHT33iaL576Tc46L/ty7f+9f8CcMW8HwLw3Uu/wQXnnMGXvnYB/dW/mI4+8i/53pwLuOLir7HwrsX87BcLAPjS177JYVMP4rvf/gafOPq/c855Fw/fh9OI8a6D/4o993q3wUi1GjQgRcQfR8RJEfGVajspIt48HJMbCfbc/a1sucXmvzf2m4d72HP3twLwp3u9nR/fdAsAD/z6IabssRsAY7feis0325RF9/6KjV/zmtXjo0eP5s1v2plHH39i4JwlDzFlj90BmPL23bjhZ78Yls8laQjUf/uJWr1iQIqIkxi4R3owcO+L+dXjORFx8tBPb2R6w6QduL4KHD+64Wcse3QguLxp50nceMtt9PWtonvpMhbf18WyRx//vXN/+5/PctPPb2fvKgi9afJO/OSmnwPwk5tu5bkVz7P8md8O46fRhq6UwrXXzOH2267l4zM+VPd0mq2/tG/bAA12DWkGsGspZWXrYEScDSwCzny5k6p7t88EOPdfTufjHz2yDVMdOf7pf53A52d9g/MvnsMB++3D6NED/xve/95DePDXD/PBGX/LuNe/jt3f8mZGdfzu3wx9fas48R/O4kNHHMbE8dsD8HfHfpx/Pvtcrr7mx+yx+1vZbtuxjBplJVbr7h3vfD9Lly5j223H8sNr53LffV387Jbb656WGmiwgNQPjAN+s8b49tVzL6v1Xu7eD+kP7bTDRC445wwAfv1QNzffOh+Azs4OTjr+mNXHfeiYT7PjxPGr9//hC1/mjyaM4yMffP/qsddtO5Yvf/5zAKxY8Tw/ufEWtth8s+H4GBohli5dBsDjjz/J1Vdfy1577W5AqklpeJfdYP+U/hTw04i4NiJmV9sPgZ8Cxw/99EamJ59eDkB/fz/nXzKXDxw+cCH5+RdeYMXzLwBw6/z/oLOjgzdM2gGAr8y+hGefXcHJLQEL4Onlz6xufLjg0st4/3vfPVwfQyPAJptszGabbbr68cHvegeLFt1X86wazJLd2pVSfljdV30K8NI/1XuABS/dvlav7DOnncmChXexfPlvOejwD/PJGR9hxfPPM/fK7wPwrnfsuzqIPPX0MxxzwmeJUaPYbtuxfP7v/w6AZY89zuxL5jJph4n81dF/A8CRf/k+jjhsKgsW3sU5511MRLDHbm/h1P/5yXo+qDZI2223LVf8n4E7S3d2djB37lVc96Mb652UGstbmGtE8RbmGm7tvIX5c6d/uG3fl5ue+h1vYS5JWk8baKmtXWzHkiSlYIYkSVk0vMvOgCRJWViykySpfmZIkpTFBroGXbsYkCQpC0t2kiTVzwxJkpJo+lp2BiRJysKSnSRJ9TNDkqQsGp4hGZAkKYuGt31bspMkpWCGJElZWLKTJGVQGh6QLNlJklIwQ5KkLBqeIRmQJCmLhq/UYMlOkpSCGZIkZWHJTpKUQsMDkiU7SVIKZkiSlEQpzc6QDEiSlIUlO0mS6meGJElZNDxDMiBJUhKuZSdJUgJmSJKURcMzJAOSJGXR7KXsLNlJknIwQ5KkJJre1GBAkqQsGh6QLNlJklIwQ5KkLBre1GBAkqQkmn4NyZKdJCkFMyRJysKSnSQpA0t2kiQlYIYkSVlYspMkZVAMSJKkFBoekLyGJElKwQxJkpKwZCdJyqHhAcmSnSQpBQOSJCVR+tu3DSYiLoqIxyLi7paxL0bEvRFxV0R8NyK2annulIjoioj7IuKQlvGp1VhXRJzcMj4pIm6vxi+LiDGDzcmAJElJDGdAAi4Gpq4x9mPgLaWUPwHuB04BiIhdgOnArtU550ZER0R0AF8HDgV2AY6sjgU4C5hVStkZeBqYMdiEDEiS1ECllJuBp9YY+1Eppa/avQ2YUD2eBswtpbxYSlkCdAFTqq2rlPJgKaUXmAtMi4gADgSuqM6/BDh8sDnZ1CBJSSTrsvsYcFn1eDwDAeol3dUYwMNrjO8NjAWWtwS31uPXygxJkrIo0bYtImZGxB0t28x1nUZEfBboA/516D7sHzJDkqQRqJQyG5j9as+LiP8B/DlwUCnlpeXHe4CJLYdNqMZYy/iTwFYR0VllSa3Hr5UZkiQlMcxNDX8gIqYCJwKHlVJWtDw1D5geERtFxCRgMjAfWABMrjrqxjDQ+DCvCmQ3AEdU5x8FXD3Y+5shSVISpT+G7b0iYg5wALBNRHQDpzHQVbcR8OOBvgRuK6X8dSllUURcDixmoJR3bCllVfU6xwHXAR3ARaWURdVbnATMjYjTgYXAhYPO6XcZ2dBY+cSDzb7jlIbVxuP2r3sKapi+3p62RZFH9ntn274vt7/lhuGLbm1ihiRJSSTrsht2BiRJSqKUDS6paSubGiRJKZghSVISluwkSSkMZ5ddRpbsJEkpmCFJUhJD/Cuc9AxIkpSEJTtJkhIwQ5KkJJqeIRmQJCmJpl9DsmQnSUrBDEmSkrBkJ0lKwbXsJElKwAxJkpJwLTtJUgr9luwkSaqfGZIkJdH0pgYDkiQl0fS2b0t2kqQUzJAkKYmmLx1kQJKkJCzZSZKUgBmSJCXR9N8hGZAkKYmmt31bspMkpWCGJElJ2GUnSUqh6deQLNlJklIwQ5KkJJre1GBAkqQkmn4NyZKdJCmFIc+QNh63/1C/hSSNCE1varBkJ0lJNP0akiU7SVIKZkiSlIQlO0lSCg1vsjMgSVIWTc+QvIYkSUrBDEmSkmh6l50BSZKS6K97AjWzZCdJSsEMSZKSKFiykyQl0N/wvm9LdpKkFMyQJCmJfkt2kqQMmn4NyZKdJCkFMyRJSqLpv0MyIElSEpbsJElKwAxJkpKwZCdJSqHpAcmSnSQpBTMkSUqi6U0NBiRJSqK/2fHIkp0kKQczJElKwrXsJEkpNPzuE5bsJEk5mCFJUhJN/x2SAUmSkuiPZl9DsmQnSUrBDEmSkmh6U4MBSZKSaPo1JEt2kqQUDEiSlER/tG9bFxFxQkQsioi7I2JORLwmIiZFxO0R0RURl0XEmOrYjar9rur5HVte55Rq/L6IOGR9P78BSZKS6Cfatg0mIsYDfwvsWUp5C9ABTAfOAmaVUnYGngZmVKfMAJ6uxmdVxxERu1Tn7QpMBc6NiI71+fwGJElqrk5g44joBDYBHgEOBK6onr8EOLx6PK3ap3r+oIiIanxuKeXFUsoSoAuYsj6TMSBJUhKljdug71VKD/Al4CEGAtEzwL8Dy0spfdVh3cD46vF44OHq3L7q+LGt4y9zzqtiQJKkJNp5DSkiZkbEHS3bzNb3ioitGchuJgHjgE0ZKLnVxrZvSRqBSimzgdmvcMi7gCWllMcBIuJK4M+ArSKis8qCJgA91fE9wESguyrxbQk82TL+ktZzXhUzJElKor+N2zp4CNgnIjaprgUdBCwGbgCOqI45Cri6ejyv2qd6/vpSSqnGp1ddeJOAycD8V/3hMUOSpDSGc6WGUsrtEXEF8B9AH7CQgYzqB8DciDi9GruwOuVC4NKI6AKeYqCzjlLKooi4nIFg1gccW0pZtT5zioEAN3Q6x4xv+moYkkawvt6etq2I+q3xH27b9+XRPd/Z4FZqNUOSpCTW9QetI5UBSZKScC07SZISMEOSpCSaniEZkCQpidLwa0iW7CRJKZghSVISluwkSSk0PSBZspMkpWCGJElJNH1ZGwOSJCXR9JUaLNlJklIwQ5KkJJre1GBAkqQkmh6QLNlJklIwQ5KkJOyykySl0PQuOwOSJCXhNSRJkhIwQ5KkJLyGJElKob/hIcmSnSQpBTMkSUqi6U0NBiRJSqLZBTtLdpKkJMyQJCkJS3aSpBSavlKDJTtJUgpmSJKURNN/h2RAkqQkmh2OLNlJkpIwQ5KkJOyykySl0PRrSJbsJEkpmCFJUhLNzo8MSJKURtOvIVmykySlYIYkSUk0vanBgCRJSTQ7HFmykyQlYYYkSUk0vanBgCRJSZSGF+0s2UmSUjBDkqQkLNlJklJoetu3JTtJUgpmSJKURLPzIwOSJKVhyU4pbLnlFlw2dzZ3//ImfnnXjeyz9x4AHPvJo7n7lzfx/+68njM//9maZ6mR6JB3H8Ciu2/m3sW3cOJnjq17OmowM6QkZp39v7nuuhv44PSZjB49mk022ZgD3rEvh73vEN6+x8H09vay7bZj656mRphRo0bxlS//M1PfcyTd3Y9w2y+u4Xvf/xH33POruqfWSE3vsjNDSmCLLTZn//325qJvzQFg5cqVPPPMbznmmI/yhS9+nd7eXgAef/zJOqepEWjKXm/jgQd+zZIlD7Fy5Uouv/xqDnvfIXVPq7FKG//bEK13QIqIo9s5kSabNOmPeOKJJ7nwm7NYMP86zj/vi2yyycZMnrwT++03hVtv+R7X/+QK9txjt7qnqhFm3PjX83D30tX73T2PMG7c62uckZrsv5Ih/WPbZtFwnR0dvO1tb+X887/NXlMO4bnnVnDSicfR2dnB1ltvxb77vY+TTj6dOf92Xt1TlTSE+tu4bYhe8RpSRNy1tqeA7V7hvJnATIDo2JJRozZd7wk2QXfPI3R3P8L8BQsBuPLKH3DiZ46jp/sRrrrqWgAW3HEn/f39bLPNa3niiafqnK5GkKU9y5g4Ydzq/Qnjt2fp0mU1zqjZNtRSW7sM1tSwHXAI8PQa4wHcuraTSimzgdkAnWPGN/tPeB08+ujjdHcv5Y1vfAP33/8ABx64H/fccz8PPPgbDjhgX2686VYmT96JMWPGGIzUVgvuuJOdd57EjjtOpKdnGR/4wDQ+8lE77VSPwQLS94HNSil3rvlERNw4JDNqqONP+BzfvuSrjBkzmiVLHmLGxz/Nc8+t4JsX/At3Lvwpvb0r+diMT9U9TY0wq1at4vhPnco1P/g3OkaN4uJLLmPx4vvrnlZjbailtnaJUoY2gTFDkjSS9fX2RLte6yM7/EXbvi8v/c2VbZvXcLHtW5KUgj+MlaQkml5OMiBJUhKuZSdJUgJmSJKUhL9DkiSl0PS2b0t2kqQUzJAkKYmmNzUYkCQpiaZfQ7JkJ0lKwQxJkpKwqUGSlEIppW3buoqIjohYGBHfr/YnRcTtEdEVEZdFxJhqfKNqv6t6fseW1zilGr8vItb7lsMGJElqtuOBe1r2zwJmlVJ2ZuDWQzOq8RnA09X4rOo4ImIXYDqwKzAVODciOtZnIgYkSUqin9K2bV1ExATgvcA3q/0ADgSuqA65BDi8ejyt2qd6/qDq+GnA3FLKi6WUJUAXMGV9Pr8BSZKSaOctzCNiZkTc0bLNfJm3PAc4kd9dvhoLLC+l9FX73cD46vF44GGA6vlnquNXj7/MOa+KTQ2SlEQ7275b79z9ciLiz4HHSin/HhEHtO2N/wsMSJLUTH8GHBYR7wFeA2wBfBnYKiI6qyxoAtBTHd8DTAS6I6IT2BJ4smX8Ja3nvCqW7CQpieG8hlRKOaWUMqGUsiMDTQnXl1I+BNwAHFEddhRwdfV4XrVP9fz1ZaCdbx4wverCmwRMBuavz+c3Q5KkJF5Nu/YQOgmYGxGnAwuBC6vxC4FLI6ILeIqBIEYpZVFEXA4sBvqAY0spq9bnjWOo/wA6x4xP8ScsSUOhr7cn2vVah048tG3fl9c+fG3b5jVczJAkKYmmr9RgQJKkJFxcVZKkBMyQJCkJ74ckSUohSZddbSzZSZJSMEOSpCQs2UmSUrDLTpKkBMyQJCmJ/oY3NRiQJCmJZocjS3aSpCTMkCQpCbvsJEkpND0gWbKTJKVghiRJSTR96SADkiQlYclOkqQEzJAkKYmmLx1kQJKkJJp+DcmSnSQpBTMkSUqi6U0NBiRJSsKSnSRJCZghSVISluwkSSk0ve3bkp0kKQUzJElKwjvGSpJSsGQnSVICZkiSlIQlO0lSCpbsJElKwAxJkpKwZCdJSsGSnSRJCZghSVISluwkSSlYspMkKQEzJElKopT+uqdQKwOSJCXR9PshWbKTJKVghiRJSRS77CRJGViykyQpATMkSUrCkp0kKYWmr9RgyU6SlIIZkiQl0fSlgwxIkpSE15AkSSnY9i1JUgJmSJKUhCU7SVIKtn1LkpSAGZIkJWHJTpKUgl12kiQlYIYkSUlYspMkpWCXnSRJCZghSVISLq4qSUrBkp0kSQmYIUlSEnbZSZJSaPo1JEt2kqQUDEiSlEQppW3buoiIqRFxX0R0RcTJQ/zxBmXJTpKSGM5rSBHRAXwdOBjoBhZExLxSyuJhm8QazJAkqZmmAF2llAdLKb3AXGBanRMyIElSEqWN2zoYDzzcst9djdVmyEt2fb09MdTvMRJFxMxSyuy656Hm8O9c/dr5fRkRM4GZLUOzs///NUPKa+bgh0ht5d+5EaSUMruUsmfLtmYw6gEmtuxPqMZqY0CSpGZaAEyOiEkRMQaYDsyrc0J22UlSA5VS+iLiOOA6oAO4qJSyqM45GZDySl3r1Yjk37mGKaVcA1xT9zxeEk1fO0mSlIPXkCRJKRiQEsq2nIdGroi4KCIei4i7656LZEBKpmU5j0OBXYAjI2KXemelEexiYGrdk5DAgJRRuuU8NHKVUm4Gnqp7HhIYkDJKt5yHJA0HA5IkKQUDUj7plvOQpOFgQMon3XIekjQcDEjJlFL6gJeW87gHuLzu5Tw0ckXEHOAXwJsiojsiZtQ9JzWXKzVIklIwQ5IkpWBAkiSlYECSJKVgQJIkpWBAkiSlYECSJKVgQJIkpWBAkiSl8P8BAsXJI9ZQ0vsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "latest = RandomForestClassifier(n_estimators= 1000)#, min_samples_leaf= 9, min_samples_split=2, max_depth=25, max_features='auto', criterion= 'entropy',bootstrap=True,oob_score =True)\n",
    "latest.fit(X_train, y_train)\n",
    "pred_latest = latest.predict(X_test)\n",
    "print_score(pred_latest,latest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf4AAAGfCAYAAAC+x7DaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuYXXVh7vHva4BABIMQrANSRiGVEwikZCIFQrhIVXqBUKOQQ2uS2uZw1Fr1oZIW24ZazgNia2rRYixC4UkBo0bQWBCFACIhTCAhAQUvgIrUgkgkJBAS3vPH/o1uJnPZc8ns2Vnv53nyzNq/9but5eC7f2utvUe2iYiIiGp4RbMnEBERESMnwR8REVEhCf6IiIgKSfBHRERUSII/IiKiQhL8ERERFZLgj4iIqJAEf0RERIUk+CMiIipkl2ZPIAZuwoQJbm9vb/Y0IiJiFFm9evVTtvfrr16CvwW1t7fT2dnZ7GlERMQoIumxRurlUn9ERESFJPgjIiIqJMEfERFRIQn+iIiICknwR0REVEie6m9B6x7fQPuC5cPe76MX/f6w9xkREaNLVvwREREVkuCPiIiokJYJfkmvlXStpB9IWi3pa5LmS/pqE+e0QlJH2X5U0oQe6iyUdG4ffVwi6buS7pe0TNLeO3LOERFRbS0R/JIELANW2D7Y9lTgr4HfaO7MhsXNwOG2jwAepnZcERERO0RLBD9wEvCi7cu6CmyvBe4A9pT0hbJqXlLeJCDp7yTdI2m9pMV15SskXSxplaSHJR1fyudK+pKkGyV9T9LHusaS9BZJd0m6V9JSSXv2NVlJ55e+vwW8sa+6tr9ue2t5uRJ43cBPT0RERGNaJfgPB1b3su+3gQ8Ak4A3AMeV8kttT7N9OLAH8Ad1bXax/abS7u/ryqcAZwKTgTMlHVgu338EOMX2UUAn8KHeJippKnBW6ev3gGkDOM4/Bf6rl37nS+qU1Llt04YBdBkREfFrO8PH+VbZ/gmApDVAO/At4CRJHwbGAfsADwBfKW2+VH6uLvW7fNP2htLXg8BBwN7U3lTcWS4a7Abc1cd8jgeW2d5U+rmhkYOQdD6wFVjS037bi4HFAGPbJrqRPiMiIrprleB/AJjVy74X6ra3AbtI2h34NNBh+8eSFgK799BmGy8/B9v1BQi42fbswU+/b5LmUrsi8WbbCfWIiNhhWuVS/y3AWEnzuwokHUFtdd2TrpB/qtyP7+1NQyNWAsdJOqSM+0pJv9VH/duBmZL2kLQX8Id9dS7pbcCHgdO6rhJERETsKC2x4rdtSWcAiySdBzwPPAp8uZf6z0j6LLAe+G/gniGM/WRZkV8jaWwp/gi1J/B7qn+vpOuAtcD/NDD2pcBY4OZyK2Gl7XMGO9+IiIi+KFeWW8/Ytolum7No2PvNV/ZGRLQuSattd/RXryVW/PFykw8YT2dCOiIiBiHBP0IkfYpff9Swy7/YvqIZ84mIiGpK8I8Q2+9t9hwiIiJa5an+iIiIGAYJ/oiIiApJ8EdERFRIgj8iIqJCEvwREREVkuCPiIiokAR/REREhST4IyIiKiTBHxERUSEJ/oiIiArJV/a2oHWPb6B9wfIdOkb+Ul9ExM4pK/6IiIgKSfBHRERUyIgFv6RtktZIWi9pqaRxvdT7mqS9++jnHEnvGsI8/lTSOkn3l7mcXsrnStp/EP3NlDRpsPOJiIgYSSO54t9se4rtw4EtwDn1O1XzCtu/Z/uZ3jqxfZntqwYzAUmvA84Hpts+Avgd4P6yey7QY/BLGtNHtzOBBH9ERLSEZl3qvwM4RFK7pIckXQWsBw6U9KikCQCS3lVW5mslXV3KFko6t2yvkHSxpFWSHpZ0fCkfJ+nzkh6UtEzS3ZI6gNcAzwIbAWxvtP2IpFlAB7CkXJXYo8zjYkn3Au+Q9OeS7ilz+WIZ41jgNOCS0u7g8u9GSasl3SHp0DKngyWtLFcb/lHSxlJ+laSZXSdG0pKuqxARERHDbcSDX9IuwKnAulI0Efi07cNsP1ZX7zDgI8DJto8E/rKXLnex/SbgA8Dfl7L3AL+wPQn4W2BqKV8L/Ax4RNIVkv4QwPYXgE7g7HJVYnOp/3PbR9m+FviS7WllLt8B3m3728ANwF+Vdj8AFgN/YXsqcC7w6dLXvwD/Ynsy8JO6+V9O7WoDksYDxwLbPbIvab6kTkmd2zZt6OVURERE9G0kg38PSWuoBeyPqAUewGO2V/ZQ/2Rgqe2nAGw/3Uu/Xyo/VwPtZXs6cG1pt55yOd/2NuBtwCzgYeATkhb2Mefr6rYPLyv4dcDZwGHdK0vak1pwLy3H+hmgrew+Blhatv+zq43t24CJkvYDZgNftL21e9+2F9vusN0xZtz4PqYcERHRu5H8HP9m21PqCyQBPDfEfl8oP7fRwPHYNrAKWCXpZuAKYGEv1evndiUw0/ZaSXOBE3uo/wrgme7H2YCrgD8GzgLmDbBtREREw0bzx/luoXZvfV8ASfsMoO2dwDtLu0nA5LK9v6Sj6upNAbpuLzwL7NVHn3sBT0jaldqKv8uv2tn+JbXbCO8o40nSkaXeSuDtZfusbn1fSe1WBbYfbOwQIyIiBm7UBr/tB4ALgdskrQX+eQDNPw3sJ+lB4B+BB4ANwK7AxyV9t1yKP5NfPztwJXBZ18N9PfT5t8Dd1N5UfLeu/FrgryTdJ+lgam8K3l3m/ADQ9aDeB4APSbofOKTMp+tYf0btuYErBnCMERERA6bale+dS/n43a62ny9h/A3gjba3NHFO46jd7rCks4DZtk+v27cOOMp2v0/ujW2b6LY5i3bofPOVvRERrUXSatsd/dXbWb+rfxxwa7ksL+A9zQz9YipwqWoPNjwD/CmApFOoPej4iUZCPyIiYih2yhX/zq6jo8OdnZ3NnkZERIwija74R+09/oiIiBh+Cf6IiIgKSfBHRERUSII/IiKiQhL8ERERFZLgj4iIqJAEf0RERIUk+CMiIiokwR8REVEhCf6IiIgKSfBHRERUyM76R3p2ause30D7guUjOmb+Wl9ExM4hK/6IiIgKSfBHRERUSII/IiKiQkY8+CW9VtK1kn4gabWkr0n6rQH2MVPSpB01xz7GXSFpu791LGmupEv7aDdD0r2Stkqa1W3fHEnfK//m7Ih5R0REdBnR4JckYBmwwvbBtqcCfw38xgC7mgmMaPBLGjOE5j8C5gL/2a3PfYC/B44G3gT8vaRXD2GciIiIPo30iv8k4EXbl3UV2F4LjJH01a4ySZdKmlu2L5L0oKT7JX1c0rHAacAlktZIOljSFEkrS51lXeFZVuifkNQp6TuSpkn6Ulld/2PdeH8saVXp7zNdIS9po6R/krQWOKb+QCTNk/SwpFXAcX0dtO1Hbd8PvNRt11uBm20/bfsXwM3A2wZ2SiMiIho30sF/OLC60cqS9gXOAA6zfQTwj7a/DdwA/JXtKbZ/AFwFnFfqrKO2iu6yxXYHcBlwPfDeMo+5kvaV9L+AM4HjbE8BtgFnl7avBO62faTtb9XNqw24gFrgT2fwVx8OAH5c9/onpaynczG/vIHp3LZpwyCHi4iIqhvtD/dtAJ4HLpf0R8Cm7hUkjQf2tn1bKfoPYEZdlRvKz3XAA7afsP0C8EPgQODNwFTgHklryus3lDbbgC/2MK+jqd2ueNL2FuC6IRxjQ2wvtt1hu2PMuPE7eriIiNhJjXTwP0AtZLvb2m0uuwPY3krt3vcXgD8AbhzEmC+Uny/VbXe93gUQ8B/l6sEU22+0vbDUed72tkGM2ajHqb356PK6UhYREbFDjHTw3wKMlTS/q0DSEdTCd5KksZL2prbqRtKewHjbXwM+CBxZmj0L7AVgewPwC0nHl31/AnSt/hvxTWCWpNeUMfeRdFA/be4GTii3CnYF3jGA8erdBLxF0qvLcwlvKWURERE7xIh+Za9tSzoDWCTpPGqX8R8FPgB8HlgPPALcV5rsBVwvaXdqbw4+VMqvBT4r6f3ALGAOcJmkcdQu4c8bwJwelPQR4OuSXgG8SO05gMf6aPOEpIXAXcAzwJq+xpA0jdqnGV4N/KGkC2wfZvtpSR8F7ilV/8H2043OPSIiYqBku9lziAEa2zbRbXMWjeiY+a7+iIjRTdLq8jB7n/JHelrQ5APG05kgjoiIQUjwDyNJ57P9/f6lti9sxnwiIiK6S/APoxLwCfmIiBi1Rvvn+CMiImIYJfgjIiIqJMEfERFRIQn+iIiICknwR0REVEiCPyIiokIS/BERERWS4I+IiKiQBH9ERESFJPgjIiIqJF/Z24LWPb6B9gXLR3zc/IW+iIjWlxV/REREhST4IyIiKqSlgl/STEmWdGiT53GipGPrXi+UdG4P9dolrW+gv9+UtLGnPiIiIoZTSwU/MBv4VvnZTCcCx/ZXaQD+GfivYewvIiKiRy0T/JL2BKYD7wbOqis/T9I6SWslXVTKDpH0jVJ2r6SDVXOJpPWl/pml7omSvlrX36WS5pbtRyVdUPpYJ+lQSe3AOcAHJa2RdHy3eU4t464F3tvAcc0EHgEeGNIJioiIaEArPdV/OnCj7Ycl/VzSVOA1pfxo25sk7VPqLgEusr1M0u7U3uD8ETAFOBKYANwj6fYGxn3K9lGS3gOca/vPJF0GbLT9cQBJb66rfwXwPtu3S7qkr47Lm5nzgN8F+rzML2k+MB9gzKv2a2DaERER22uZFT+1y/vXlu1ry+tTgCtsbwKw/bSkvYADbC8rZc+X/dOBa2xvs/0z4DZgWgPjfqn8XA2091VR0t7A3ra73lBc3U/fC4FP2N7Y3yRsL7bdYbtjzLjx/VWPiIjoUUus+MtK/mRgsiQDYwADS4eh+628/A3Q7t32v1B+bmP4z9fRwCxJHwP2Bl6S9LztS4d5nIiICKB1VvyzgKttH2S73faB1O6LbwDmSRoHtTcItp8FflLunSNpbNl/B3CmpDGS9gNmAKuAx4BJpd7ewJu3H347zwJ7dS+0/QzwjKTppejsvjqxfXw5nnZgEfD/EvoREbEjtUrwzwaWdSv7ItAG3AB0SlrDr++T/wnwfkn3A98GXlva3w+sBW4BPmz7v23/GPg8sL78vK+B+XwFOKOnh/uAecCnynw0sMOMiIjYsWS72XOIARrbNtFtcxaN+Lj5yt6IiNFL0mrbHf3Va4l7/PFykw8YT2dCOCIiBiHBPwIkvRW4uFvxI7bPaMZ8IiKiuhL8I8D2TcBNzZ5HREREqzzcFxEREcMgwR8REVEhCf6IiIgKSfBHRERUSII/IiKiQhL8ERERFZLgj4iIqJAEf0RERIUk+CMiIiokwR8REVEh+creFrTu8Q20L1je7Gn0KH/BLyJidMuKPyIiokIS/BERERWS4I+IiKiQlgp+STMlWdKhTZ7HiZKOrXu9UNK5PdRrl7S+j37OlrSm7t9LkqbsqHlHRES0VPADs4FvlZ/NdCJwbH+V+mN7ie0ptqcAfwI8YnvNUPuNiIjoTcsEv6Q9genAu4Gz6srPk7RO0lpJF5WyQyR9o5TdK+lg1VwiaX2pf2ape6Kkr9b1d6mkuWX7UUkXlD7WSTpUUjtwDvDBsko/vts8p5Zx1wLvHcAhzgau7eP450vqlNS5bdOGAXQbERHxa630cb7TgRttPyzp55KmAq8p5Ufb3iRpn1J3CXCR7WWSdqf2BuePgCnAkcAE4B5Jtzcw7lO2j5L0HuBc238m6TJgo+2PA0h6c139K4D32b5d0iUDOL4zy7H0yPZiYDHA2LaJHkC/ERERv9IyK35eviK+trw+BbjC9iYA209L2gs4wPayUvZ82T8duMb2Nts/A24DpjUw7pfKz9VAe18VJe0N7G276w3F1Y0cmKSjgU22e30eICIiYji0xIq/rORPBiZLMjAGMLB0GLrfysvfAO3ebf8L5ec2dtz5Ogu4Zgf1HRER8SutsuKfBVxt+yDb7bYPBB4BNgDzJI2D2hsE288CP5E0s5SNLfvvAM6UNEbSfsAMYBXwGDCp1NsbePP2w2/nWWCv7oW2nwGekTS9FJ3dX0eSXgG8kz7u70dERAyXVgn+2cCybmVfBNqAG4BOSWuAro/U/Qnwfkn3A98GXlva3w+sBW4BPmz7v23/GPg8sL78vK+B+XwFOKOnh/uAecCnynzUQF8zgB/b/mEDdSMiIoZEdp4TazVj2ya6bc6iZk+jR/mu/oiI5pC02nZHf/Va4h5/vNzkA8bTmYCNiIhBSPCPAElvBS7uVvyI7TOaMZ+IiKiuBP8IsH0TcFOz5xEREdEqD/dFRETEMEjwR0REVEiCPyIiokIS/BERERWS4I+IiKiQBH9ERESFJPgjIiIqJMEfERFRIQn+iIiICsk397WgdY9voH3B8mZPo1f5Qz0REaNXVvwREREVkuCPiIiokAR/REREheyQ4Jd0vqQHJN0vaY2ko/uou1DSucM8foekT/ZTp13S/x5Im37629hL+ZWSZvXRTpIulPSwpO9Iev9g5xAREdGfYX+4T9IxwB8AR9l+QdIEYLfhHqeP8Xex3Ql09lO1HfjfwH8CNNhmR5gLHAgcavslSa9pwhwiIqIidsSKvw14yvYLALafsv1TSY+WNwFdq+sVdW2OlHSXpO9J+vNSp03S7eWKwXpJx5fyt0m6V9JaSd8sZQslXS3pTuBqSSdK+mq3fS/rH7gIOL70/8FubfaR9OVyxWKlpCPq+vqcpBWSftjT6rys4C+V9JCkbwD9Bfn/Bf7B9kvlfP3PgM94REREg3bEx/m+DvydpIeBbwDX2b6tnzZHAL8DvBK4T9JyYDZwk+0LJY0BxknaD/gsMMP2I5L2qetjEjDd9mZJJzbQ/wLgXNt/ANCtzQXAfbZnSjoZuAqYUvYdCpwE7AU8JOnfbL9Y1/YM4I1lPr8BPAh8ro9jPxg4U9IZwJPA+21/r3slSfOB+QBjXrVfH91FRET0bthX/LY3AlOphdSTwHWS5vbT7Hrbm20/BdwKvAm4B5gnaSEw2faz1ML7dtuPlLGeruvjBtubB9B/X6YDV5cxbgH2lfSqsm+57RdKX/9DLdzrzQCusb3N9k+BW/oZayzwvO0Oam9qenyTYHux7Q7bHWPGje+ny4iIiJ7tkIf7SuitsP33wPuAtwNb68bbvXuT7bvw7dRC9HHgSknv6mfY5/qaUj+vB+KFuu1tDP2qyU+AL5XtZdSuTkREROwQwx78kt4oaWJd0RTgMeBRalcCoPZGoN7pknaXtC9wInCPpIOAn9n+LPDvwFHASmCGpNeXsfahMdv1DzxL7XJ9T+4Azi5jnEjtmYVfNjjW7dQu3Y+R1EbttkBfvlxX5wTg4QbHiYiIGLAdcY9/T+BfJe1NbZX/fWqX/f8XcLmkjwIrurW5n9ol+AnAR8vDgHOAv5L0IrAReJftJ8u97i9JegW1S+2/28Cceur/SWCbpLXAlcB9dfUXAp+TdD+wCZgzgONfBpxM7d7+j4C7+ql/EbBE0gepHeefDWCsiIiIAZE9lKveo195RmCj7Y83ey7DZWzbRLfNWdTsafQq39UfETHyJK0uz4v1KX+kpwVNPmA8nQnXiIgYhJ0++G0vbPYcACQtA17frfg82zc1Yz4REVFNO33wjxa2z2j2HCIiIvJHeiIiIiokwR8REVEhCf6IiIgKSfBHRERUSII/IiKiQhL8ERERFZLgj4iIqJAEf0RERIUk+CMiIiokwR8REVEh+creFrTu8Q20L1je7GkMq/xFv4iIkZEVf0RERIUk+CMiIiqk5YJf0jZJayStl7RU0rgh9DVX0qVDaLt/3esVkjoGOkbZ/2Q5pjWS/mww84mIiGhEywU/sNn2FNuHA1uAc+p3qmYkjmsusH9/lRp0XTmmKbb/fZj6jIiI2E4rBn+9O4BDJLVLekjSVcB64EBJsyWtK1cGLu5qIGmepIclrQKOqyu/UtKsutcb67bPK32tlXRRqdcBLCmr9D3qJ9XbGBEREc3WssEvaRfgVGBdKZoIfNr2YcCLwMXAycAUYJqkmZLagAuohfF0YFID45wKnA4cbftI4GO2vwB0AmeXVfrmuvoDHgN4u6T7JX1B0oG9zGO+pE5Jnds2bWigy4iIiO21YvDvIWkNteD9EXB5KX/M9sqyPQ1YYftJ21uBJcAM4Oi68i3AdQ2Mdwpwhe1NALaf7qf+QMf4CtBu+wjgZuA/eqpke7HtDtsdY8aNb2DaERER22vFz/Fvtj2lvkASwHND7Hcr5Y1QeUZgtyH21xDbP697+e/Ax0Zi3IiIqKZWXPE3YhVwgqQJksYAs4HbgLtL+b6SdgXeUdfmUWBq2T4N2LVs3wzM6/r0gKR9SvmzwF49jN3XGNsptwa6nAZ8p7FDjIiIGLhWXPH3y/YTkhYAtwICltu+HkDSQuAu4BlgTV2zzwLXS1oL3Ei5gmD7RklTgE5JW4CvAX8DXAlcJmkzcEy3sXsboyfvl3QatSsOT1P7tEBERMQOIdvNnkMM0Ni2iW6bs6jZ0xhW+creiIihkbTa9nbfJ9PdznqpPyIiInqwU17qH40knc/29/uX2r5woH1NPmA8nVkhR0TEICT4R0gJ+AGHfERExHDKpf6IiIgKSfBHRERUSII/IiKiQhL8ERERFZLgj4iIqJAEf0RERIUk+CMiIiokwR8REVEhCf6IiIgKSfBHRERUSL6ytwWte3wD7QuWN3saIyZ/uS8iYvhkxR8REVEhCf6IiIgKSfBHRERUSL/BL2mbpDWS1ktaKmncYAeTNFfSpUNou3/d610lXSTpe5LulXSXpFP76eMDQ5n/jiDpdEn3l3PcKWl6s+cUERE7r0ZW/JttT7F9OLAFOKd+p2pG4srBXGD/utcfBdqAw20fBcwE9uqnjw8AOzT4JQ30gclvAkfangL8KfDvwz+riIiImoEG9h3AIZLaJT0k6SpgPXCgpNmS1pUrAxd3NZA0T9LDklYBx9WVXylpVt3rjXXb55W+1pZV/SygA1hSVsavBP4c+AvbLwDY/pntz5f2/1ZWzw9IuqCUvZ/aG4dbJd1ayt5SrhTcW65m7FnKf0/SdyWtlvRJSV8t5ftI+nJZoa+UdEQpXyjpakl3AldLul3SlLrj+ZakI3s6obY32nZ5+UrAPdWLiIgYDg0Hf1nJngqsK0UTgU/bPgx4EbgYOBmYAkyTNFNSG3ABtcCfDkxqYJxTgdOBo20fCXzM9heATuDssjI+GPiR7V/20s35tjuAI4ATJB1h+5PAT4GTbJ8kaQLwEeCUcsWgE/iQpN2BzwCn2p4K7FfX7wXAfbaPAP4GuKpu36TS12zgcmpXKJD0W8Duttf2ccxnSPousJzaqr+nOvPLm5nObZs29NZVREREnxoJ/j0kraEWjD+iFmoAj9leWbanAStsP2l7K7AEmAEcXVe+BbiugfFOAa6wvQnA9tONH86vvFPSvcB9wGH0/Ibjd0r5neX45gAHAYcCP7T9SKl3TV2b6cDVZV63APtKelXZd4PtzWV7KfAHknalFuRX9jVZ28tsH0rtdsVHe6mz2HaH7Y4x48b31V1ERESvGrkfvbmssn9FEsBzQxx7K+WNR3lGYLcBtP0+8JuSXtV91S/p9cC5wDTbv5B0JbB7D30IuLms0OvbT+mhbiN+dT5sb5J0M7UrF+8EpjbSge3bJb1B0gTbTw1yHhEREb0arofyVlG7pD5B0hhgNnAbcHcp37esft9R1+ZRfh2IpwG7lu2bgXldT99L2qeUP0t5eK9cDbgc+BdJu5V6+0l6B/AqaiG8QdJvULs9Qfc+gJXAcZIOKe1fWS7LPwS8QVJ7qXdmXfs7gLNL/ROBp/q43fDvwCeBe2z/opc6SDpE5Z2UpKOAscDPe6sfERExFMPylb22n5C0ALiV2kp6ue3rofbgG3AX8Aywpq7ZZ4HrJa0FbqSsmG3fWFbdnZK2AF+jdj/9SuAySZuBY6jdn/9H4EFJz5f2f2d7raT7gO8CPwburBtzMXCjpJ+W+/xzgWskjS37P2L7YUnvKfWeA+6pa78Q+Jyk+4FN1G4P9HZOVkv6JXBFP6fv7cC7JL0IbAbOrHvYLyIiYlgpGbM9SXva3lhW4p8Cvmf7EwPsY39gBXCo7ZeGc35j2ya6bc6i4exyVMt39UdE9E/S6vJge5/yR3p69ueS5lB77uA+ak/5N0zSu4ALgQ8Nd+gDTD5gPJ0Jw4iIGIQEfw/K6n5AK/xu7a/i5R/1Q9I84C+7Vb3T9nsHO05ERMRAJfhHiO0r6P9+f0RExA6VP9ITERFRIQn+iIiICknwR0REVEiCPyIiokIS/BERERWS4I+IiKiQBH9ERESFJPgjIiIqJMEfERFRIQn+iIiICslX9ragdY9voH3B8mZPY0TlL/RFRAyPrPgjIiIqJMEfERFRIS0X/JK2SVojab2kpZLGDaGvuZIuHULb/eter5DUMdAxJM2QdK+krZJmDWYuERERjWq54Ac2255i+3BgC3BO/U7VjMRxzQX2769SA35U+vrPYegrIiKiT60Y/PXuAA6R1C7pIUlXAeuBAyXNlrSuXBm4uKuBpHmSHpa0CjiurvzK+hW3pI112+eVvtZKuqjU6wCWlKsPe9RPqrcxemL7Udv3Ay8N7VRERET0r2Wf6pe0C3AqcGMpmgjMsb2yXIK/GJgK/AL4uqSZwN3ABaV8A3ArcF8/45wKnA4cbXuTpH1sPy3pfcC5tjtLva76bQMdo8HjnQ/MBxjzqv2G2l1ERFRUK67495C0Buikdpn88lL+mO2VZXsasML2k7a3AkuAGcDRdeVbgOsaGO8U4ArbmwBsP91P/cGM0S/bi2132O4YM278cHQZEREV1Ior/s22p9QXlNX2c0PsdyvljVB5RmC3IfYXEREx6rTiir8Rq4ATJE2QNAaYDdxG7VL/CZL2lbQr8I66No9SuzwPcBqwa9m+GZjX9ekBSfuU8meBvXoYu68xIiIimmqnDH7bTwALqN1fXwustn19KV8I3AXcCXynrtlnqQX2WuAYyhUE2zcCNwCd5RbDuaX+lcBl3R/u62eM7UiaJukn1N4gfEbSA4M/8oiIiL7JdrPnEAM0tm2i2+YsavY0RlS+sjciom+SVtve7vtkumvFe/yVN/mA8XQmCCPnHEQCAAAVJElEQVQiYhAS/CNE0vlsf79/qe0LmzGfiIiopgT/CCkBn5CPiIim2ikf7ouIiIieJfgjIiIqJMEfERFRIQn+iIiICknwR0REVEiCPyIiokIS/BERERWS4I+IiKiQBH9ERESFJPgjIiIqJF/Z24LWPb6B9gXLmz2NUSN/uS8ionFZ8UdERFRIgj8iIqJCEvwREREV0nLBL2mbpDWS1ktaKmncEPqaK+nSIbTdv+71Ckkdgx1D0tsluac+IiIihkvLBT+w2fYU24cDW4Bz6neqZiSOay6wf3+VGiFpL+AvgbuHo7+IiIjetGLw17sDOERSu6SHJF0FrAcOlDRb0rpyZeDirgaS5kl6WNIq4Li68islzap7vbFu+7zS11pJF5V6HcCScvVhj/pJ9TZGHz4KXAw831sFSfMldUrq3LZpQwNdRkREbK9lg1/SLsCpwLpSNBH4tO3DgBepBenJwBRgmqSZktqAC6iF8XRgUgPjnAqcDhxt+0jgY7a/AHQCZ5erD5vr6g9oDElHAQfa7vPzebYX2+6w3TFm3Pj+ph0REdGjVvwc/x6S1pTtO4DLqV1yf8z2ylI+DVhh+0kASUuAGWVfffl1wG/1M94pwBW2NwHYfrqf+kc3Oka5JfHP1G4bRERE7HCtGPybbU+pL5AE8NwQ+91KuQJSAnm3IfbXiL2Aw4EV5RheC9wg6TTbnSMwfkREVEzLXurvxyrgBEkTJI0BZgO3UXt47gRJ+0raFXhHXZtHgall+zRg17J9MzCv69MDkvYp5c9SC+7u+hrjZWxvsD3BdrvtdmAlkNCPiIgdphVX/P2y/YSkBcCtgIDltq8HkLQQuAt4BlhT1+yzwPWS1gI3Uq4g2L5R0hSgU9IW4GvA3wBXApdJ2gwc023s3saIiIhoKtlu9hxigMa2TXTbnEXNnsaoke/qj4gASatt9/tdMDvlin9nN/mA8XQm7CIiYhAS/CNE0vlsf79/qe0LmzGfiIiopgT/CCkBn5CPiIim2lmf6o+IiIgeJPgjIiIqJMEfERFRIQn+iIiICknwR0REVEiCPyIiokIS/BERERWS4I+IiKiQBH9ERESF5Jv7WtC6xzfQvmB5s6cxquUP90RE9Cwr/oiIiApJ8EdERFRIgj8iIqJCWi74JW2TtEbSeklLJY0bQl9zJV06hLb7171eIaljoGNIOkfSunJM35I0aTDziYiIaETLBT+w2fYU24cDW4Bz6neqZiSOay6wf3+VGvCftifbngJ8DPjnYegzIiKiR60Y/PXuAA6R1C7pIUlXAeuBAyXNLivp9ZIu7mogaZ6khyWtAo6rK79S0qy61xvrts8rfa2VdFGp1wEsKSv1Peon1dsYPbH9y7qXrwQ8qDMRERHRgJb9OJ+kXYBTgRtL0URgju2V5RL8xcBU4BfA1yXNBO4GLijlG4Bbgfv6GedU4HTgaNubJO1j+2lJ7wPOtd1Z6nXVbxvEGO8FPgTsBpzcS535wHyAMa/ar6/uIiIietWKK/49JK0BOoEfAZeX8sdsryzb04AVtp+0vRVYAswAjq4r3wJc18B4pwBX2N4EYPvpfuoPeAzbn7J9MHAe8JFe6iy23WG7Y8y48Q1MOyIiYnutuOLfXO6H/0pZbT83xH63Ut4IlWcEdhtif4NxLfBvTRg3IiIqohVX/I1YBZwgaYKkMcBs4DZql/pPkLSvpF2Bd9S1eZTa5XmA04Bdy/bNwLyuTw9I2qeUPwvs1cPYfY2xHUkT617+PvC9xg4xIiJi4Fpxxd8v209IWkDt/rqA5bavB5C0ELgLeAZYU9fss8D1ktZSe27gudLXjZKmAJ2StgBfA/4GuBK4TNJm4JhuY/c2Rk/eJ+kU4EVqzyPMGfyRR0RE9E12HiJvNWPbJrptzqJmT2NUy3f1R0TVSFpte7vvk+lup1zx7+wmHzCezgRbREQMQoJ/hEg6n+3v9y+1fWEz5hMREdWU4B8hJeAT8hER0VQ761P9ERER0YMEf0RERIUk+CMiIiokwR8REVEhCf6IiIgKSfBHRERUSII/IiKiQhL8ERERFZLgj4iIqJAEf0RERIXkK3tb0LrHN9C+YHmzpzHq5S/0RURsLyv+iIiICknwR0REVEhLBb+kmZIs6dAmz+NEScfWvV4o6dwe6rVLWt9HP78rabWkdeXnyTtqzhEREdBiwQ/MBr5VfjbTicCx/VVqwFPAH9qeDMwBrh6GPiMiInrVMsEvaU9gOvBu4Ky68vPKinmtpItK2SGSvlHK7pV0sGoukbS+1D+z1D1R0lfr+rtU0tyy/aikC0of6yQdKqkdOAf4oKQ1ko7vNs+pZdy1wHv7Oibb99n+aXn5ALCHpLFDOlERERF9aKWn+k8HbrT9sKSfS5oKvKaUH217k6R9St0lwEW2l0nandobnD8CpgBHAhOAeyTd3sC4T9k+StJ7gHNt/5mky4CNtj8OIOnNdfWvAN5n+3ZJlwzg+N4O3Gv7hZ52SpoPzAcY86r9BtBtRETEr7XMip/a5f1ry/a15fUpwBW2NwHYflrSXsABtpeVsufL/unANba32f4ZcBswrYFxv1R+rgba+6ooaW9gb9tdbygaunQv6TDgYuD/9FbH9mLbHbY7xowb30i3ERER22mJFX9ZyZ8MTJZkYAxgYOkwdL+Vl78B2r3b/q4V+DZ2wPmS9DpgGfAu2z8Y7v4jIiLqtcqKfxZwte2DbLfbPhB4BNgAzJM0DmpvEGw/C/xE0sxSNrbsvwM4U9IYSfsBM4BVwGPApFJvb+DN2w+/nWeBvboX2n4GeEbS9FJ0dl+dlPGWAwts39nAuBEREUPSKsE/m9qquN4XgTbgBqBT0hqg6yN1fwK8X9L9wLeB15b29wNrgVuAD9v+b9s/Bj4PrC8/72tgPl8Bzujp4T5gHvCpMh/108/7gEOAvyt9rZH0mgbGj4iIGBTZbvYcYoDGtk1025xFzZ7GqJev7I2IKpG02nZHf/Va4h5/vNzkA8bTmVCLiIhBSPCPAElvpfbUfr1HbJ/RjPlERER1JfhHgO2bgJuaPY+IiIhWebgvIiIihkGCPyIiokIS/BERERWS4I+IiKiQBH9ERESFJPgjIiIqJMEfERFRIQn+iIiICknwR0REVEiCPyIiokLylb0taN3jG2hfsLzZ04hRKn+VMCL6khV/REREhST4IyIiKiTBHxERUSEtEfyStklaI2m9pKWSxg1Tv9/upfxKSbMG2NdCSef2UN4uaX0f7d5Ujm2NpLWSzhjIuBEREQPREsEPbLY9xfbhwBbgnPqdqhnwsdg+djgmJ2koD0muBzpsTwHeBnxmiP1FRET0qlWCv94dwCFlJf2QpKuoheeBkmZLWleuDFwMIOkcSZd0NZY0V9KlZXtj+SlJl5b+vgG8pq7+VEm3SVot6SZJbaV8haRFkjqBv6yfYGmzVtJa4L19HYztTba3lpe7A+6pnqT5kjoldW7btGEg5ysiIuJXWir4y0r4VGBdKZoIfNr2YcCLwMXAycAUYJqkmcAXgfrL52cC13br+gzgjcAk4F3AsWW8XYF/BWbZngp8Driwrt1utjts/1O3/q4A/sL2kQ0e19GSHijHdU7dG4Ffsb24jNUxZtz4RrqNiIjYTqsE/x6S1gCdwI+Ay0v5Y7ZXlu1pwArbT5bgXALMsP0k8ENJvyNpX+BQ4M5u/c8ArrG9zfZPgVtK+RuBw4Gby/gfAV5X1+667hOVtDewt+3bS9HV/R2c7bvLm5dpwF9L2r2/NhEREYPRKveSN5d74L8iCeC5BttfC7wT+C6wzHaPl9N7IOAB28f0sr/R8Rti+zvl9sPh1N7kREREDKtWWfE3YhVwgqQJksYAs4Hbyr5lwOmlrPtlfoDbgTMljSn38E8q5Q8B+0k6BmqX/iUd1tckbD8DPCNpeik6u6/6kl7f9TCfpIOoXZF4tM8jjYiIGKRWWfH3y/YTkhYAt1JbqS+3fX3Z9wtJ3wEm2V7VQ/Nl1J4NeJDarYS7Srst5WN9n5Q0ntr5WgQ80M905gGfk2Tg6/3UnQ4skPQi8BLwHttP9X/EERERA6fGr3rHaDG2baLb5ixq9jRilMp39UdUk6TVtjv6q7fTrPirZPIB4+nM/7lHRMQgJPhHiKS3Uvu4Yb1HbOeb+iIiYsQk+EeI7ZuAm5o9j4iIqLad6an+iIiI6EeCPyIiokIS/BERERWS4I+IiKiQBH9ERESFJPgjIiIqJMEfERFRIQn+iIiICknwR0REVEi+ua8FrXt8A+0Lljd7GrGTyB/1iaiWrPgjIiIqJMEfERFRIQn+iIiICmmJ4Je0TdIaSeslLZU0bpj6/XYv5VdKmjXAvhZKOreH8nZJ6/tot6+kWyVtlHTpQMaMiIgYqJYIfmCz7Sm2Dwe2AOfU71TNgI/F9rHDMTlJQ3lI8nngb4Ht3jREREQMt1YJ/np3AIeUlfRDkq4C1gMHSpotaV25MnAxgKRzJF3S1VjS3K6VtaSN5ackXVr6+wbwmrr6UyXdJmm1pJsktZXyFZIWSeoE/rJ+gqXNWklrgff2dTC2n7P9LWpvACIiInaolgr+srI+FVhXiiYCn7Z9GPAicDFwMjAFmCZpJvBF4Iy6bs4Eru3W9RnAG4FJwLuAY8t4uwL/CsyyPRX4HHBhXbvdbHfY/qdu/V0B/IXtI4dwuC8jab6kTkmd2zZtGK5uIyKiYlol+PeQtAboBH4EXF7KH7O9smxPA1bYftL2VmAJMMP2k8APJf2OpH2BQ4E7u/U/A7jG9jbbPwVuKeVvBA4Hbi7jfwR4XV2767pPVNLewN62by9FVw/+sH/N9uLyJqNjzLjxw9FlRERUUKt8gc9m21PqCyQBPNdg+2uBdwLfBZbZdoPtBDxg+5he9jc6fkRExKjQKiv+RqwCTpA0QdIYYDZwW9m3DDi9lHW/zA9wO3CmpDHlHv5JpfwhYD9Jx0Dt0r+kw/qahO1ngGckTS9FZw/loCIiIoZTq6z4+2X7CUkLgFuprdSX276+7PuFpO8Ak2yv6qH5MmrPBjxI7VbCXaXdlvKxvk9KGk/tfC0CHuhnOvOAz0ky8PX+5i7pUeBVwG7luYS32H6wv3YREREDpcavesdoMbZtotvmLGr2NGInke/qj9g5SFptu6O/ejvNir9KJh8wns78n3VERAxCgn+ESHortY8b1nvE9hk91Y+IiNgREvwjxPZNwE3NnkdERFTbzvRUf0RERPQjwR8REVEhCf6IiIgKSfBHRERUSII/IiKiQhL8ERERFZLgj4iIqJAEf0RERIUk+CMiIiokwR8REVEh+creFrTu8Q20L1je7GlERMQQNeOvY2bFHxERUSEJ/oiIiApJ8DdI0q3lT+vWl31A0r/10Wbjjp9ZRERE4xL8jbsGOKtb2VmlPCIioiUk+Bv3BeD3Je0GIKkd2B+4T9I3Jd0raZ2k07s3lHSipK/Wvb5U0tyyPVXSbZJWS7pJUttIHExERFRTgr9Btp8GVgGnlqKzgM8Dm4EzbB8FnAT8kyQ10qekXYF/BWbZngp8Driwl7rzJXVK6ty2acPQDiYiIiorH+cbmK7L/deXn+8GBPw/STOAl4ADgN8A/ruB/t4IHA7cXN4rjAGe6Kmi7cXAYoCxbRM9pKOIiIjKSvAPzPXAJyQdBYyzvbpcst8PmGr7RUmPArt3a7eVl19d6dov4AHbx+zYaUdERNTkUv8A2N4I3ErtknzXQ33jgf8poX8ScFAPTR8DJkkaK2lv4M2l/CFgP0nHQO3Sv6TDduhBREREpWXFP3DXAMv49RP+S4CvSFoHdALf7d7A9o8lfR5YDzwC3FfKt0iaBXxS0nhq/3ssAh7Y4UcRERGVlOAfINtfpnaJvuv1U0CPl+pt71m3/WHgwz3UWQPMGP6ZRkREbC/B34ImHzCeziZ8v3NERLS+3OOPiIiokAR/REREhST4IyIiKiTBHxERUSEJ/oiIiApJ8EdERFSI7Hzte6uR9Cy1b/2LwZkAPNXsSbSwnL+hyzkcmpy/nh1ke7/+KuVz/K3pIdsdzZ5Eq5LUmfM3eDl/Q5dzODQ5f0OTS/0REREVkuCPiIiokAR/a1rc7Am0uJy/ocn5G7qcw6HJ+RuCPNwXERFRIVnxR0REVEiCf5SR9DZJD0n6vqQFPewfK+m6sv9uSe11+/66lD8k6a0jOe/RYrDnT1K7pM2S1pR/l4303EeDBs7fDEn3StoqaVa3fXMkfa/8mzNysx49hnj+ttX9/t0wcrMePRo4fx+S9KCk+yV9U9JBdfsq//vXMNv5N0r+AWOAHwBvAHYD1gKTutV5D3BZ2T4LuK5sTyr1xwKvL/2MafYxtdD5awfWN/sYWuD8tQNHAFcBs+rK9wF+WH6+umy/utnH1Crnr+zb2OxjaIHzdxIwrmz/37r/fiv/+zeQf1nxjy5vAr5v+4e2twDXAqd3q3M68B9l+wvAmyWplF9r+wXbjwDfL/1VyVDOXzRw/mw/avt+4KVubd8K3Gz7adu/AG4G3jYSkx5FhnL+orHzd6vtTeXlSuB1ZTu/fwOQ4B9dDgB+XPf6J6Wsxzq2twIbgH0bbLuzG8r5A3i9pPsk3Sbp+B092VFoKL9D+f0b+jnYXVKnpJWSZg7v1FrCQM/fu4H/GmTbSss390XUPAH8pu2fS5oKfFnSYbZ/2eyJRWUcZPtxSW8AbpG0zvYPmj2p0UjSHwMdwAnNnksryop/dHkcOLDu9etKWY91JO0CjAd+3mDbnd2gz1+5RfJzANurqd1r/K0dPuPRZSi/Q/n9G+I5sP14+flDYAXw28M5uRbQ0PmTdApwPnCa7RcG0jZqEvyjyz3AREmvl7QbtYfPuj/dewPQ9cTqLOAW155uuQE4qzy1/npgIrBqhOY9Wgz6/EnaT9IYgLLimkjtAaEqaeT89eYm4C2SXi3p1cBbSlmVDPr8lfM2tmxPAI4DHtxhMx2d+j1/kn4b+Ay10P+ful35/RuIZj9dmH8v/wf8HvAwtRXn+aXsH6j9ogPsDiyl9vDeKuANdW3PL+0eAk5t9rG00vkD3g48AKwB7gX+sNnHMkrP3zRq90+fo3al6YG6tn9azuv3gXnNPpZWOn/AscA6ak+yrwPe3exjGaXn7xvAz8p/p2uAG+raVv73r9F/+ea+iIiICsml/oiIiApJ8EdERFRIgj8iIqJCEvwREREVkuCPiIiokAR/REREhST4IyIiKiTBHxERUSH/H7N1qPS30MNUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#let's see the feature importances\n",
    "plt.figure(figsize=(7,7))\n",
    "feat_importances = pd.Series(latest.feature_importances_, index=train.columns)\n",
    "feat_importances.nlargest(15).plot(kind='barh')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value                0.231963\n",
       "ProviderId_1         0.092479\n",
       "ProviderId_3         0.084861\n",
       "AccountId_2          0.046184\n",
       "ProductId_3          0.045509\n",
       "ProductId_4          0.042175\n",
       "ProductId_1          0.038053\n",
       "ProductCategory_3    0.037571\n",
       "ProductId_5          0.033220\n",
       "SubscriptionId_6     0.024726\n",
       "AccountId_7          0.024056\n",
       "AccountId_4          0.018465\n",
       "CustomerId_10        0.018218\n",
       "PricingStrategy      0.015821\n",
       "ChannelId_2          0.013839\n",
       "dtype: float64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_importances.nlargest(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are going to modify the X_train set and X_test test to make sure we get out only the useful features and work with those.\n",
    "\n",
    "### Important Features\n",
    "\n",
    "Let's use the most important features in the model and see if our scores increase\n",
    "\n",
    "This [Link](https://chrisalbon.com/machine_learning/trees_and_forests/feature_selection_using_random_forest/) is very important to read for this part it helps you see how to do the feature selection\n",
    "\n",
    "- [Another Article](https://towardsdatascience.com/feature-selection-using-random-forest-26d7b747597f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's use the most important features\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "#identify the features that have importance more than\n",
    "sfm = SelectFromModel(latest)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SelectFromModel(estimator=RandomForestClassifier(bootstrap=True,\n",
       "                                                 class_weight=None,\n",
       "                                                 criterion='entropy',\n",
       "                                                 max_depth=25,\n",
       "                                                 max_features='auto',\n",
       "                                                 max_leaf_nodes=None,\n",
       "                                                 min_impurity_decrease=0.0,\n",
       "                                                 min_impurity_split=None,\n",
       "                                                 min_samples_leaf=9,\n",
       "                                                 min_samples_split=2,\n",
       "                                                 min_weight_fraction_leaf=0.0,\n",
       "                                                 n_estimators=300, n_jobs=None,\n",
       "                                                 oob_score=True,\n",
       "                                                 random_state=None, verbose=0,\n",
       "                                                 warm_start=False),\n",
       "                max_features=None, norm_order=1, prefit=False, threshold=None)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the selector\n",
    "sfm.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.011111111111111112"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we find the threshold to use when choosing out features\n",
    "sfm.threshold_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.     , 0.     , 0.     , ..., 0.     , 0.     , 0.     ],\n",
       "       [0.     , 0.     , 1.     , ..., 1.     , 0.00023, 1.     ],\n",
       "       [1.     , 0.     , 0.     , ..., 0.     , 0.     , 0.5    ],\n",
       "       ...,\n",
       "       [0.     , 1.     , 1.     , ..., 1.     , 0.05814, 0.39237],\n",
       "       [0.     , 1.     , 1.     , ..., 1.     , 0.05814, 0.24197],\n",
       "       [0.46698, 0.53302, 0.46698, ..., 1.     , 0.05814, 0.5    ]])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get the features in the data that fit this threshold\n",
    "xfm = SelectFromModel(latest,threshold = 0.011)\n",
    "\n",
    "xfm.fit_transform(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AccountId_2\n",
      "AccountId_4\n",
      "AccountId_7\n",
      "SubscriptionId_6\n",
      "CustomerId_2\n",
      "CustomerId_10\n",
      "ProviderId_1\n",
      "ProviderId_3\n",
      "ProductId_1\n",
      "ProductId_3\n",
      "ProductId_4\n",
      "ProductId_5\n",
      "ProductCategory_3\n",
      "Value\n",
      "PricingStrategy\n"
     ]
    }
   ],
   "source": [
    "#print the features with the most importance\n",
    "feat_labels = [col for col in train.columns]\n",
    "\n",
    "for feature_list_index in sfm.get_support(indices=True):\n",
    "    print(feat_labels[feature_list_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((120846, 16), (15133, 16))"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's put the most important fetures into a new data set\n",
    "X_imp_train = xfm.transform(X_train)\n",
    "X_imp_test = xfm.transform(X_test)\n",
    "\n",
    "X_imp_train.shape, X_imp_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000232</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000116</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2    3    4    5    6    7    8    9   10   11   12   13  \\\n",
       "0  1.0  0.0  0.0  0.0  0.0  1.0  1.0  0.0  1.0  1.0  0.0  0.0  1.0  0.0   \n",
       "1  0.0  0.0  1.0  0.0  0.0  0.0  1.0  1.0  1.0  0.0  0.0  1.0  0.0  1.0   \n",
       "2  1.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0  1.0  1.0  0.0  0.0  1.0  0.0   \n",
       "3  1.0  0.0  1.0  0.0  1.0  1.0  1.0  0.0  1.0  0.0  0.0  1.0  0.0  1.0   \n",
       "\n",
       "         14   15  \n",
       "0  0.000002  0.0  \n",
       "1  0.000232  1.0  \n",
       "2  0.000004  0.5  \n",
       "3  0.000116  0.5  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(X_imp_train).head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
       "                       n_jobs=-1, oob_score=False, random_state=None, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#now let's run it with the best features only\n",
    "latest_c = RandomForestClassifier(n_estimators= 1000,n_jobs=-1,bootstrap=True )\n",
    "\n",
    "#fit the model\n",
    "latest_c.fit(X_imp_train, y_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Number of features of the model must match the input. Model n_features is 16 and input n_features is 90 ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-63-53422b4aa63f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m#print the score with the scoring function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mprint_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred_latest\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlatest_c\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-50-488ea39d66b4>\u001b[0m in \u001b[0;36mprint_score\u001b[0;34m(m, md)\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0;31m#let's see how the model does on validation data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m     \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_valid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m     \u001b[0mf2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf1_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_valid\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/ensemble/forest.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mThe\u001b[0m \u001b[0mpredicted\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m         \"\"\"\n\u001b[0;32m--> 545\u001b[0;31m         \u001b[0mproba\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    546\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    547\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_outputs_\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/ensemble/forest.py\u001b[0m in \u001b[0;36mpredict_proba\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    586\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'estimators_'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    587\u001b[0m         \u001b[0;31m# Check data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 588\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_X_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    589\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    590\u001b[0m         \u001b[0;31m# Assign chunk of trees to jobs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/ensemble/forest.py\u001b[0m in \u001b[0;36m_validate_X_predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    357\u001b[0m                                  \"call `fit` before exploiting the model.\")\n\u001b[1;32m    358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 359\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimators_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_X_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    360\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    361\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/sklearn/tree/tree.py\u001b[0m in \u001b[0;36m_validate_X_predict\u001b[0;34m(self, X, check_input)\u001b[0m\n\u001b[1;32m    400\u001b[0m                              \u001b[0;34m\"match the input. Model n_features is %s and \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    401\u001b[0m                              \u001b[0;34m\"input n_features is %s \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 402\u001b[0;31m                              % (self.n_features_, n_features))\n\u001b[0m\u001b[1;32m    403\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Number of features of the model must match the input. Model n_features is 16 and input n_features is 90 "
     ]
    }
   ],
   "source": [
    "#predict from the model\n",
    "pred_latest = latest_c.predict(X_imp_test)\n",
    "\n",
    "#print the score with the scoring function.\n",
    "print_score(pred_latest,latest_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bst_params = grid.best_params_\n",
    "#bst_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_2 = select_feature.transform(test)\n",
    "#pred_new_2 = rm_clf_new.predict(test_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the best parameters after the recent run of RandomSearchCV\n",
    "{'n_estimators': 215,\n",
    " 'min_samples_leaf': 25,\n",
    " 'max_features': 'sqrt',\n",
    " 'criterion': 'entropy'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#the functional Classifier\n",
    "#rm_clf = RandomForestClassifier( n_estimators=64,max_depth =10,min_samples_split=0.2,min_samples_leaf=0.1,max_features=14)\n",
    "#knn_clf = KNeighborsClassifier(n_neighbors=5,algorithm='auto')\n",
    "#lgs_cls = LogisticRegression(penalty='l2', dual=True, max_iter=30)#87%\n",
    "#gsn_cls= GaussianNB(priors=None, var_smoothing=1e-09)\n",
    "#svc_cls= SVC()\n",
    "#ada_cls= AdaBoostClassifier(n_estimators=10,learning_rate=1,algorithm='SAMME.R') #this takes a base estimator and corrects its issues"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](http://)### Feature Importance for Random Forest  Classifier\n",
    "\n",
    "- I want to know which of the features are affecting the model in a good way and which ones are not important at all. \n",
    "- When I find the important ones I remove the 0: importance features and then work with those that are important. \n",
    "- I will use the classifier to help me make these decisions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fin_pred = rm_clf.predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "EOF while scanning triple-quoted string literal (<ipython-input-64-8cf861e71257>, line 4)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-64-8cf861e71257>\"\u001b[0;36m, line \u001b[0;32m4\u001b[0m\n\u001b[0;31m    })'''\u001b[0m\n\u001b[0m         \n^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m EOF while scanning triple-quoted string literal\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\"Sub = pd.DataFrame({\n",
    "    'TransactionId':fin_Id,\n",
    "    'FraudResult':pred_new_2\n",
    "})'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sub.to_csv('EnsembleTrain_3.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subset the Data\n",
    "\n",
    "For rapid prototyping of the models, we are going to chop the data into small bits so that we can do it quickly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X__train = X_train[:40000,]\n",
    "#y__train = y_train[:40000]\n",
    "#X__test = X_test[:40000,]\n",
    "#y__test = y_test[:40000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNeighbour Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 6.91 µs\n",
      "Fitting 3 folds for each of 30 candidates, totalling 90 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\"from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import GridSearchCV \n",
    "rm_clf = KNeighborsClassifier()\n",
    "\n",
    "\n",
    "# Define parameter grid \n",
    "params = {\n",
    "    \n",
    "    'n_neighbors':list(range(1,6)),\n",
    "    #'weights':['uniform','distance'],\n",
    "    #'algorithm':['auto','ball_tree','kd_tree','brute'],\n",
    "    'leaf_size':list(range(1,7)),\n",
    "    #'p':[1,2,3,4,5],\n",
    "    'metric':['minkowski'],\n",
    "    'metric_params':[None],\n",
    "    'n_jobs':[-1],\n",
    "    \n",
    "}\n",
    "\n",
    "\n",
    "# Initialize GridSearchCV\n",
    "grid = GridSearchCV(estimator = rm_clf, \n",
    "                    param_grid = params, \n",
    "                    cv = 3,\n",
    "                    scoring = \"f1\",\n",
    "                    verbose = 1,\n",
    "                    #n_iter=20,\n",
    "                    n_jobs = -1)\n",
    "\n",
    "# Fit GridSearchCV\n",
    "%time\n",
    "grid.fit(X__train, y__train)\n",
    "\n",
    "\n",
    "y_pred = grid.predict(X__test)\n",
    "\n",
    "false_positive_rate, true_positive_rate, thresholds = roc_curve(y__test, y_pred)\n",
    "roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "\n",
    "\n",
    "# Print results\n",
    "print(\"The ROC of the tuned classifier is\", roc_auc)\n",
    "\n",
    "\n",
    "print(grid.best_params_)\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['roc_auc-->', 0.8544534145050291, 'F1_score-->', 0.009457329430308489, 'roc_auc_score-->', 0.8544534145050291]\n",
      "\n",
      "\n",
      "Model Performance on out of sample data\n",
      "['Valid_data_F1_score-->', 0.0]\n",
      "\n",
      "\n",
      "Model Performance on train data\n",
      "['Train_data_F1_score-->', 1.0]\n",
      "\n",
      "\n",
      "--------------------\n",
      "Scores on the Validation Data according to the Confusion_matrix\n",
      "Error rate  :  0.0033\n",
      "Accuracy  :  0.9967\n",
      "Precision  :  nan\n",
      "Sensitivity  :  0.0\n",
      "Specificity  :  0.0\n",
      "False positive rate  :  0.0\n",
      "True positive rate  :  0.0\n",
      "--------------------\n",
      "\n",
      "\n",
      "Drawing the Confusion Matrix on Out_of_sample_data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGfCAYAAAAQ8aZbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGRZJREFUeJzt3X+0XWV54PHvkxuCBJVgQpH8GAgSWkGrVAgOQuWHJUGEROtgWIJZmq5QAQu4RgIjM06naEGtQTpSCYIgKiFDUaJg0SmpgSWQoFBKwq9AFO4l/IgkdA0B7r257/xxd+I1JLlJeu7dT+7+flh7rXPes/c57wlZ58nz7Ge/O0opSJJUt2F1T0CSJDAgSZKSMCBJklIwIEmSUjAgSZJSMCBJklIwIEmSUjAgSZJSMCBJklIYPtAf0LX6SZeC0KDZbexRdU9BDdPd2RGteq9W/l7uMmb/ls1rsJghSZJSGPAMSZK0jXrW1z2DWpkhSZJSMEOSpCxKT90zqJUBSZKy6Gl2QLJkJ0lKwQxJkpIoluwkSSlYspMkqX5mSJKUhSU7SVIKXhgrSVL9zJAkKQtLdpKkFOyykySpfmZIkpSEF8ZKknKwZCdJUv3MkCQpC0t2kqQUvDBWkqT6mSFJUhaW7CRJKdhlJ0lS/cyQJCkLS3aSpBQs2UmSVD8zJElKopRmX4dkQJKkLBp+DsmSnSQpBTMkScqi4U0NBiRJyqLhJTsDkiRl4eKqkiTVzwxJkrKwZCdJSqHhTQ2W7CRJKZghSVIWluwkSSlYspMkqX5mSJKURcMzJAOSJCXR9NW+LdlJklIwQ5KkLCzZSZJSaHjbtyU7SVIKZkiSlIUlO0lSCpbsJEmqnxmSJGVhyU6SlIIlO0mS6meGJElZWLKTJKXQ8IBkyU6SlIIZkiRl0fCmBgOSJGVhyU6SpPqZIUlSFpbsJEkpWLKTJKl+ZkiSlIUlO0lSCpbsJEmqnxmSJGXR8AzJgCRJWZRS9wxqZclOkpSCGZIkZWHJTpKUQsMDkiU7SVIKZkiSlEXDL4w1Q5KkLHp6Wrf1IyKuiYjnI+KhTcY/ExGPRsSyiPhyn/ELI2JF9dqUPuNTq7EVEXFBn/GJEXFvRDweETdGxIj+5mRAkqRmuhaY2ncgIo4BpgF/XEo5GPhqNX4QMAM4uDrmiohoi4g24BvACcBBwKnVvgCXAnNLKZOANcCs/iZkQJKkLEpp3dbvR5XFwIubDH8auKSU8lq1z/PV+DRgfinltVLKSmAFMLnaVpRSniyldALzgWkREcCxwE3V8dcB0/ubkwFJkrJoYckuImZHxH19ttnbMIMDgaOqUtvPI+Kwanwc8HSf/dqrsS2NjwbWllK6NxnfKpsaJGkIKqXMA+Zt52HDgT2B9wKHAQsiYn8gNvcRbD6pKVvZv98PlyRlUP91SO3AzaWUAiyJiB5gTDU+oc9+44FnqsebG18NjIqI4VWW1Hf/LbJkJ0lZlJ7WbTvmh/Se+yEiDgRG0BtcFgIzImLXiJgITAKWAEuBSVVH3Qh6Gx8WVgFtEfDR6n1nArf09+FmSJLUQBFxA3A0MCYi2oEvANcA11St4J3AzCq4LIuIBcByoBs4q5Syvnqfs4HbgTbgmlLKsuoj5gDzI+Ji4H7g6n7nVAZ4ddmu1U82e/laDardxh5V9xTUMN2dHZs7X7JD1s07r2W/lyNnz23ZvAaLGZIkZVH/OaRaeQ5JkpSCGZIkZdHwtewMSJKURU+zT7lbspMkpWCGJElZNLypwYAkSVkYkCRJKQzwdaHZeQ5JkpSCGZIkZdHwkp0Z0gC76Etf409PnMH00/5y49gjjz/Jx2efx4dP/zRnnf8F/t/LLwPQ1dXFRV/8Gh8+/dN8ZOaZLPnVgxuPOeOzF/GRmWcy7eNn8Ndf/nvWr1//e5/z7e/fxDvedwJr1r40OF9MQ8aU449m2UOLeWT5XZz/ubPqnk6z9ZTWbTshA9IAm/7BP+ObX7v498a+cMllnPvpT/KD6/+B4/70CL79vX8E4KaF/wTAD67/B6667Et89X9fRU/1L6a/+5sLufm6K/jhd7/JmrUvcfuiOze+36rnXuDupfezz95/MEjfSkPFsGHDuPzrX+RDJ53GO991DB/72HTe/vZJdU9LDdVvQIqIP4qIORFxeUR8vXr89sGY3FBw6LvfyR5vftPvjf36qXYOffc7AfjPh/0JP/v5XQA88eunOPzQdwMwes9RvOmNu7PskccBeOPuuwPQvX49Xd1dRJ/7X3358iv57JmziJ1uKUXVbfJhh/DEE79m5cqn6OrqYsGCWzj5pCl1T6u56r/9RK22GpAiYg6990gPfnfviwBuiIgLBn56Q9MB++/HorvuAeCni+7k2edWA/CHB0xk0Z130929nvZnnmX5oyt49rkXNh43+7zP8/4PncruI0dy/DFHArDoznv4g73G8EeT9h/8L6Kd3thxb+Xp9t/dN629YxVjx761xhk1nCW7rZoFHFZKuaSU8t1quwSYXL22WX3v5f6t79zQyvkOCX/z387jhn/8Ead86jO8vO4Vdtmlt7fkwydOYe+9xvCxWX/FpV+/kne/4+20DW/beNy8uV9k0S3fo7Ozi3t/+a+88uqrzPvOfM7+i9Pr+iraycVm0uqBviWNtCX9ddn1AGOB32wyvk/12mb1vZe790N6vf33ncBVl30J6C3fLf7FEgCGD29jzjlnbNzv42d8ln3Hj/29Y3fddQTHHHk4i+68hzGj96TjmWf585lnAvDcC6v5L5/6DPOvuowxo98ySN9GO7OO9lVM6PN3bPy4fVi16rkaZ9RspeFddv0FpHOBf46Ix4Gnq7H/BBwAnD2QExvKfrtmLaP3HEVPTw9XXjefU6Z/EIBXXn2VUmDkbm/gF0t+xfC2Nt42cV/WrXuFl9e9wl5j3kJ393oW330f73nXwRz4toksvnX+xvc9/s9ncuPVl7PnqD3q+mraySy97wEOOGAi++03gY6OZznllGmc/gk77Wqzk5baWmWrAamU8k/VfdUnA+PoPX/UDizdcPtabd3nvnAJS+9/kLVr/53jpp/GmbNOZ90rrzD/5h8D8IH3H8GHTzwegBfXvMQZ532eGDaMvfcazd/+j/8KwLpXX+XsOf+Tzq4uetb3cPh73sUp00+s7Ttp6Fi/fj3nnHsRt936fdqGDePa625k+fLH6p6WGspbmGtI8RbmGmytvIX5yxef1rLfy90v+u5O13frSg2SlEXDS3ZeGCtJSsEMSZKysMtOkpSCJTtJkupnhiRJWeyka9C1igFJkrKwZCdJUv3MkCQpCdeykyTlYMlOkqT6mSFJUhYNz5AMSJKURcPbvi3ZSZJSMEOSpCws2UmSMigND0iW7CRJKZghSVIWDc+QDEiSlEXDV2qwZCdJSsEMSZKysGQnSUqh4QHJkp0kKQUzJElKopRmZ0gGJEnKwpKdJEn1M0OSpCwaniEZkCQpCdeykyQpATMkScqi4RmSAUmSsmj2UnaW7CRJOZghSVISTW9qMCBJUhYND0iW7CRJKZghSVIWDW9qMCBJUhJNP4dkyU6SlIIZkiRlYclOkpSBJTtJkhIwQ5KkLCzZSZIyKAYkSVIKDQ9InkOSJKVghiRJSViykyTl0PCAZMlOkpSCAUmSkig9rdv6ExHXRMTzEfFQn7GvRMQjEfFgRPwgIkb1ee3CiFgREY9GxJQ+41OrsRURcUGf8YkRcW9EPB4RN0bEiP7mZECSpCQGMyAB1wJTNxn7GfCOUsofA48BFwJExEHADODg6pgrIqItItqAbwAnAAcBp1b7AlwKzC2lTALWALP6m5ABSZIaqJSyGHhxk7GfllK6q6f3AOOrx9OA+aWU10opK4EVwORqW1FKebKU0gnMB6ZFRADHAjdVx18HTO9vTjY1SFISybrsPgXcWD0eR2+A2qC9GgN4epPxw4HRwNo+wa3v/ltkhiRJWZRo2RYRsyPivj7b7G2dRkR8HugGvrdhaHOz3YHxrTJDkqQhqJQyD5i3vcdFxEzgQ8BxpZQNQaQdmNBnt/HAM9XjzY2vBkZFxPAqS+q7/xaZIUlSEoPc1PA6ETEVmAOcXEpZ1+elhcCMiNg1IiYCk4AlwFJgUtVRN4LexoeFVSBbBHy0On4mcEt/n2+GJElJlJ7NVboGRkTcABwNjImIduAL9HbV7Qr8rLcvgXtKKX9ZSlkWEQuA5fSW8s4qpayv3uds4HagDbimlLKs+og5wPyIuBi4H7i63zn9LiMbGF2rn2z2Hac0qHYbe1TdU1DDdHd2tCyKrDrymJb9Xu5z16LBi24tYoYkSUkk67IbdAYkSUqilJ0uqWkpmxokSSmYIUlSEpbsJEkpDGaXXUaW7CRJKZghSVISA3wVTnoGJElKwpKdJEkJmCFJUhJNz5AMSJKURNPPIVmykySlYIYkSUlYspMkpeBadpIkJWCGJElJuJadJCmFHkt2kiTVzwxJkpJoelODAUmSkmh627clO0lSCmZIkpRE05cOMiBJUhKW7CRJSsAMSZKSaPp1SAYkSUqi6W3fluwkSSmYIUlSEnbZSZJSaPo5JEt2kqQUzJAkKYmmNzUYkCQpiaafQ7JkJ0lKYcAzpN3GHjXQHyFJQ0LTmxos2UlSEk0/h2TJTpKUghmSJCVhyU6SlELDm+wMSJKURdMzJM8hSZJSMEOSpCSa3mVnQJKkJHrqnkDNLNlJklIwQ5KkJAqW7CRJCfQ0vO/bkp0kKQUzJElKoseSnSQpg6afQ7JkJ0lKwQxJkpJo+nVIBiRJSsKSnSRJCZghSVISluwkSSk0PSBZspMkpWCGJElJNL2pwYAkSUn0NDseWbKTJOVghiRJSbiWnSQphYbffcKSnSQpBzMkSUqi6dchGZAkKYmeaPY5JEt2kqQUzJAkKYmmNzUYkCQpiaafQ7JkJ0lKwQxJkpJw6SBJUgo9RMu2bRER50XEsoh4KCJuiIg3RMTEiLg3Ih6PiBsjYkS1767V8xXV6/v1eZ8Lq/FHI2LKjn5/A5IkNVBEjAP+Cji0lPIOoA2YAVwKzC2lTALWALOqQ2YBa0opBwBzq/2IiIOq4w4GpgJXRETbjszJgCRJSZQWbttoOLBbRAwHRgKrgGOBm6rXrwOmV4+nVc+pXj8uIqIan19Kea2UshJYAUzeri9eMSBJUhI90botImZHxH19ttl9P6uU0gF8FXiK3kD0EvBLYG0ppbvarR0YVz0eBzxdHdtd7T+67/hmjtkuNjVI0hBUSpkHzNvS6xGxJ73ZzURgLfB/gBM291YbDtnCa1sa324GJElKYpCvQ/oAsLKU8gJARNwMHAGMiojhVRY0Hnim2r8dmAC0VyW+PYAX+4xv0PeY7WLJTpKSGORzSE8B742IkdW5oOOA5cAi4KPVPjOBW6rHC6vnVK/fUUop1fiMqgtvIjAJWLLdXx4zJElqpFLKvRFxE/AroBu4n94S363A/Ii4uBq7ujrkauD6iFhBb2Y0o3qfZRGxgN5g1g2cVUpZvyNzit4AN3CGjxjX9OWZJA1h3Z0dLbuc9erxp7Xs93JW+3d3ustszZAkKQnXspMkKQEzJElKoukZkgFJkpIoO91Zn9ayZCdJSsEMSZKSsGQnSUqh6QHJkp0kKQUzJElKoumrCBiQJCkJb2EuSVICZkiSlETTmxoMSJKURNMDkiU7SVIKZkiSlIRddpKkFJreZWdAkqQkPIckSVICZkiSlITnkCRJKfQ0PCRZspMkpWCGJElJNL2pwYAkSUk0u2BnyU6SlIQZkiQlYclOkpRC01dqsGQnSUrBDEmSkmj6dUgGJElKotnhyJKdJCkJMyRJSsIuO0lSCk0/h2TJTpKUghmSJCXR7PzIgCRJaTT9HJIlO0lSCmZIkpRE05saDEiSlESzw5ElO0lSEmZIkpRE05saDEiSlERpeNHOkp0kKQUzJElKwpKdJCmFprd9W7KTJKVghiRJSTQ7PzIgSVIaluyUwh57vJkb58/joX/7Of/24L/w3sPfA8BZZ36SZQ8t5l8fuINL/vbzNc9SQ9GU449m2UOLeWT5XZz/ubPqno4azAwpiblf+1/cfvsiPjZjNrvssgsjR+7G0e8/gpNPmsIhf/IBOjs72Wuv0XVPU0PMsGHDuPzrX2TqB0+lvX0V99x9Gz/68U95+OHH655aIzW9y84MKYE3vemNHHXk4Vzz7RsA6Orq4qWX/p0zzvgEX/7KN+js7ATghRd+W+c0NQRNPuwQnnji16xc+RRdXV0sWHALJ580pe5pNVZp4X87ox0OSBHxyVZOpMn2339fVq/+LVd/ay5Ll9zOld/8CiNH7sakSftz5JGT+cVdP+KO/3sTh77nXXVPVUPM2HFv5en2ZzY+b+9Yxdixb61xRmqy/0iG9Nctm0XDDW9r45BD3smVV36HwyZP4eWX1zHn/LMZPryNUaP24IgjT2LOBRdzw/e/WfdUNcRExOvGStk5/3U9FPS0cNsZbfUcUkQ8uKWXgL23ctxsYDZAtO3BsGG77/AEm6C9YxXt7atYsvR+AG6++VbO/9zZdLSv4oc//AkAS+97gJ6eHsaMeQurV79Y53Q1hHS0r2LC+LEbn48ftw+rVj1X44yabWcttbVKf00NewNTgDWbjAfwiy0dVEqZB8wDGD5iXLP/hLfBc8+9QHv7Mxx44Nt47LEnOPbYI3n44cd44snfcMwx7+Pni+9m0qT9GTFihMFILbX0vgc44ICJ7LffBDo6nuWUU6Zx+ifstFM9+gtIPwbeWEp5YNMXIuJfBmRGDXXOef+d71z394wYsQsrVz7FrL/4LC+/vI5vXfV3PHD/P9PZ2cWnZp1b9zQ1xKxfv55zzr2I2279Pm3DhnHtdTeyfPljdU+rsXbWUlurxEDXi82QJA1l3Z0drz8Rt4NO3/cjLfu9vP43N7dsXoPFtm9JUgpeGCtJSTS9nGRAkqQkXMtOkqQEzJAkKQmvQ5IkpdD0tm9LdpKkFMyQJCmJpjc1GJAkKYmmn0OyZCdJSsEMSZKSsKlBkpRCKaVl27aKiLaIuD8iflw9nxgR90bE4xFxY0SMqMZ3rZ6vqF7fr897XFiNPxoRO3zLYQOSJDXbOcDDfZ5fCswtpUyi99ZDs6rxWcCaUsoBwNxqPyLiIGAGcDAwFbgiItp2ZCIGJElKoofSsm1bRMR44ETgW9XzAI4Fbqp2uQ6YXj2eVj2nev24av9pwPxSymullJXACmDyjnx/A5IkJdHKW5hHxOyIuK/PNnszH3kZcD6/O301GlhbSumunrcD46rH44CnAarXX6r23zi+mWO2i00NkpREK9u++965e3Mi4kPA86WUX0bE0RuGNzutrb+2tWO2iwFJkprpfcDJEfFB4A3Am+nNmEZFxPAqCxoPPFPt3w5MANojYjiwB/Bin/EN+h6zXSzZSVISg3kOqZRyYSllfCllP3qbEu4opXwcWAR8tNptJnBL9Xhh9Zzq9TtKbzvfQmBG1YU3EZgELNmR72+GJElJbE+79gCaA8yPiIuB+4Grq/GrgesjYgW9mdEMgFLKsohYACwHuoGzSinrd+SDY6D/AIaPGJfiT1iSBkJ3Z8fmzqHskBMmnNCy38ufPP2Tls1rsJghSVISTV+pwYAkSUm4uKokSQmYIUlSEt4PSZKUQpIuu9pYspMkpWCGJElJWLKTJKVgl50kSQmYIUlSEj0Nb2owIElSEs0OR5bsJElJmCFJUhJ22UmSUmh6QLJkJ0lKwQxJkpJo+tJBBiRJSsKSnSRJCZghSVISTV86yIAkSUk0/RySJTtJUgpmSJKURNObGgxIkpSEJTtJkhIwQ5KkJCzZSZJSaHrbtyU7SVIKZkiSlIR3jJUkpWDJTpKkBMyQJCkJS3aSpBQs2UmSlIAZkiQlYclOkpSCJTtJkhIwQ5KkJCzZSZJSsGQnSVICZkiSlEQpPXVPoVYGJElKoun3Q7JkJ0lKwQxJkpIodtlJkjKwZCdJUgJmSJKUhCU7SVIKTV+pwZKdJCkFMyRJSqLpSwcZkCQpCc8hSZJSsO1bkqQEzJAkKQlLdpKkFGz7liQpATMkSUrCkp0kKQW77CRJSsAMSZKSsGQnSUrBLjtJkhIwQ5KkJFxcVZKUgiU7SZISMEOSpCTsspMkpdD0c0iW7CRJKRiQJCmJUkrLtm0REVMj4tGIWBERFwzw1+uXJTtJSmIwzyFFRBvwDeDPgHZgaUQsLKUsH7RJbMIMSZKaaTKwopTyZCmlE5gPTKtzQgYkSUqitHDbBuOAp/s8b6/GajPgJbvuzo4Y6M8YiiJidillXt3zUHP4d65+rfy9jIjZwOw+Q/M2+f+7uc+qtc3PDCmv2f3vIrWUf+eGkFLKvFLKoX22Tf+x0Q5M6PN8PPDM4M3w9QxIktRMS4FJETExIkYAM4CFdU7ILjtJaqBSSndEnA3cDrQB15RSltU5JwNSXtbyNdj8O9cwpZTbgNvqnscG0fS1kyRJOXgOSZKUggEpoWzLeWjoiohrIuL5iHio7rlIBqRk+izncQJwEHBqRBxU76w0hF0LTK17EhIYkDJKt5yHhq5SymLgxbrnIYEBKaN0y3lI0mAwIOWTbjkPSRoMBqR80i3nIUmDwYCUT7rlPCRpMBiQkimldAMblvN4GFhQ93IeGroi4gbgbuAPI6I9ImbVPSc1lys1SJJSMEOSJKVgQJIkpWBAkiSlYECSJKVgQJIkpWBAkiSlYECSJKVgQJIkpfD/AZWtx4qM/m7aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"\"kn_1 = KNeighborsClassifier(weights='distance', p=5, n_neighbors=5, n_jobs=-1, metric_params=None, metric= 'minkowski', leaf_size= 3, algorithm= 'auto')\n",
    "kn_1.fit(X_train, y_train)\n",
    "pred_kn = kn_1.predict(X_test)\n",
    "print_score(pred_kn,kn_1)\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}