Sarc_Detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/asel/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /Users/asel/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#@unpublished{SARC,\n",
    "#  authors={Mikhail Khodak and Nikunj Saunshi and Kiran Vodrahalli},\n",
    "#  title={A Large Self-Annotated Corpus for Sarcasm},\n",
    "#  url={https://arxiv.org/abs/1704.05579},\n",
    "#  year=2017\n",
    "#}\n",
    "#https://www.kaggle.com/danofer/sarcasm#train-balanced-sarcasm.csv\n",
    "import os\n",
    "import string\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type:\n",
      "Labels array  <class 'pandas.core.series.Series'>\n",
      "Comments array  <class 'pandas.core.series.Series'>\n",
      "one comment line  <class 'str'>\n",
      "Shape:\n",
      "Labels array  (1010773,)\n",
      "Comments array  (1010773,)\n",
      "Two first entries:\n",
      "0 NC and NH.\n",
      "0 You do know west teams play against west teams more than east teams right?\n"
     ]
    }
   ],
   "source": [
    "# df - data frame\n",
    "df = pd.read_csv('../train-balanced-sarcasm.csv')\n",
    "# dropping empty comment entries\n",
    "df.dropna(subset=['comment'], inplace=True)\n",
    "\n",
    "print('Type:')\n",
    "print('Labels array ',type(df.label))\n",
    "print('Comments array ',type(df.comment))\n",
    "print('one comment line ', type(df.comment[0]))\n",
    "print('Shape:')\n",
    "print('Labels array ',df.label.shape)\n",
    "print('Comments array ',df.comment.shape)\n",
    "print('Two first entries:')\n",
    "print (df.label[0], df.comment[0])\n",
    "print (df.label[1], df.comment[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "167435\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# create the transform\n",
    "tf_idf_vectorizer = TfidfVectorizer()\n",
    "# tokenize and build vocab\n",
    "T = tf_idf_vectorizer.fit(df.comment)\n",
    "print(len(T.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess with nltk\n",
    "def my_tokenizer(corpus):\n",
    "    corpus_tokenized = []\n",
    "    tokenizer = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "    sbs = nltk.stem.SnowballStemmer('english', ignore_stopwords=False)\n",
    "    for comment in corpus:\n",
    "        words = tokenizer.tokenize(comment)\n",
    "        # Remove stop words\n",
    "        stop_words = set(stopwords.words('english'))\n",
    "        words = [word for word in words if not word in stop_words]\n",
    "        \n",
    "        cmnt_t = []\n",
    "        for token in words:\n",
    "            cmnt_t.append(sbs.stem(token))\n",
    "            # make a string to be compatible with TfidfVectorizer\n",
    "            c = ' '.join(cmnt_t)\n",
    "        # Lemmitize\n",
    "        #words = [lemmatizer.lemmatize(word) for word in words]\n",
    "        \n",
    "        corpus_tokenized.append(c)\n",
    "    return corpus_tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.Series(my_tokenizer(df.comment))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NC and NH.\n",
      "nc nh \n",
      "\n",
      "You do know west teams play against west teams more than east teams right?\n",
      "you know west team play west team east team right \n",
      "\n",
      "They were underdogs earlier today, but since Gronk's announcement this afternoon, the Vegas line has moved to patriots -1\n",
      "they underdog earlier today sinc gronk announc afternoon vega line move patriot 1 \n",
      "\n",
      "This meme isn't funny none of the \"new york nigga\" ones are.\n",
      "this meme funni none new york nigga one \n",
      "\n",
      "I could use one of those tools.\n",
      "i could use one tool \n",
      "\n",
      "I don't pay attention to her, but as long as she's legal I wouldn't kick her out of bed (before she took a load)\n",
      "i pay attent long legal i kick bed took load \n",
      "\n",
      "Trick or treating in general is just weird...\n",
      "trick treat general weird \n",
      "\n",
      "Blade Mastery+Masamune or GTFO!\n",
      "blade masteri masamun gtfo \n",
      "\n",
      "You don't have to, you have a good build, buy games or save it\n",
      "you good build buy game save \n",
      "\n",
      "I would love to see him at lolla.\n",
      "i would love see lolla \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(df.comment[i])\n",
    "    print(df2[i], '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131021\n"
     ]
    }
   ],
   "source": [
    "T = tf_idf_vectorizer.fit(df2)\n",
    "print(len(T.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6268559"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_idf_vectorizer = TfidfVectorizer(ngram_range=(1,3))\n",
    "T = tf_idf_vectorizer.fit(df2)\n",
    "len(T.vocabulary_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "167435 features from before now reduced to 131022 features, that is about 22%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "# divide into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(df2, df.label, train_size=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(333555, 2342042)\n",
      "  (0, 2295377)\t0.08403192040579566\n",
      "  (0, 884033)\t0.13565504784920923\n",
      "  (0, 642503)\t0.18483129966185147\n",
      "  (0, 990024)\t0.1619485054723261\n",
      "  (0, 930743)\t0.26084501135689486\n",
      "  (0, 618452)\t0.11365271992434346\n",
      "  (0, 475620)\t0.11315049929769513\n",
      "  (0, 1669374)\t0.09442393824396139\n",
      "  (0, 2300958)\t0.21473026021717992\n",
      "  (0, 884434)\t0.26084501135689486\n",
      "  (0, 642546)\t0.26084501135689486\n",
      "  (0, 990162)\t0.26084501135689486\n",
      "  (0, 930744)\t0.26084501135689486\n",
      "  (0, 619072)\t0.17140456369170137\n",
      "  (0, 477615)\t0.23575544888861052\n",
      "  (0, 2300963)\t0.26084501135689486\n",
      "  (0, 884435)\t0.26084501135689486\n",
      "  (0, 642547)\t0.26084501135689486\n",
      "  (0, 990163)\t0.26084501135689486\n",
      "  (0, 930745)\t0.26084501135689486\n",
      "  (0, 619127)\t0.26084501135689486\n",
      "  (1, 2295377)\t0.264014369817753\n",
      "  (1, 1282354)\t0.5085902430761836\n",
      "  (1, 2303673)\t0.8195318036400072\n",
      "  (2, 2097702)\t0.1690946995003176\n",
      "  (2, 1899166)\t0.18333120537780853\n",
      "  (2, 1057303)\t0.12420892486074264\n",
      "  (2, 1852652)\t0.2111247700340524\n",
      "  (2, 1377326)\t0.1888517085430904\n",
      "  (2, 2099642)\t0.3576114729142696\n",
      "  (2, 1899735)\t0.32321435969219003\n",
      "  (2, 1065820)\t0.31342113564368973\n",
      "  (2, 1852995)\t0.3576114729142696\n",
      "  (2, 2099643)\t0.3576114729142696\n",
      "  (2, 1899737)\t0.3576114729142696\n",
      "  (2, 1065826)\t0.3576114729142696\n"
     ]
    }
   ],
   "source": [
    "# Vetorize the training data set\n",
    "X_train = tf_idf_vectorizer.fit_transform(X_train)\n",
    "print(X_train.shape)\n",
    "print(X_train[0:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(677218, 2342042)\n",
      "  (0, 2296310)\t0.35205279172210907\n",
      "  (0, 2296286)\t0.27627736876649334\n",
      "  (0, 2295377)\t0.1134147516133758\n",
      "  (0, 1802726)\t0.19725488802358238\n",
      "  (0, 1670773)\t0.341092984246088\n",
      "  (0, 1669374)\t0.12744047084228027\n",
      "  (0, 1349511)\t0.33331687709081304\n",
      "  (0, 1347657)\t0.15993758060105553\n",
      "  (0, 744329)\t0.13095213862473232\n",
      "  (0, 593937)\t0.35205279172210907\n",
      "  (0, 593687)\t0.27976998914823636\n",
      "  (0, 593384)\t0.3845654210324432\n",
      "  (0, 180036)\t0.2975895376802706\n",
      "  (0, 177647)\t0.1512633771756192\n",
      "  (1, 2233143)\t0.34124038347017366\n",
      "  (1, 2232047)\t0.1857660176093302\n",
      "  (1, 1780567)\t0.16766011995092808\n",
      "  (1, 1742468)\t0.20480676039123955\n",
      "  (1, 1741580)\t0.16935454128026448\n",
      "  (1, 1708331)\t0.32467493712598416\n",
      "  (1, 1545915)\t0.29074366906421906\n",
      "  (1, 1115659)\t0.10893978479476327\n",
      "  (1, 1009166)\t0.289819561589443\n",
      "  (1, 1009143)\t0.2585562202170115\n",
      "  (1, 1000062)\t0.12475286576368891\n",
      "  (1, 307216)\t0.34124038347017366\n",
      "  (1, 304023)\t0.1598881028246156\n",
      "  (1, 222808)\t0.34752918304163555\n",
      "  (1, 222798)\t0.3122228738383578\n",
      "  (1, 222216)\t0.14770542862896835\n",
      "  (2, 1751128)\t0.4876363039761829\n",
      "  (2, 1684339)\t0.6055233197730527\n",
      "  (2, 1669050)\t0.4668281841986518\n",
      "  (2, 1323945)\t0.23533279489731257\n",
      "  (2, 316188)\t0.34963161518815516\n"
     ]
    }
   ],
   "source": [
    "# Vetorize the test data set\n",
    "X_test = tf_idf_vectorizer.transform(X_test)\n",
    "print(X_test.shape)\n",
    "print(X_test[0:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oob_score is  0.6784278454827539\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "# bagging with Logistic Regressoin\n",
    "# oob Out-Of-Bag\n",
    "bag_log_reg = BaggingClassifier(\n",
    "      LogisticRegression(), n_estimators=200,\n",
    "    max_samples=50000, bootstrap=True, n_jobs=-1, oob_score=True)\n",
    "\n",
    "\n",
    "#clf = make_pipeline(TfidfVectorizer(ngram_range=(1,3)), bag_log_reg)\n",
    "bag_log_reg.fit(X_train,y_train)\n",
    "print('oob_score is ', bag_log_reg.oob_score_)    \n",
    "\n",
    "#cross_val_score(clf, X_train, y_train, cv=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaggingClassifier(base_estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "         bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "         max_samples=50000, n_estimators=200, n_jobs=-1, oob_score=True,\n",
       "         random_state=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_log_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6786824921960137"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "bag_log_reg_pred = bag_log_reg.predict(X_test)\n",
    "accuracy_score(y_test, bag_log_reg_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4VGXax/Hvk4QEUgiEJJRUSAFCkRJDUem9W1axu+sua3d1V0FRVLCta913bVh21S24i0JCExVBEEEJiAFCb2lAQoCE9GTmef84g0QMZIDJnCn357q8zMyczNzHhJ8Pp9y30lojhBDCs/iYXYAQQgjHk3AXQggPJOEuhBAeSMJdCCE8kIS7EEJ4IAl3IYTwQBLuQgjhgSTchRDCA0m4CyGEB/Iz64PDw8N1fHy8WR8vhBBuaePGjUe11hGNbWdauMfHx5OZmWnWxwshhFtSSh20Zzs5LCOEEB5Iwl0IITyQhLsQQnggCXchhPBAEu5CCOGBGg13pdT7SqlCpdTWs7yulFJ/VUrtUUplKaX6OL5MIYQQ58Oelfs/gDHneH0skGT7Zxrw5sWXJYQQHir3e1jzkvHvJtTode5a69VKqfhzbDIZ+FAb8/rWK6VaKaXaa60POahGIYRwf1pT/cPH+C++G6Wt4BsAt2ZATFqTfJwjbmKKAnLrPc6zPfeLcFdKTcNY3RMbG+uAjxZCCBemNRzeAtnpVPz4KYGl+/hparWlBg6scelwVw081+DUba31XGAuQGpqqkzmFkJ4Hq2hYBNkZ0B2OhzfjxUfNlm6khswmev0ZyhrHfj6Q/wVTVaGI8I9D4ip9zgaKHDA+wohhHuwWiE/0wjz7AwoyQEfP3THwbxcOZ7/lHTn6kG9eGBEMj6HNxor9vgrmmzVDo4J9wzgHqXUPKAfUCLH24UQHs9qgZz1RqBvXwQnC4zVeMIwygf+icAeE1GBYXTbepj3WzWnZ3Qr4/ti0po01E9pNNyVUv8BhgDhSqk84AmgGYDW+i1gKTAO2ANUAL9uqmKFEMJUljo4uPZ0oJcXgl9zSBwBKU+hk0axcEcZTy3KZroq4/q0MMZ0b2dKqfZcLXN9I69r4G6HVSSEEK7EUgv7vzYCfccSqCiGZoGQNApSJhv/Dgim4EQlM+dtYeXOInrHtiI1rrWpZZvW8lcIIVxWXTXsXQnbM4xArzoB/iGQPNoI9MQR4B/40+bpm/OZuWArFqtm1oQUbh0Yj69PQ9eaOI+EuxBCANRWwp4Vxgp912dQXQoBodBlnBHonYZCs+YNfmtoi2b0imnFc1f1ICYssMFtnE3CXQjhvWrKYffntkD/HGrLoUVrI8xTJkPHweDn/4tvq7NYee+b/dRarNwzLIkhnSMZnByBUuau1uuTcBdCeJeqUlugL4TdX0JdJQSGQ89rjUCPvxx8m53127MLSpn+SRZb8ksY37M9WmuUUi4V7CDhLoTwBpUnYOcyY4W+d4Vxd2hwO+hzsxHosQPAx/ecb1FdZ+FvX+3hzVV7aRXYjDdu7MPY7u1cLtRPkXAXQnimimPGydDsdNi3Cqy10DIaLv2tEejRaeBjf9fzA0creOvrvUzq1YHHx6fQOuiXh2tciYS7EMJzlBXBjkVGoO9fA9oCreKg/52QMgWi+sB5rLTLq+v4IvsIU3pH0bldCCseHEJsG9c4YdoYCXchhHsrPQQ7FhuBfnAtaCuEJcDlf4Cuk6D9JecV6Kes2V3EI59uIf9EJd2jWpIYGeI2wQ4S7kIId1SSd7oxV+53gIaILjDoIeOQS2TKBQU6QElFLc8szea/mXl0Cg/i42kDSIwMcWz9TiDhLoRwD8cPnA70/EzjubY9YOhMSJkEEZ0v+iMsVs3Vb33L/qPl3DUkgfuGJ9G82blPtLoqCXchhOs6uge2pxuBfuhH47n2vWD4E8YKvU2CQz7mWHkNrVo0w9dH8dDozkS1akH3qFCHvLdZJNyFEK6lcIetMVcGHLGNbo6+FEY9DV0nQut4h32U1ppPN+Uze3E208d04YZ+sYzuZk6jL0eTcBdCmEtrOLLN1gs9HY7uBJRx7fmY541AD412+MfmHa/g0QVbWb2riL5xrUnrGObwzzCThLsQwvm0hkObTwf6sX2gfCDuMkj7nRHoIU23gl7wQx6PLdiKBp6a1I2b+8fhY3KjL0eTcBdCOIfVCvkbTx9DP5EDyhc6DoKB90GXCRAc4ZRSwoIC6BsfxrNXdie6tftc3ng+JNyFEE3HajUuVTx1DL00H3yaQcJQGDwdOo+DwKY/HFJrsfLOmn3UWTT3DU9icHIEg5LCXbZ1gCNIuAshHMtSBznf2gJ9MZQdBt8Aowf68FmQPAZatHJaOVvzS5j+SRbbCkqZeEkHl2305WgS7kKIi2epNYY+nwr0iqPg1wKSRhqXLCaPhgDn3ghUVWvhryt28/bqfbQO9Oetm/owpnt7p9ZgJgl3IcSFqasxGnJlp8POJVB5HPyDz5hWFGRaeQeLK3hnzT6u6h3FY+NTCA08extfTyThLoSwX22V0TI3O8NooVtdAgEtofNYI9AThkGzFqaVV15dx/Jth7mqTzSd24Xw1R+HuMxkJGeTcBdCnFtNBez5wjataDnUlEHzVsbliimTodNg8Aswu0q+3lXEo59uoaCkkp7RoSRGhnhtsIOEuxCiIdUnjSDfngG7v4DaCghsAz2uMTotdhx0zmlFznS8vIY5S7L5dFM+CRFB/O/37tnoy9Ek3IUQhqoS2PmZsULf8yVYqiG4LfS6wTataCD4ulZknGr0dbC4gnuGJnLPsES3bfTlaK71kxJCOFfFMdi51DiGvvcrY1pRSAdI/Y0R6DFpjY6fM0NxWTWtA/3x9VHMGNOFqNYt6NbBvRt9OZqEuxDepvzo6eEW+1eDtQ5CY6Hf723Tivqe1/g5Z9Ja87+NeTy9OJvpY7twY784RnlIoy9Hk3AXwhucPAzbFxnH0A98Y5tW1AkG3mus0Nv3uuDhFs6Se6yCRxdsYc3uo6TFhzGgUxuzS3JpEu5CeKqSfCPQs9MhZx2gITwZrvijEehtu7t8oJ/y6aY8Hlu4FQXMmdKdG9NiPa7Rl6NJuAvhSY4fNFbn2emQt8F4LrIbDHnENn6ui7n1XaDw4ADSOobxzJU9iGpl3nX07kTCXQh3V7z3dKAX/GA81/4So49L18kQnmhufReg1mLl7a/3YrHC/SOSGJQcwaBk53SM9BQS7kK4o6Jdp3uhH9liPBfVF0bONq5DD+tobn0XYWt+CQ/Nz2L7oVIm9zrd6EucH7vCXSk1BngN8AXe1Vo/f8brscAHQCvbNjO01ksdXKsQ3ktrKMy2BXoGFG03no/pD6OfM+4WbRVjbo0XqarWwqtf7uadNfsIC/Ln7Zv7eszIOzM0Gu5KKV/gdWAkkAdsUEplaK2z6232GPBfrfWbSqkUYCkQ3wT1CuE9tIbDWadX6MV7Tk8rSv0LdJ0ALTuYXaXD5Byr4L1v9nFNn2geHdfV6xp9OZo9K/c0YI/Weh+AUmoeMBmoH+4aaGn7OhQocGSRQngNrSF/E2QvtE0rOmibVnQFDLjbNq0o0uwqHeZkVS2fbT3Mr1JjSG4bwso/DfHYyUjOZk+4RwG59R7nAf3O2OZJ4HOl1L1AEDDCIdUJ4Q2sVsj73jjcsj0DSnKNaUWdhsCgh4xpRUGed033yh2FzFywhcOlVfSObUViZIgEuwPZE+4NncnQZzy+HviH1volpdQA4COlVHettfVnb6TUNGAaQGxs7IXUK4RnsFqMa8+z041r0U8eAl9/SBgOQ2dC5zHQorXZVTaJY+U1zFmczYIf8kmKDGb+nQOl0VcTsCfc84D6Z2qi+eVhl9uBMQBa63VKqeZAOFBYfyOt9VxgLkBqauqZ/4MQwrNZ6k5PK9qxGMqLwK+5Ma2oq21aUfOWjb+PG7NYNde8+S05xyq4b3gSdw9NIMDP9XrXeAJ7wn0DkKSU6gjkA1OBG87YJgcYDvxDKdUVaA4UObJQIdxSXY3RvyV7IexYApXHoFkQJI+yTSsaCQHBZlfZ5IpOVtMmyGj09ei4rkS1bkHX9p79PzKzNRruWus6pdQ9wHKMyxzf11pvU0rNBjK11hnAH4F3lFIPYByyuU1rLStz4Z1qq2DfStv4uaVGK13/kNPTihKHmzqtyJm01vw3M5enl2xn+pgu3NQ/jhEpbc0uyyvYdZ277Zr1pWc8N6ve19nAZY4tTQg3Ultp9EDPTjd6otechOah0Hm8bfzcUJeYVuRMOcUVzPg0i2/3FtOvYxiXJ4abXZJXkTtUhbhQ1WWw+3Mj0Hd/AbXl0CIMul9pBHr8IPDzN7tKU8zfmMfjC7fi66N45sruXH+pNPpyNgl3Ic5HVakxfi57obFSr6uCoEi45Doj0OMud7lpRWZo2zKAgQltePrK7rQP9Y5DUK5GfguFaEzlcdi5zFih7/0KLDUQ0h763GobP9ffJacVOVNNnZU3V+3FqjUPjEzmiqQIrkiSRl9mknAXoiHlxbBziRHo+1bZphXFQNo0ozFX9KUuO63I2X7MPcHD87PYeeQkV/WOkkZfLkLCXYhTygpPD7c48A1oC7SON277T5kMHfq4zXALZ6issfDyFzt575v9RIY0591bUuVKGBci4S68W2mBLdAz4OBaQEObRLj8ASPQ2/WQQD+L3OMVfPDtQaamxTJjbBdaNpdGX65Ewl14nxO5p4db5H5nPBeZAoOn26YVdZVAP4tSW6Ova22NvlY9NIQOMhnJJUm4C+9wbJ+xOs9Oh4JNxnPtesCwx4xb/yOSza3PDXy14wiPfrqVwpNV9IltTWJksAS7C5NwF57r6O7TvdAPZxnPdegDI56ClEkQ1snc+txEcVk1sxdnk765gM5tQ3jr5r4kRnp+ywR3J+EuPIfWULTjdKAX2kYORKfBqGeMaUWt48yt0c1YrJpfvbWO3OMVPDAimTuHJODvJ1cJuQMJd+HetIbDW2ytczPg6C5AQdxAGPuCMdwiNMrsKt1O4ckqwoMC8PVRzBzflejWgXRuJ2153YmEu3A/WkPBD6dX6Mf3G+Pn4q+Afr+HLhMhRC7JuxBWq+Y/G3J4bukOpo/tws394xjeVf5buiMJd+EerFbIzzw9ILokB3z8oONg47LFLuMhSBpTXYwDR8uZ8WkW6/cdY2BCGwbLHaZuTcJduC6rxbhU8VSgnyywTSsaBkNmGC10A8PMrtIj/Dczl8cXbsXf14fnr+rBdZfGyF2mbk7CXbgWS51xM9Gp8XPlhca0osQRkPKUbVpRqNlVepyoVi0YlBzBnMndaRfa3OxyhANIuAvzWWpt04ps4+cqiqFZICTZphUljfKKaUXOVF1n4Y2Ve9Fa8+CozlyWGM5l0m/do0i4C3PUVRsNubLTjfFzVSfAPxiSx9imFY0A/0Czq/RIP+QcZ/onWew6UsbVfaKl0ZeHknAXzlNbabTMzU43WuhWl0JAKHQZZ3RaTBgGzeSQQFOpqKnjpc938f7a/bRr2Zz3b0tlWBe5EsZTSbiLplVTbkwpyk43hlzUlkOL1sYdoilTjKtdvHRakbPlH6/ko/UHubFfLNPHdCFEGn15NAl34XjVJ23Timzj5+oqITAcel5rGz93OfhKsDhDSWUty7YcYmpaLEltQ/j6oSEyGclLSLgLx6g8Abs+MwJ9zwqwVENwO+h9k2383ECvn1bkbJ9vO8xjC7dSXF5DanwYiZHBEuxeRMJdXLiKY8bJ0J+mFdVCyyi49HYj0KPTZFqRCY6WVfNkxjYWZx2iS7sQ3r01VRp9eSEJd3F+yoqMyxWz043LF7UFWsVB/ztPTyuSQDeNxaq55s1vKThRxZ9GJfP7wQk085WfhzeScBeNKz10OtAPrgVthbAEuOx+I9DbXyLDLUx2pLSKiGCj0dcTE7sR3boFSW2l0Zc3k3AXDSvJM275354BOesBDRFdYNBDtmlFKRLoLsBq1fzr+xz+vGwH08d05uYB8QztEml2WcIFSLgLyP0eDqwxZoceP2is0PMzjdfa9oChjxrXoUd2MbdO8TP7isqY8ekWvt9/jMsTwxnSWUJdnCbh7u1yv4d/TDCubjmlfS8Y/oSxQm+TYF5t4qw+3pDDrPRtBPj58MI1PflV32i5y1T8jIS7t9v7Vb1gVzDwXhg1x9SSROOiWwcypLPR6CuypdzVK35Jwt3bFe00/q18wDfAGEUnXE51nYX/W7EHgD+NlkZfonES7t7swFrYtsA4nt6hlzHJKCbN7KrEGTYePMbD87PYW1TOtanS6EvYR8LdW1WXwcI7oXU8XPkW+AeZXZE4Q3l1HX9ZvpMP1h2gQ2gLPvhNGoOTZTqSsI9ddzcopcYopXYqpfYopWacZZtrlVLZSqltSql/O7ZM4XBfzIITOTDlTQl2F1VwopJ/f5/DLf3jWP7AIAl2cV4aXbkrpXyB14GRQB6wQSmVobXOrrdNEvAIcJnW+rhSSq7JcmV7V0LmezDgHogbYHY1op6SilqWbDnEDf2MRl9rHh5KWzlhKi6APYdl0oA9Wut9AEqpecBkILveNr8DXtdaHwfQWhc6ulDhIFWlkHEvhCfDsMfMrkbU89nWwzyevpVj5TX06xRGQkSwBLu4YPYclokCcus9zrM9V18ykKyUWquUWq+UGtPQGymlpimlMpVSmUVFRRdWsbg4yx+F0nzjcEwz6RDoCgpPVnHXvzZyxz83EhEcQPrdl5EQIY2+xMWxZ+Xe0Gl53cD7JAFDgGhgjVKqu9b6xM++Seu5wFyA1NTUM99DNLVdn8MPH8HlD0J0qtnVCIxGX9e+tY6CkioeGt2ZaYM6SaMv4RD2hHseEFPvcTRQ0MA267XWtcB+pdROjLDf4JAqxcWrPA6L7jN6wgxp8Jy4cKJDJZW0DWluNPqa1I2Y1oHSllc4lD1LhA1AklKqo1LKH5gKZJyxzUJgKIBSKhzjMM0+RxYqLtKy6VBeZByO8QswuxqvZbVq/rF2P8Nf+pp/fncQgKGdIyXYhcM1unLXWtcppe4BlgO+wPta621KqdlAptY6w/baKKVUNmABHtJaFzdl4eI8bF8MWR/D4BnGzUrCFHsKy5jxSRaZB48zKDmCYdK9UTQhpbU5h75TU1N1ZmamKZ/tVcqL4Y1+ENIefveVzC41ybzvc5iVsY0WzXyZNSGFq/pEyV2m4oIopTZqrRs9aSZ3qHq6JQ8a801vSZdgN1Fsm0BGdI3kqUndiQiRw2Ki6Um4e7Ktn0D2Qhg+C9p2M7sar1JVa+GvK3YD8PCYLgxMCGdggjT6Es4j11x5qpNHYMkfIaovDLzf7Gq8SuaBY4z76xreWLWXY+U1mHXoU3g3Wbl7Iq1h8R+gthKmvAW+8mN2hrLqOv7y2Q4+XH+QqFYt+PA3aQySfjDCJPKn3hP9OA92LoVRz0BEstnVeI3DJZXM25DLrQPieWh0Z4IC5I+XMI/89nmaknzjmvbYAdD/TrOr8XjHy2tYvOUQN/ePIzHSaPQlk5GEK5Bw9yRaG03BrLUw5Q3w8TW7Io+ltWbZ1sPMSt/KiYpaBia0ISEiWIJduAwJd0+y6QPYuwLGvQhhncyuxmMVllbxePpWlm87Qo+oUD78TT9p9CVcjoS7pzh+EJbPhI6DIPV2s6vxWBar5ldvr+NwSRWPjO3C7Zd3xE8afQkXJOHuCaxWSL8bUDD5dfCRsHG0ghOVtGtpNPqaPbk7Ma1b0ElW68KFSQp4gg3vwoE1MPoZaBVrdjUexWLV/P2MRl+DkyMk2IXLk5W7uyveC18+AYkjoc8tZlfjUfYUnuTh+VlsyjnBkM4RDO/a1uyShLCbhLs7s1pg4V1Gz5hJfwVpROUw//4uhyczthEU4Msr113ClF7S6Eu4Fwl3d7b+DchdD1e+DS07mF2NR4kPD2RUt7Y8Oakb4cHS6Eu4Hwl3d1W0E1bMgc7joed1Zlfj9qpqLbzy5S4UihljpdGXcH9yQtUdWepgwR3gHwQTX5XDMRfpu33FjH1tDW9/vY+TVbXS6Et4BFm5u6O1r0LBJrjm7xAs03wu1MmqWv782Q7+uT6H2LBA/v3bfgxMlNW68AwS7u7m8FZY9Tx0uxK6X2V2NW7tSGk18zfm8dvLO/LgqGQC/eWPg/Ac8tvsTupqYOEd0KIVjHvJ7Grc0rHyGpZkFXDzgHgSI4NZ8/AwmYwkPJKEuztZ8yIc3gLX/QuC2phdjVvRWrM46xBPZmyjtKqWyxLD6RQRLMEuPJaEu7so2AyrXzSujOk6wexq3MqR0ipmLtjKl9uP0DM6lH9d00/uMBUeT8LdHdRVG1fHBEfC2D+bXY1bsVg119oafc0c15VfXxYvjb6EV5BwdwernoOi7XDjfGjR2uxq3ELe8Qrah7bA10cxZ3J3YsMCiQ8PMrssIZxGljCuLncDrH0Net8MSSPNrsblWayad9fsY8TLX/PP9Uajr0HJERLswuvIyt2V1VbCwjuhZRSMftbsalzezsMnefiTLH7MPcHwLpGM6iaNvoT3knB3ZSvmQPFuuCUdmrc0uxqX9s/1B3lq0TZCmjfjtam9mHRJB2n0JbyahLurOvit0Rjs0t9CpyFmV+OytNYopUiMDGZcj/bMmpBCG2n0JYSEu0uqLjMOx7SOgxFPmV2NS6qssfDyFzvx8VE8MrYr/Tu1oX8nufZfiFPkhKor+vIJYybqlDchQK7HPtO6vcWMeW0176zZT0W1RRp9CdEAWbm7mn2rjLF5/e+GuIFmV+NSSqtqeW7pDv7zfQ5xbQL59+/6SVteIc7CrpW7UmqMUmqnUmqPUmrGOba7RimllVKpjivRi1SVQvo90CYJhj9udjUup7C0moU/5DNtUCc+u3+QBLsQ59Doyl0p5Qu8DowE8oANSqkMrXX2GduFAPcB3zVFoV5h+aNQmg+3fwHNWphdjUsoLqtm0Y8F3HZZRxIjg/lm+lA5YSqEHexZuacBe7TW+7TWNcA8YHID280BXgCqHFif99j1OfzwEVx2P0TLX3y01qRvzmfEy1/zzNLt7CsqA5BgF8JO9oR7FJBb73Ge7bmfKKV6AzFa68UOrM17VB6HRfdBZAoMecTsakxXcKKS2z/I5P55m4lrE8SS+66QRl9CnCd7Tqg2dCfIT5cnKKV8gFeA2xp9I6WmAdMAYmNj7avQGyybDuVFcP088PPulWmdxcrUuespOlnN4xNSuG1gPL4+cjOSEOfLnnDPA2LqPY4GCuo9DgG6A6tsdwS2AzKUUpO01pn130hrPReYC5CamirXrwFsXwxZH8PgGdChl9nVmCb3WAUdWrXAz9eHZ6/sQWxYILFtAs0uSwi3Zc9hmQ1AklKqo1LKH5gKZJx6UWtdorUO11rHa63jgfXAL4JdNKC8GBb/Adr1hEF/MrsaU9RZrMxdvZcRL3/NR+sOAHB5UrgEuxAXqdGVu9a6Til1D7Ac8AXe11pvU0rNBjK11hnnfgdxVksehMoTRu8Y32ZmV+N02w+VMv2TLLLyShiZ0paxPdqbXZIQHsOum5i01kuBpWc8N+ss2w65+LK8wNZPIHshDJ8FbbuZXY3TfbTuAE8tyia0RTP+dkNvxvdoL42+hHAguUPVDCePwJI/QlRfGHi/2dU41alGX8ltQ5h4SQcen5BCWJC/2WUJ4XEk3J1Na+M4e02F0TvG1zt+BBU1dby4fBd+vopHx3WlX6c29JNGX0I0GWkc5mw/zoOdS432AhGdza7GKdbuOcroV1fz/tr91NRZpdGXEE7gHctGV1GSb1zTHtMf+t9ldjVNrqSylmeXbOfjzFw6hgfx398PIK1jmNllCeEVJNydRWvIuBestTDlDfDxNbuiJne0rJpFWQXcMTiBP4xIonkzz99nIVyFhLuzbPoA9q6AsX+BNglmV9Nkik4ajb5+c3lHEiKC+Wb6MDlhKoQJJNyd4fhBWD4T4q8wxuZ5IK01Czfn89SibCqqLQztEknH8CAJdiFMIuHe1KxWSL/b+Hry6+Djeeew809UMnPBFlbtLKJPbCteuKYnHcODzC5LCK8m4d7UNrwLB9bAxNeMmagexmj0tY7ishqenJjCzQOk0ZcQrkDCvSkV7zXmoSaOgD63ml2NQ+UUVxDV2mj09fxVPYkNCyQmTPrBCOEqPO8YgauwWozDMT7NYOJfwUNura+zWHlz1V5GvPI1H647AMBlieES7EK4GFm5N5X1b0LOOpjyFoRGNb69G9hWUML0T7LYml/K6G5tGS+NvoRwWRLuTaFoF6yYDZ3HwSVTza7GIT749gBzFmfTKtCfN2/sIx0chXBxEu6OZqmDhXeAfyBMeNXtD8ecavTVpV0Ik3tF8fiErrQKlMsbhXB1Eu6O9u1rkL8RrnkfQtqaXc0FK6+u4y/Ld9LMVzFzfIo0+hLCzcgJVUc6sg1WPgcpU6D71WZXc8FW7ypi1Cur+WDdAWotWhp9CeGGZOXuKJZaWHAHtGgF4182u5oLUlJRy5wl2czfmEenCKPR16Xx0uhLCHck4e4oq1+Ew1lw3b8gyD0PXxwtr2bZlkPcNSSB+4ZLoy8h3JmEuyMUbIY1L0LP66DrBLOrOS+FJ6vI2FzAb6/o9FOjr9bSD0YItyfhfrHqqo3DMUERMPbPZldjN601n2zKZ87ibCprLQzv2paO4UES7EJ4CAn3i7XqOSjaDjf8D1q0Nrsau+Qeq+DRBVtYs/soqXGtef5qafQlhKeRcL8YuRtg7WvQ+yZIHmV2NXaps1i5/p31HC+vYc7kbtzYLw4fafQlhMeRcL9QtZWw8E4I6QCjnzW7mkYdOFpOTFggfr4+vHCN0egrurX0gxHCU8l17hdqxRwo3g2T/wbNQ82u5qxqLVZeX7mHUa+s/qnR18CEcAl2ITycrNwvxMFvYf0bkHo7JAw1u5qz2ppfwsPzs8g+VMr4Hu2Z0LOD2SUJIZxEwv18VZcZh2Nax8HI2WZXc1Z/X7ufp5dsJyzIn7du6suY7u3MLkkI4UQS7ufryyeMmai3LYGAYLOr+YVTjb66dQjlqt5RPDY+hdDAZmaXJYRwMgn387FvlTE2r/9dEH+Z2dX8TFl1HS98tgN/Xx8em5BCWscw0jpK6wDMbP3CAAANJElEQVQhvJWcULVXVSmk3wNtEmH4LLOr+ZlVOwsZ/cpqPlp/EA3S6EsIISt3uy1/FErz4TefQ7MWZlcDYFyrviSbTzflkxgZzPw7BtI3zj1upBJCNC0Jd3vs+hx++Agu+wPEXGp2NT85XlHD59uOcN+wRO4elkiAnzT6EkIY7Doso5Qao5TaqZTao5Sa0cDrDyqlspVSWUqpFUqpOMeXapLK47DoPojoCkMfNbsaCkurmLt6L1prOkUEs3b6MB4c1VmCXQjxM42Gu1LKF3gdGAukANcrpVLO2OwHIFVr3ROYD7zg6EJNs2w6lBXClW+CX4BpZWit+e+GXIa//DUvfb6LA8UVAHIljBCiQfYclkkD9mit9wEopeYBk4HsUxtorVfW2349cJMjizTN9sWQ9TEMng4deptWRu6xCh75dAvf7DlKWscwnr+qhzT6EkKckz3hHgXk1nucB/Q7x/a3A8saekEpNQ2YBhAbG2tniSYpL4bFf4B2PeCKP5lWxqlGXycqanl6SnduSIuVRl9CiEbZE+4NJUmD19oppW4CUoHBDb2utZ4LzAVITU117ev1ljwIlSfglnTwc36P8/1Hy4m1Nfr6yzWXENcmkA6tXOMqHSGE67PnhGoeEFPvcTRQcOZGSqkRwExgkta62jHlmWTrJ5C9EIbMgLbdnPrRtRYr/7diN6NfWc0H3x4AYEBCGwl2IcR5sWflvgFIUkp1BPKBqcAN9TdQSvUG3gbGaK0LHV6lM508Akv+CFF9jUsfnSgr7wQPz89ix+GTTLykA5N6SaMvIcSFaTTctdZ1Sql7gOWAL/C+1nqbUmo2kKm1zgD+AgQD/1NKAeRorSc1Yd1NQ2tY/ADUVMCUN8HXebcBvP/Nfp5ekk1ESADv3JLKyJS2TvtsIYTnsSu9tNZLgaVnPDer3tcjHFyXObI+hp1LYNTTENHZKR95qtFXz+hQrrs0hhljuxLaQi5vFEJcHLlD9ZTSAlj6MMT0NxqDNbGTVbU8v2wHAX6+zJqYQmp8GKnx0uhLCOEY0jgMjMMxGfeCpQamvAE+TXu358odhYx6ZTX/+T4HP18ljb6EEA4nK3eATR/Cni9h7AvQJqHJPuZYeQ2zF21j4eYCktsG88aNA+kdK42+hBCOJ+F+IgeWz4T4K+DS3zXpR5VU1rJieyH3D0/i7qGJ+PvJX5yEEE3Du8PdaoX0uwENk18HH8eH7eGSKhZuzuf3gzrRMTyIb2YMkxOmQogm593hnvke7F8NE141ZqI6kNaaeRtyeXbJdmqtVsZ0a0d8eJAEuxDCKbw33Iv3whezIGE49L3NoW99sLicGZ9sYd2+Yvp3CuP5q3oSL42+hBBO5J3hbrUYh2N8msGk/wPluEZcdRYrN7zzHSWVtTx7ZQ+mXhojjb6EEE7nneG+/k3IWWfchRoa5ZC33FtURpyt0ddL1xqNvtqHSj8YIYQ5vO9yjaJdsGI2JI+FS66/6LerqbPy6pe7GPPqaj5cdxCA/p3aSLALIUzlXSt3Sx0svAP8A2Hiaxd9OGZz7gmmz89i55GTTO7VgSm9HfO3ACGEuFjeFe7fvgb5G+Hq9yDk4hpzvffNfp5Zkk1kSHPeuzWV4V2l0ZcQwnV4T7gf2QYrn4OUydD96gt+m1ONvnrFhDI1LZYZY7vQsrlc3iiEcC3eEe6WWlhwBzQPhfEvX9DhmNKqWp5buoPmzXx4YmI3+saF0TdOGn0JIVyTd5xQXf0iHM6Cia9CUPh5f/uX2UcY+fLXfLwhB38/H2n0JYRweZ6/ci/YDGtehB7XQteJ5/WtxWXVPLUom4wfC+jSLoS5N6dySUyrJipUCCEcx7PDva7aOBwTGA7jXjjvbz9ZVcfKnYU8MCKZO4ckSKMvIYTb8OxwX/UcFG2HG/4HLexrrVtwopIFP+Rz15AE4sODWDtjmJwwFUK4Hc8N99wNsPY16H0TJI9qdHOrVfPv73N4ftkOLFbN+B7tiQ8PkmAXQrglzwz32kpYeCeEdIDRzza6+f6j5cz4JIvv9h/jssQ2PHdlT2LbBDqhUCGEaBqeGe4r5kDxbrh5oXH54znUWazc9O53lFbV8sLVPflVajTKgY3EhBDCDJ4X7ge/hfVvQOrtkDD0rJvtKTxJfJsg/Hx9eOW6XsS1CaRty+ZOLFQIIZqOZ13+UV1mHI5pFQsjZze8SZ2Fl7/YxZhX1/CBrdFXWscwCXYhhEfxrJX7l0/A8QNw2xIICP7Fy5tyjjN9fha7C8u4qncUV0mjLyGEh/KccN+3Cja8C/3uhPjLf/HyO6v38eyy7bRv2Zy///pShnaOdH6NQgjhJJ4R7lWlkH4PhCXA8Fk/e8lq1fj4KPrEteLGfrFMH9OFELm8UQjh4Twj3D+fCaX58JvlRq92oKSylmeWZNOimS9PTe4ujb6EEF7F/U+o7v4CNn0IA++FmDQAlm87zMiXv+aTTfkEBfhJoy8hhNdx75V75XHIuBciusCQRzlaVs0T6dtYsuUQKe1b8v5tl9I96tzXuQshhCdy73BfNgPKCuH6/0Cz5pSVlLNmdxEPje7MtEGdaObr/n8xEUKIC2FX+imlxiildiql9iilZjTweoBS6mPb698ppeIdXegv7FgCWfMoTbuPv+0IRmtNfHgQ3z4ynLuHJkqwCyG8WqMJqJTyBV4HxgIpwPVKqZQzNrsdOK61TgReAf7s6EJ/prwYveh+joV0ZtC6VF5fuZeDxRUABAe4919GhBDCEexZ3qYBe7TW+7TWNcA8YPIZ20wGPrB9PR8YrpqqQUvu91S/Nw5L+TFuOPpresRF8PkDg4gPD2qSjxNCCHdkzzI3Csit9zgP6He2bbTWdUqpEqANcNQRRf4k93v0P8YRYKmlFl8eGhbNsBFp0uhLCCHOYM/KvaHkPPPaQnu2QSk1TSmVqZTKLCoqsqe+nzuwBmWxAOCnYHjzXRLsQgjRAHvCPQ+Iqfc4Gig42zZKKT8gFDh25htpredqrVO11qkRERHnX238FeAXAMoX5etvPBZCCPEL9hyW2QAkKaU6AvnAVOCGM7bJAG4F1gHXAF/pprhzKCYNbs2AA2uMYLfdtCSEEOLnGg132zH0e4DlgC/wvtZ6m1JqNpCptc4A3gM+UkrtwVixT22yimPSJNSFEKIRdl03qLVeCiw947lZ9b6uAn7l2NKEEEJcKLnTRwghPJCEuxBCeCAJdyGE8EAS7kII4YEk3IUQwgMpswZZKKWKgIMX+O3hOLq1geuTffYOss/e4WL2OU5r3ehdoKaF+8VQSmVqrVPNrsOZZJ+9g+yzd3DGPsthGSGE8EAS7kII4YHcNdznml2ACWSfvYPss3do8n12y2PuQgghzs1dV+5CCCHOwaXD3SUHczcxO/b5QaVUtlIqSym1QikVZ0adjtTYPtfb7hqllFZKuf2VFfbss1LqWtvPeptS6t/OrtHR7PjdjlVKrVRK/WD7/R5nRp2OopR6XylVqJTaepbXlVLqr7b/HllKqT4OLUBr7ZL/YLQX3gt0AvyBH4GUM7a5C3jL9vVU4GOz63bCPg8FAm1f3+kN+2zbLgRYDawHUs2u2wk/5yTgB6C17XGk2XU7YZ/nAnfavk4BDphd90Xu8yCgD7D1LK+PA5ZhTLLrD3znyM935ZW7aw3mdo5G91lrvVJrXWF7uB5jMpY7s+fnDDAHeAGocmZxTcSeff4d8LrW+jiA1rrQyTU6mj37rIGWtq9D+eXEN7eitV5NAxPp6pkMfKgN64FWSqn2jvp8Vw73hgZzR51tG611HXBqMLe7smef67sd4//87qzRfVZK9QZitNaLnVlYE7Ln55wMJCul1iql1iulxjituqZhzz4/CdyklMrDmB9xr3NKM835/nk/L3YN6zCJwwZzuxG790cpdROQCgxu0oqa3jn3WSnlA7wC3OasgpzAnp+zH8ahmSEYfztbo5TqrrU+0cS1NRV79vl64B9a65eUUgMwprt111pbm748UzRpfrnyyt1hg7ndiD37jFJqBDATmKS1rnZSbU2lsX0OAboDq5RSBzCOTWa4+UlVe3+307XWtVrr/cBOjLB3V/bs8+3AfwG01uuA5hg9WDyVXX/eL5Qrh/tPg7mVUv4YJ0wzztjm1GBuaMrB3M7T6D7bDlG8jRHs7n4cFhrZZ611idY6XGsdr7WOxzjPMElrnWlOuQ5hz+/2QoyT5yilwjEO0+xzapWOZc8+5wDDAZRSXTHCvcipVTpXBnCL7aqZ/kCJ1vqQw97d7DPKjZxtHgfswjjLPtP23GyMP9xg/PD/B+wBvgc6mV2zE/b5S+AIsNn2T4bZNTf1Pp+x7Src/GoZO3/OCngZyAa2AFPNrtkJ+5wCrMW4kmYzMMrsmi9yf/8DHAJqMVbptwN3AHfU+xm/bvvvscXRv9dyh6oQQnggVz4sI4QQ4gJJuAshhAeScBdCCA8k4S6EEB5Iwl0IITyQhLsQQnggCXchhPBAEu5CCOGB/h8D/9zNcduMmQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# calculate roc curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, bag_log_reg_pred)\n",
    "# plot no skill\n",
    "plt.plot([0, 1], [0, 1], linestyle='--')\n",
    "# plot the roc curve for the model\n",
    "plt.plot(fpr, tpr, marker='.')\n",
    "# show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X2clXP+x/HXZ2aa7qhkclNNN1LIbRzd0Z1IJYWSoigplVjssiyWZVnW3f7ovqQ7ShJGUtGNbpSalJKkJErRkEJ0//39cU27UzM1pzpnvnPOeT8fjx4751yXc95XM733zPW9ru/XnHOIiEh8SfIdQEREIk/lLiISh1TuIiJxSOUuIhKHVO4iInFI5S4iEodU7iIicUjlLiISh1TuIiJxKMXXG6elpbkqVar4ensRkZi0aNGiH51z5fLbz1u5V6lShczMTF9vLyISk8zsm3D202kZEZE4pHIXEYlDKncRkTikchcRiUMqdxGROJRvuZvZMDPbZGafHWS7mdkLZrbazJaa2fmRjykiIocjnE/uw4Hmh9jeAqie/acHMODoYx3CugUw+9ngf0VEJE/5XufunJtlZlUOsUsbYKQL1uubb2ZlzOxk59zGCGX8n3ULYEQr2L0TUorCTe9Aeu2Iv42ISKyLxDn3CsC6HI/XZz+Xi5n1MLNMM8vMyso6/HdaOzsodhzs3g6LRhxJXhGRuBeJcrc8nstz1W3n3GDnXMg5FypXLt+7Z3Or0gBSioElBW+7ZDS8cQv8tunwX0tEJI5FYvqB9UB6jscVgQ0ReN3c0mvDTRnBJ/iKtWHtHJjzHKyaCpc+Aud3gSRdACQiEokmzABuzL5qpi6wNSrn2/dJrw0N/gxVG0CT+6HXR3DSOTDxLhh2OfywPGpvLSISK8K5FHIMMA84zczWm1k3M+tpZj2zd5kErAFWA0OA3lFLm5e06sHA6lUDYfNXMKghvP932LmtQGOIiBQmFlzkUvBCoZCL+KyQv28Oin3xKChdCa54Fmo0i+x7iIh4ZGaLnHOh/PaLrxPUJcpCm77Q9T0oUhxevRbG3Qi/RO8skYhIYRRf5b5P5frQcw5c8hB8OQX6XggfD4K9e3wnExEpEPFZ7gApqdDwL9B7HqRfCO/dC0ObwoYlvpOJiERd/Jb7PmVPgU4ToO1LsPU7GNIEJt8PO371nUxEJGriv9wBzODsdtBnIVzQFeYPgH51YMVE38lERKIiMcp9n+JloNVz0O19KH4cvHYDjOkIW9bl/9+KiMSQxCr3fdIvhB4z4bLHYM3M4FP8Ry/Cnt2eg4mIREZiljtAchG46A647ePgbtepD8KQxrB+ke9kIiJHLXHLfZ8ylaDjWGg/Crb9FFxR8+6fYftW38lERI6Yyh2CAdearYNP8XVuhcxhwbXxn00AT3fwiogcDZV7TsVKQYunoPt0OPYkGN8VXmkHm7/2nUxE5LCo3PNSvhbcMh2aPwnfzof+dWH2c7Bnl+9kIiJhUbkfTHIK1O0Fty2A6pfBtH/AwAZB2YuIFHIq9/yUrgDXjYaOr8HO34I54zNuD2agFBEppFTu4TqteTDgWv92WPxKMOD66WsacBWRQknlfjhSS0Kzf8KtH8JxVeDNHjCyDfy42ncyEZH9qNyPxElnB1MYXPFcMMvkgPow8ynYvcN3MhERQOV+5JKS4MJuwWRkZ7SCmU/AgIvg69m+k4mIqNyP2rEnQrth0OkN2LsLRrSCN3vCth99JxORBKZyj5RTL4Xe86HBn2HZeOgbgk9GacBVRLxQuUdSkeLQ9O/BEn/lToeMPvByS9j0he9kIpJgVO7RcMLp0GUStH4RslbAwIth2mOw6w/fyUQkQajcoyUpCc6/EfpkBqtAzX4G+teD1dN8JxORBKByj7aSaXD1QLgxA5KSYfQ1ML4b/PqD72QiEsdU7gXllEbQ6yNofD+syAjucF34Euzd6zuZiMQhlXtBSikKje+DXvOg/Lnw7t0wrBl8/5nvZCISZ1TuPqSdGpymuXoQbF4DgxrC1Idg5zbfyUQkTqjcfTGDczsEA661boCPXoB+dWHlZN/JRCQOqNx9K1E2uGSy62RILQFjroPXOsMvG3wnE5EYpnIvLCrXg1tnBzdBrZoKfWvD/IGwd4/vZCISg8IqdzNrbmYrzWy1md2Xx/bKZjbNzJaa2Uwzqxj5qAkgJTWYvqD3fEivDZP/CkMugQ2LfScTkRiTb7mbWTLQD2gB1AQ6mlnNA3Z7BhjpnDsHeBT4V6SDJpSyVYOJyNoNg183BgX/3n2w41ffyUQkRoTzyb02sNo5t8Y5txMYC7Q5YJ+awL5bL2fksV0Olxmc1TZYwzV0M3w8MDhV83mGJiMTkXyFU+4VgHU5Hq/Pfi6nT4G22V9fDRxrZscffTyheBm44lm45QMocTyM6wxjOsKWb30nE5FCLJxytzyeO/Cj41+ARma2GGgEfAfszvVCZj3MLNPMMrOysg47bEKrGIIeM4Nl/r7+EPrVgbkvwJ5dvpOJSCEUTrmvB9JzPK4I7HednnNug3PuGudcLeCB7Oe2HvhCzrnBzrmQcy5Urly5o4idoJJTggW6b/sYqjaC9x+CwY1h3ULfyUSkkAmn3BcC1c2sqpmlAh2AjJw7mFmame17rfuBYZGNKfspUwk6joHrRsPvm+Gly2Di3fDHFt/JRKSQyLfcnXO7gT7AFGAFMM45t9zMHjWz1tm7NQZWmtmXwInA41HKK/uYwRlXQp8FUKcnLHoZ+tWGz97QgKuIYM5TEYRCIZeZmenlvePShiXwzp9g4xKo1jQYhC1b1XcqEYkwM1vknAvlt5/uUI0X5c+D7tOhxb9h3QLoXxdmPQO7d/pOJiIeqNzjSVIy1Lk1OFVTvRlMfwwGNYBv5vlOJiIFTOUej0qVh+tGQcfXgmmEX24Ob/cJBl9FJCGo3OPZac2Dyybr3wFLXoW+IVgyRgOuIglA5R7vUktCs8fg1llQ9hR4qyeMbA0/rvadTESiSOWeKE46C26eClc8Bxs+hQH1YMa/YNd238lEJApU7okkKQku7AZ9FsIZreHDJ2HgRfD1LN/JRCTCVO6J6NgTod1L0GkC7N0NI66ECbfCth99JxORCFG5J7JTmwYLgzT4S3Bn64sXwCcjYe9e38lE5Cip3BNdkeLQ9CHoOQdOqAkZt8PwlrBphe9kInIUVO4SOOF06PIutO4LWV/AwIth2qOw6w/fyUTkCKjc5X+SkuD8ztAnE85uD7OfDaYxWP2B72QicphU7pJbyTS4egDc9A4kpcDotvB6V/j1e9/JRCRMKnc5uKoNoddH0Phv8MW7wRquC4dqwFUkBqjc5dBSikLjvwYlX/5cePfPweIg3y/znUxEDkHlLuFJOxVuzICrB8PPa2FQI5jyAOz4zXcyEcmDyl3CZwbnXhfc4VqrE8zrGwy4rnzPdzIROYDKXQ5fibLQ+gXoOhlSj4ExHWDsDbD1O9/JRCSbyl2OXOV6wWyTTR+G1dOCNVznD4C9e3wnE0l4Knc5Oimp0OBu6D0PKtWFyffBkCbw3Se+k4kkNJW7REbZqnDDeGj3cnA9/NCm8N5fYfsvvpOJJCSVu0SOGZx1TTDgGuoGHw8KTtV8/rZWfxIpYCp3ibxipeGKZ+CWaVAiDcbdCK9eBz9/4zuZSMJQuUv0VLwAesyEZo/D2jnBZZNz/w/27PKdTCTuqdwlupJToH6fYKHuUxrD+38PboBat8B3MpG4pnKXglEmHTqOgetege1b4KVmMPEu+GOL72QicUnlLgXrjFbBp/i6vWHRcOh7ISwbrwFXkQhTuUvBK3osNH8Cus+A0hXgjW4w+hrYvMZ3MpG4oXIXf8qfF1xR0+JpWLcQ+teDWU/D7p2+k4nEPJW7+JWUDHV6QJ8FUONymP7PYIm/tXN9JxOJaSp3KRxKlYf2I+H614N1W4e3hLdvg983+04mEpPCKncza25mK81stZndl8f2SmY2w8wWm9lSM2sZ+aiSEGo0g9vmw0V/gk/HQt8QLHlVA64ihynfcjezZKAf0AKoCXQ0s5oH7PYgMM45VwvoAPSPdFBJIKkl4bJHgxkny1aDt3rBiCvhx1W+k4nEjHA+udcGVjvn1jjndgJjgTYH7OOAUtlflwY2RC6iJKwTz4Sbp0Cr5+H7pTCgPsx4AnZt951MpNALp9wrAOtyPF6f/VxOjwCdzGw9MAm4Pa8XMrMeZpZpZplZWVlHEFcSTlIShG6GPplQsw18+FRQ8mtm+k4mUqiFU+6Wx3MHngDtCAx3zlUEWgKjzCzXazvnBjvnQs65ULly5Q4/rSSuY06AtkOh85vg9sLINjChB/ymDwkieQmn3NcD6TkeVyT3aZduwDgA59w8oBiQFomAIvupdkmwMEjDe+CzCcGA66IRsHev72QihUo45b4QqG5mVc0slWDANOOAfb4FmgKY2RkE5a6PVBIdRYrDJQ9Cr7nBefl37oCXW8CmFb6TiRQa+Za7c2430AeYAqwguCpmuZk9amats3f7M9DdzD4FxgBdnNO1axJl5U6DLu9Cm37w45fBzU8fPAI7f/edTMQ789XBoVDIZWZmenlviUPbfoL3H4Ilr0CZynDFc1D9Ut+pRCLOzBY550L57ac7VCU+lDweruoPN02E5FR4pS283iVYz1UkAancJb5UbRCci2/yAHwxKZhSeMEQ2LvHdzKRAqVyl/iTUhQa3RtcVVO+Fkz6C7x0GWxc6juZSIFRuUv8Or4a3Pg2XDMEtnwLgxvDlAdgx2++k4lEncpd4psZnNMe+iyE8zvDvL7Qr05wykYkjqncJTEUPw6u/L9grppipWBsRxh7A2xd7zuZSFSo3CWxVKobzDZ56SOwelrwKX5ef9iz23cykYhSuUviSS4CF98VzBtfqR5MuR+GNIHvFvlOJhIxKndJXMdVgRteh2uHw2+bYEhTmHQPbP/FdzKRo6Zyl8RmBmdeHazhWrt7cE18v9qw/C2t/iQxTeUuAlCsNLR8Gm6ZBiXT4PWb4NX28PM3vpOJHBGVu0hOFS+A7jPh8idg7dxgwHXOf2DPLt/JRA6Lyl3kQMkpUO82uO3jYP74Dx6GQY1g3QLfyUTCpnIXOZgy6dDxVejwKmzfEkxh8M6f4I+ffScTyZfKXSQ/p18RfIqvext8MjKYjGzp6xpwlUJN5S4SjqLHQvMnoMdMKJ0OE26BUVfDT1/5TiaSJ5W7yOE4+Vy45QNo+Qysz4T+9eDDp2H3Dt/JRPajchc5XEnJwTXxfRbCaS1gxj+DJf7WzvGdTOS/VO4iR6rUydB+BFz/OuzeDsOvgLduC5b8E/FM5S5ytGo0g94fw0V3wtKx0DcEi1/RgKt4pXIXiYTUEnDZP4IZJ9Oqw9u9YXgryPrSdzJJUCp3kUg68UzoOjmYO/6HZTCgPkx/HHZt951MEozKXSTSkpLggi7QJzOYlGzWv2FAPfhqhu9kkkBU7iLRcswJ0HYIdH4reDzqKnijezC9sEiUqdxFoq1aE+g1DxreC8vfDAZcM1+GvXt9J5M4pnIXKQhFisElD0Cvj+DEs2HinfByc/hhue9kEqdU7iIFqVwN6DIR2vSHH1fBoIbw/sOw83ffySTOqNxFCpoZ1LohGHA9pwPM/Q/0rwNfTvWdTOKIyl3El5LHw1X9oMu7kFIMXr0Wxt0Ev2z0nUzigMpdxLcqF0PPOdDkQVj5XrCG68eDYe8e38kkhoVV7mbW3MxWmtlqM7svj+3Pm9mS7D9fmtmWyEcViWMpRaHRPdB7HlQ4H967B4ZeChs/9Z1MYlS+5W5myUA/oAVQE+hoZjVz7uOcu8s5d55z7jzgRWBCNMKKxL3jqwXXxV8zFLaug8GNYfLfYMdvvpNJjAnnk3ttYLVzbo1zbicwFmhziP07AmMiEU4kIZnBOdcGUwqffxPM7xecqvniXd/JJIaEU+4VgHU5Hq/Pfi4XM6sMVAWmH300kQRX/Di48j9w81QoVhrGXg9jroet630nkxiQEsY+lsdzB5vLtAMw3jmX50iQmfUAegBUqlQprIAiCa9SnWC2yXn9YOaT0Lc21OoEJcvBKY0gvbbvhFIIhfPJfT2QnuNxRWDDQfbtwCFOyTjnBjvnQs65ULly5cJPKZLokovAxXfCbfODmScXDApWgBrRCtYt8J1OCqFwyn0hUN3MqppZKkGBZxy4k5mdBhwHzItsRBH5r+OqQI3m/PcX6t07YNazmqdGcsm33J1zu4E+wBRgBTDOObfczB41s9Y5du0IjHVOy8+IRFXVBsFNT5YMlgSrJsPoa+CXg/1CLYnIfHVxKBRymZmZXt5bJOatWwBrZ0Pli+GHz2Dqg5CcCq2eh7Ou8Z1OosjMFjnnQvntF86AqogUNum1/zeQWqkOVG0Eb/aA8V3hy8nQ4t9QvIzfjOKVph8QiQdpp8LNU6Dx/bBsPAy4CL6e7TuVeKRyF4kXyUWg8X3QbWowncGIK2HKA8GgqyQclbtIvKkYgp6zIdQV5vWFwU3g+898p5ICpnIXiUepJYPB1evHwbYsGNIE5r6gSyYTiMpdJJ7VuDyYabJ6M3j/IRjZGrZ86zuVFACVu0i8K5kG142GNv1gw+JgsPXT10C3pMQ1lbtIIjAL5qPpOQdOqBlcNvl6F/h9s+9kEiUqd5FEUrYqdJ0ETR8OphDuXw9WT/OdSqJA5S6SaJKSocHd0H1aMJXw6Gtg0j2w83ffySSCVO4iierkc+HWD6FOL1gwGAY3Cs7JS1xQuYsksiLFocWT0PlN2PFrsG7rrKdhz27fyeQoqdxFBKpdAr0+gjNaw/R/wvCWsHmN71RyFFTuIhIoURbaDQsW5970BQxsAJ+M1CWTMUrlLiL/s29x7l5zoXwtyLgdxt4Av2X5TiaHSeUuIrmVSYcbM6DZ47D6fRhQD1ZO9p1KDoPKXUTylpQE9ftAj5lwzIkw5jp450+w4zffySQMKncRObQTz4Tu06H+HbBoBAxqAOsW+k4l+VC5i0j+UopCs8egy0TYswuGXQ4zngi+lkJJ5S4i4atycTDYek57+PApeKkZ/LjKdyrJg8pdRA5PsdJw9UC4dgT8/HVwyeSCIbpkspBRuYvIkTnzKug1DyrXh0l/gVfawa/f+04l2VTuInLkSp0Mnd6Als/A2jnBLJOfZ/hOJajcReRomUHt7nDrbChTCcZ1hjd7wfZffCdLaCp3EYmMcjXglg+g4T2wdGyw4tM3H/lOlbBU7iISOclF4JIH4eYpwbzxL7eE9x+G3Tt8J0s4KncRibz02sGSfud3hrn/gSFNYdMK36kSispdRKKj6DHQ+kXoMAZ+3QiDGsG8/rB3r+9kCUHlLiLRdXpL6D0PqjWBKffDqKtg63e+U8U9lbuIRN8xJ0DHsXDlC7A+M5hlctl436nimspdRAqGGVxwE/ScDWk14I1uML4b/PGz72RxKaxyN7PmZrbSzFab2X0H2ae9mX1uZsvN7NXIxhSRuHF8Neg6GZo8CJ+/FVwyuWam71RxJ99yN7NkoB/QAqgJdDSzmgfsUx24H7jIOXcmcGcUsopIvEhOgUb3QLf3oUgJGNkGJv8Ndm33nSxuhPPJvTaw2jm3xjm3ExgLtDlgn+5AP+fczwDOuU2RjSkicanC+XDrLLiwO8zvB4Mbw8alvlPFhXDKvQKwLsfj9dnP5VQDqGFmc81svpk1z+uFzKyHmWWaWWZWltZkFBEgtQRc8Qzc8EZw/n3IJTDnedi7x3eymBZOuVsezx04t2cKUB1oDHQEhppZmVz/kXODnXMh51yoXLlyh5tVROJZ9UuDSyZPawEfPALDW8HP3/hOFbPCKff1QHqOxxWBDXns87Zzbpdz7mtgJUHZi4iEr0RZaD8SrhoI3y8LBlsXv6K54o9AOOW+EKhuZlXNLBXoABw4p+dbQBMAM0sjOE2zJpJBRSRBmMF5HYMVn046G97uHcw0ue0n38liSr7l7pzbDfQBpgArgHHOueVm9qiZtc7ebQrwk5l9DswA7nHO6TshIkfuuMrBmq2X/gNWTg5ufFr1vu9UMcOcp193QqGQy8zM9PLeIhJjvl8GE3rAps/hwlvgsseCgdgEZGaLnHOh/PbTHaoiUviddDZ0nwH1+sDCoTCoAXy3yHeqQk3lLiKxoUgxuPxxuDEjuNlp6GUw8ynYs9t3skJJ5S4iseWURsFg61ltYeYTMOxy+Okr36kKHZW7iMSe4mWg7RBoNwx+WgUDL4bMl3XJZA4qdxGJXWe1hV7zoOKFMPFOGNMBftPsJ6ByF5FYV7oCdH4Lmj8JX82A/vXgi0m+U3mncheR2JeUBHV7BZOQlToZxnaEt/vAjl99J/NG5S4i8eOE0+GW6XDx3bB4dHAu/tuPfafyQuUuIvElJRUufRi6vgduL7zcHKY9Crt3+k5WoFTuIhKfKteDnnPh3Oth9rPw0qWQtdJ3qgKjcheR+FWsFFzVD64bDVvXw6CG8PEg2LvXd7KoU7mLSPw748rgksmqDeG9e+GVtvDLgTOXxxeVu4gkhmNPhOvHwRXPwbfzg0sml7/pO1XUqNxFJHGYwYXd4NbZcHw1eL1LMNvk9q2+k0Wcyl1EEk/aqXDzFGh8PywbH6z4tHaO71QRpXIXkcSUXAQa3wfdpkJyarBm69QHYfcO38kiQuUuIomtYgh6zoZQV/joRRjcBH5Y7jvVUVO5i4ikloRWzwcDrtuyYHDjoOhj+JJJlbuIyD41Lofe86B6s+AUzcjWsGWd71RHROUuIpJTybTgpqc2/WDD4mCwdem4mJsrXuUuInIgM6jVCXrOgRPOgAndYXxX+H2z72RhU7mLiBxM2arQdRI0fRhWTIQB9eGr6b5ThUXlLiJyKEnJ0OBu6D4NipaCUVfDpHth1x++kx2Syl1EJBwnnwu3fgh1esGCQcEkZBuW+E51UCp3EZFwFSkOLZ6Ezm8GqzwNbQqznoG9e3wny0XlLiJyuKpdAr0+gjNaw/TH4OUWsPlr36n2o3IXETkSJcpCu2FwzVDY9EWwpN8nowrNJZMqdxGRI2UG51wLveZC+VqQ0Qde6wTbfvSdTOUuInLUyqTDjRnQ7HFYNRX614Uvp3iNpHIXEYmEpCSo3wd6zIRjToRX28M7d8LObX7ihLOTmTU3s5VmttrM7stjexczyzKzJdl/bol8VBGRGHDimdB9OtS/AxYND87Fr88s8Bj5lruZJQP9gBZATaCjmdXMY9fXnHPnZf8ZGuGcIiKxI6UoNHsMukyEPbvgpWYw41/B1wUknE/utYHVzrk1zrmdwFigTXRjiYjEgSoXB4Ot57SHD58MSn7ZBJj9LKxbENW3TgljnwpAzjkv1wN18tivrZk1BL4E7nLORW2ezOsGzcv1XKtzTqZzvSr8sXMPXV7O/ZfW7oKKXBtKZ/O2nfQavSjX9k51K3PlueXZsOUP7not911n3RucwqU1T+SrrN/424Rlubbffkl1Lq6exvINW3n0nc9zbb+3+WlcULksi77ZzL8nr8y1/e9X1uTM8qWZs+pHXpy+Ktf2J645m2rljuGDz39gyOw1ubY/f915lC9TnHc+3cDo+d/k2j6g0wWULZnK65nrGL9ofa7tw7vWpnhqMqPmrWXi0o25tr92az0ABs/6imkrNu23rViRZEbcXBuAF6atYu7q/a8UOK5EKgM7XwDAU5O/4JNvft5v+8mli/GfDrUA+Mc7y/l8wy/7bT+lXEn+dc05ANw/YSlrsvY/h1mzfCkevvJMAO4cu5iNW7fvt/38ysfx1+anA9Bz1CJ+/n3nftsvOjWNO5pWB+CmYQvYvmv/G1KannECPRpWA/Szp5+9I/zZu3ogg7+vwQ0bnqbEG10xDFKKwU0ZkF471zFHQjif3C2P5w68kPMdoIpz7hzgA2BEni9k1sPMMs0sMysr6/CSiojEsE+OacTUkq2yHznYsxPWzo7a+5nL54J7M6sHPOKcuzz78f0Azrl/HWT/ZGCzc670oV43FAq5zMyCH2QQEfFm3QIY0Too9uTUI/rkbmaLnHOh/PYL57TMQqC6mVUFvgM6ANcf8GYnO+f2/T7VGlhxWGlFRBJBeu2g0NfOhioNonZKBsIod+fcbjPrA0wBkoFhzrnlZvYokOmcywDuMLPWwG5gM9AlaolFRGJZeu2olvo++Z6WiRadlhEROXzhnpbRHaoiInFI5S4iEodU7iIicUjlLiISh1TuIiJxyNvVMmaWBeS+Vzk8aYD/2fALlo45MeiYE8PRHHNl51y5/HbyVu5Hw8wyw7kUKJ7omBODjjkxFMQx67SMiEgcUrmLiMShWC33wb4DeKBjTgw65sQQ9WOOyXPuIiJyaLH6yV1ERA6hUJd7GAtzFzWz17K3f2xmVQo+ZWSFccx3m9nnZrbUzKaZWWUfOSMpv2POsV87M3NmFvNXVoRzzGbWPvt7vdzMXi3ojJEWxs92JTObYWaLs3++W/rIGSlmNszMNpnZZwfZbmb2Qvbfx1IzOz+iAZxzhfIPwfTCXwGnAKnAp0DNA/bpDQzM/roDwSLd3rNH+ZibACWyv+6VCMecvd+xwCxgPhDynbsAvs/VgcXAcdmPT/CduwCOeTDQK/vrmsBa37mP8pgbAucDnx1ke0vgPYLV7uoCH0fy/QvzJ/dwFuZuw/+W9BsPNDWzvJYFjBX5HrNzboZz7vfsh/OBigWcMdLCXYD9MeDfwPY8tsWacI65O9DPOfczgHNuE7EtnGN2QKnsr0sDGwowX8Q552YRrG9xMG2AkS4wHyhjZidH6v0Lc7nntTB3hYPt45zbDWwFji+QdNERzjHn1I3g//ljWb7HbGa1gHTn3MSCDBZF4XyfawA1zGyumc03s+YFli46wjnmR4BOZrYemATcXjDRvDncf++HJZxl9nwJZ2HucPaJJWEfj5l1AkJAo6gmir5DHrOZJQHPE1+re4XzfU4hODXTmOC3s9lmdpZzbkuUs0VLOMfcERjunHs2e+3mUdnHvDf68byIan8V5k/u64H0HI8rkvvXtP/uY2YpBL/KHerXoMIunGPGzC4FHgBaO+d2FFC2aMnvmI8FzgJmmtlagnOTGTE+qBruz/bbzrldzrmvgZUEZR+rwjnmbsBoY1vCAAABMklEQVQ4AOfcPKAYwRws8Sqsf+9HqjCX+38X5jazVIIB04wD9skAbsr+uh0w3WWPVMSofI85+xTFIIJij/XzsJDPMTvntjrn0pxzVZxzVQjGGVo752J5jcZwfrbfIhg8x8zSCE7TrCnQlJEVzjF/CzQFMLMzCMo9q0BTFqwM4Mbsq2bqAludcxsj9uq+R5TzGW1uCXxJMMr+QPZzjxL844bgm/86sBpYAJziO3MBHPMHwA/Akuw/Gb4zR/uYD9h3JjF+tUyY32cDngM+B5YBHXxnLoBjrgnMJbiSZgnQzHfmozzeMcBGYBfBp/RuQE+gZ47vcb/sv49lkf651h2qIiJxqDCflhERkSOkchcRiUMqdxGROKRyFxGJQyp3EZE4pHIXEYlDKncRkTikchcRiUP/D/oZLZTimoFDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, bag_log_reg_pred)\n",
    "# plot no skill\n",
    "plt.plot([0, 1], [0.5, 0.5], linestyle='--')\n",
    "# plot the precision-recall curve for the model\n",
    "plt.plot(recall, precision, marker='.')\n",
    "# show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEGCAYAAADscbcsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAG3pJREFUeJzt3X+wX3V95/Hny/CjWMQAEaQkbViNVmQUIWKmbrcoawiOa+iOTGFnJUWmjCx2tdUq6s6yxTKLtSPT7CqzacmQuC6Roi6Zbdh4jWypHX6lCEREzRUVrqTE/JAyUsDcvPaP87n25Pr9cU7yvbk/vq+Hc+ae7/t8vud8boLvnM/5nM/nI9tERERvL5ruCkREzAZJlhERDSRZRkQ0kGQZEdFAkmVERANJlhERDSRZTiFJKyR9R9KopKunuz7Rn6S1knZK+uZ01yVmliTLKSJpHvAZ4ALgdOASSadPb62igZuBFdNdiZh5kiynzjnAqO3HbL8AbABWTnOdog/bdwF7prseMfMkWU6dU4Enap/HSiwiZqEky6mjDrGMLY2YpZIsp84YsKj2eSHw5DTVJSIOUZLl1LkfWCLpNElHARcDG6e5ThFxkJIsp4jtfcD7gM3Ao8Ctth+Z3lpFP5JuAe4GXi1pTNLl012nmBmUKdoiIvrLnWVERANJlhERDSRZRkQ0kGQZEdFAkuVhIOmK6a5DtJO/s5gsyfLwyP/xZp/8ncUBkiwjIhqYUe9ZLjhhnhcvOnK6qzFwP949zstOnDfd1ZgS3334xdNdhSnxM57nSI6e7moM3HP8lBf8fKd5Cxo7/y2/7N17xhuV/fuHn99se05MeXfEdFegbvGiI7lv86L+BWPGOP9XzpzuKkQL93rLIZ9j955x7tv8q43Kzjtl+4JDvuAMMaOSZUTMfAb2s3+6q3HY5ZllRLRizM883mjrRdIiSXdKelTSI5LeP+n4hyRZ0oLyWZJWl2VaHpZ0Vq3sKknby7aqFj9b0rbyndWSVOInSBop5UckHd/v906yjIjW9jf8Xx/7gA/afg2wDLhqYukVSYuAtwGP18pfACwp2xXAjaXsCcA1wJuoVii4ppb8bixlJ7438fz0amCL7SXAlvK5pyTLiGjFmHE323qex95h+4Gy/wzV7FwTqwncAHyYAyfMXgmsd+UeYL6kU4DzgRHbe2zvBUaAFeXYcbbvdtWTvR64sHaudWV/XS3eVZ5ZRkRr+5tP+r9A0tba5zW210wuJGkx8AbgXknvBH5k+6HSap7QbamWXvGxDnGAk23vgCppSzqp3y+SZBkRrRgYb54sd9le2quApGOBLwIfoGqafxxY3qlol+q0jR+UNMMjorX9uNHWj6QjqRLl521/CXgFcBrwkKQfUC3H8oCkl9N9qZZe8YUd4gBPlWY65efOfnVNsoyIVgz8zG609VJ6pm8CHrX9aQDb22yfZHux7cVUCe8s2/9AtSzLpaVXfBnwdGlKbwaWSzq+dOwsBzaXY89IWlaudSlwe7n8RmCi13xVLd5VmuER0Ypxm2Z4L28G3g1sk/RgiX3M9qYu5TcBbwdGgWeBywBs75H0Cap1rwCutT2x9vuVwM3AMcAdZQO4Hri1LBvyOHBRv8omWUZEO4bxAeRK21+n83PFepnFtX0DV3UptxZY2yG+FTijQ3w3cF6b+iZZRkQr1Qie4ZNkGREtifHeN4RzUpJlRLRSdfAkWUZE9FS9Z5lkGRHR1/7cWUZE9JY7y4iIBowYH8LxLEmWEdFamuEREX0Y8YLn5ppSvSRZRkQr1UvpaYZHRPSVDp6IiD5sMe7cWUZE9LU/d5YREb1VHTzDlzqG7zeOiEOSDp6IiIbG855lRERvGcETEdHQ/vSGR0T0Vk2kkWQZEdGTET8bwuGOw/fPQ0QcEhvG/aJGWy+SFkm6U9Kjkh6R9P4S/5Skb0t6WNKXJc2vfeejkkYlfUfS+bX4ihIblXR1LX6apHslbZf0BUlHlfjR5fNoOb643++dZBkRLYn9Dbc+9gEftP0aYBlwlaTTgRHgDNuvA74LfBSgHLsYeC2wAvispHmS5gGfAS4ATgcuKWUBPgncYHsJsBe4vMQvB/bafiVwQynXU5JlRLRiBnNnaXuH7QfK/jPAo8Cptr9ie18pdg+wsOyvBDbYft7296nWDz+nbKO2H7P9ArABWClJwFuB28r31wEX1s61ruzfBpxXyneVZBkRrY3zokYbsEDS1tp2RafzlWbwG4B7Jx16D3BH2T8VeKJ2bKzEusVPBH5SS7wT8QPOVY4/Xcp3lQ6eiGjFqM3kv7tsL+1VQNKxwBeBD9j+x1r841RN9c9PhDpWp/NNn3uU73WurpIsI6KVaincwaQOSUdSJcrP2/5SLb4KeAdwnu2JJDYGLKp9fSHwZNnvFN8FzJd0RLl7rJefONeYpCOAlwJ7etU1zfCIaEmMN9x6nqV6RngT8KjtT9fiK4CPAO+0/WztKxuBi0tP9mnAEuA+4H5gSen5PoqqE2hjSbJ3Au8q318F3F4716qy/y7ga7Wk3FHuLCOiFTOwETxvBt4NbJP0YIl9DFgNHA2MlD6Xe2y/1/Yjkm4FvkXVPL/K9jiApPcBm4F5wFrbj5TzfQTYIOlPgG9QJWfKz89JGqW6o7y4X2WTLCOitUHMlG7763R+dripx3euA67rEN/U6Xu2H6PqLZ8cfw64qE19kywjohVbGRseEdFP1cEzfMMdkywjoqWswRMR0VfVwZPJfyMi+soUbRERfbQcwTNnJFlGRGtZsCwiog8bfrY/yTIioqeqGZ5kGRHR1yBG8Mw2SZYR0cqwvjo0pffS3dbFiIjZrGqGN9nmkin7bfqsixERs9iA1uCZVaayGf7zdTEAJG2gWvfiW1N4zYiYYlVv+PCNDZ/K++Ru62IcQNIVE+tz/Hj3+BRWJyIGYeKl9CbbXDKVybLRGhe219heanvpy04cvn+tImajNMMHq9d6GRExSw1rb/hUJsufr4sB/Ihq2vZ/N4XXi4jDZK71dDcxZcnS9r4e62JExCxli31JloPVbV2MiJjdhrEZPnz/PETEIZl4ZnmoveGSFkm6U9Kjkh6R9P4SP0HSiKTt5efxJS5Jq8sgl4clnVU716pSfntZc3wifrakbeU7q8vyu12v0UuSZUS0NqBXh/YBH7T9GmAZcFUZuHI1sMX2EmBL+QzVAJclZbsCuBGqxAdcA7yJ6v3ua2rJ78ZSduJ7K0q82zW6SrKMiFYG9Z6l7R22Hyj7zwCPUr2LvRJYV4qtAy4s+yuB9a7cA8yXdApwPjBie4/tvcAIsKIcO8723bYNrJ90rk7X6CoTaUREay3eoVwgaWvt8xrbayYXkrQYeANwL3Cy7R1QJVRJJ5Vi3Qa69IqPdYjT4xpdJVlGRCs27Gs++e8u20t7FZB0LPBF4AO2/7E8VuxYtFN1DiJ+UNIMj4jWBjXcUdKRVIny87a/VMJPlSY05efOEu820KVXfGGHeK9rdJVkGRGtDOqZZemZvgl41Pana4c2AhM92quA22vxS0uv+DLg6dKU3gwsl3R86dhZDmwux56RtKxc69JJ5+p0ja7SDI+I1jyY9yzfDLwb2CbpwRL7GHA9cKuky4HHgYvKsU3A24FR4Fngsqou3iPpE1SjBgGutb2n7F8J3AwcA9xRNnpco6sky4hobRCTZNj+Op2fKwKc16G8gau6nGstsLZDfCtwRof47k7X6CXJMiJasYdzBE+SZUS0JMazFG5ERH8DemY5qyRZRkQrmc8yIqIJV88th02SZUS0NteWjGgiyTIiWnE6eCIimkkzPCKigfSGR0T0YSdZRkQ0kleHIiIayDPLiIg+jNif3vCIiP6G8MYyyTIiWkoHT0REQ0N4a5lkGRGt5c4yIqIPA/v3J1lGRPRmIHeWERH9DeN7lsP3slREHDo33PqQtFbSTknfrMXOlHSPpAclbZV0TolL0mpJo5IelnRW7TurJG0v26pa/GxJ28p3VpclcZF0gqSRUn6kLKHbU5JlRLQk7GZbAzcDKybF/hT4Y9tnAv+5fAa4AFhStiuAG6FKfMA1wJuAc4BrasnvxlJ24nsT17oa2GJ7CbClfO4pyTIi2hvQnaXtu4A9k8PAcWX/pcCTZX8lsN6Ve4D5kk4BzgdGbO+xvRcYAVaUY8fZvrsso7seuLB2rnVlf10t3lWeWUZEOwZPbW/4B4DNkv6M6obuN0r8VOCJWrmxEusVH+sQBzjZ9g4A2zskndSvUrmzjIiDoIYbC8pzx4ntigYnvxL4A9uLgD8AbqpddDIfRPyg5M4yItprnnJ22V7a8uyrgPeX/b8C/rLsjwGLauUWUjXRx4BzJ8X/X4kv7FAe4ClJp5S7ylOAnf0qlTvLiGhvQM8su3gS+K2y/1Zge9nfCFxaesWXAU+XpvRmYLmk40vHznJgczn2jKRlpRf8UuD22rkmes1X1eJd5c4yItoZ4Evpkm6huitcIGmMqlf794A/l3QE8BxVbzbAJuDtwCjwLHAZgO09kj4B3F/KXWt7otPoSqoe92OAO8oGcD1wq6TLgceBi/rVNckyIlob1Evpti/pcujsDmUNXNXlPGuBtR3iW4EzOsR3A+e1qWvjZCnpaNvPtzl5RMxRQzg2vO8zS0nnSNpGeW4g6fWS/tuU1ywiZiy52TaXNOngWQ28A9gNYPsh4C1TWamImMGadu7MsWTZpBn+Its/LEMqJ4xPUX0iYsZTZh3q4okykN2S5gG/D3x3aqsVETPaHLtrbKJJsrySqin+q8BTwFdLLCKG1f7prsDh1zdZ2t4JXHwY6hIRs0Em/+1M0l/Q4abbdpMxnhExB821nu4mmjTDv1rb/yXgtzlwho+IGDZJlr/I9hfqnyV9jmq+uIiIoXEwwx1PA35t0BUB+O5jJ/K2Sy6bilPHFPmth++e7ipEC9/6ncH0zKQZ3oGkvfzzTfeLqGY17jsFe0TMUWYohzv2TJZlWqPXAz8qof1lMHtEDLMhzAI9hzuWxPhl2+NlG8I/ooiYLGPDO7uvvuRkRETGhtdIOsL2PuBfAr8n6XvAT6nWtbDtJNCIYTXHEmETvZ5Z3gecRYMlIiNieMzFJnYTvZKlAGx/7zDVJSJmi/SGH+Blkv6w20Hbn56C+kTELJA7ywPNA46l89q7ETHMkiwPsMP2tYetJhExOwzwmaWktVQrMey0fUYt/vvA+4B9wF/b/nCJfxS4nGoC8v9oe3OJrwD+nOom7y9tX1/ipwEbgBOAB4B3235B0tHAeqqF0XYDv2P7B73q2uvVodxRRkRng3t16GZgRT0g6S3ASuB1tl8L/FmJn041XeRry3c+K2lemZT8M8AFwOnAJaUswCeBG2wvAfZSJVrKz722XwncUMr11CtZtlomMiKGh/Y32/qxfRfVEOq6K4HrJ1aTLXPqQpVAN9h+3vb3qdYPP6dso7Yfs/0C1Z3kyjIC8a3AbeX76/jnt3tWls+U4+dp0to5k3VNlrVFyiMiDqdXAb8p6V5JfyPpjSV+KgdODzlWYt3iJwI/Ke+L1+MHnKscf7qU7+pgZh2KiGHX/JnlAklba5/X2F7T5ztHAMcDy4A3ArdK+hd0fjRoOt/0uUd5+hzrWqmIiObadfDssr205RXGgC+VuSjuk7QfWFDii2rlFgJPlv1O8V3A/NpoxHr5iXONSToCeCm/+DjgAE3GhkdEHGhqx4b/b6pnjUh6FXAUVeLbCFws6ejSy72EaqTh/cASSadJOoqqE2hjSbZ3Au8q510F3F72N5bPlONf6zdRUO4sI6K9wb06dAtwLlVzfQy4BlgLrJX0TeAFYFVJZI9IuhX4FtUrRVfZHi/neR+wmerVobW2HymX+AiwQdKfAN8Abirxm4DPSRqluqPsuyhjkmVEtCKa9XQ3YfuSLof+fZfy1wHXdYhvAjZ1iD9G1Vs+Of4ccFGbuiZZRkQ7mUgjIqKhJMuIiAaSLCMi+kszPCKiiSTLiIg+PLje8NkkyTIi2sudZUREf3lmGRHRRJJlREQfc3BN8CaSLCOiFZFmeEREI0mWERFNJFlGRDSQZBkR0UdmHYqIaCjJMiKivwx3jIhoIM3wiIh+8lJ6RERDSZYREb0N6wierBseEa1pvxttfc8jrZW0syx7O/nYhyRZ0oLyWZJWSxqV9LCks2plV0naXrZVtfjZkraV76yWpBI/QdJIKT8i6fh+dU2yjIh23GLr72ZgxeSgpEXA24DHa+ELgCVluwK4sZQ9gWq98TdRLXt7TS353VjKTnxv4lpXA1tsLwG2lM89JVlGRGtys60f23cBezocugH4MAem3JXAelfuAeZLOgU4Hxixvcf2XmAEWFGOHWf7btsG1gMX1s61ruyvq8W7yjPLiGiv+TPLBZK21j6vsb2m1xckvRP4ke2HSqt5wqnAE7XPYyXWKz7WIQ5wsu0dALZ3SDqp3y+SZBkRrbXo4Nlle2nj80ovBj4OLO90uEPMBxE/KGmGR0R7g3tmOdkrgNOAhyT9AFgIPCDp5VR3hotqZRcCT/aJL+wQB3iqNNMpP3f2q1iSZUS0U1Z3bLK1PrW9zfZJthfbXkyV8M6y/Q/ARuDS0iu+DHi6NKU3A8slHV86dpYDm8uxZyQtK73glwK3l0ttBCZ6zVfV4l2lGR4RrQzyPUtJtwDnUj3bHAOusX1Tl+KbgLcDo8CzwGUAtvdI+gRwfyl3re2JTqMrqXrcjwHuKBvA9cCtki6n6nG/qF9dkywjoj0PJlvavqTP8cW1fQNXdSm3FljbIb4VOKNDfDdwXpu6JllGRGvDOIInyTIi2hnSiTSmrIOn1zCmiJjdpqqDZyabyt7wm+kwjCkiZr9hTJZT1gy3fZekxVN1/oiYJmZgHTyzybQ/s5R0BdVAd44++qXTXJuIaGIYO3im/aV022tsL7W99Kgjf3m6qxMRTUzdCJ4Za9rvLCNidhnWyX+TLCOiHTeb2HeumcpXh24B7gZeLWmsDCuKiLkgzfDB6TeMKSJmrzTDIyL6MTCEzfAky4hob/hyZZJlRLSXZnhERAPD2BueZBkR7czBnu4mkiwjopXqpfThy5ZJlhHR3hybUaiJJMuIaC13lhER/QzpM8tpn3UoImabamx4k62fTisqSPqUpG9LeljSlyXNrx37qKRRSd+RdH4tvqLERiVdXYufJuleSdslfUHSUSV+dPk8Wo4v7lfXJMuIaM9utvV3M7+4osIIcIbt1wHfBT4KIOl04GLgteU7n5U0T9I84DPABcDpwCWlLMAngRtsLwH2AhNzVFwO7LX9SuCGUq6nJMuIaMeDW1bC9l3Ankmxr9jeVz7eAyws+yuBDbaft/19qvXDzynbqO3HbL8AbABWShLwVuC28v11wIW1c60r+7cB55XyXSVZRkR7ze8sF0jaWtuuaHml9wB3lP1TgSdqx8ZKrFv8ROAntcQ7ET/gXOX406V8V+ngiYj2mnfw7LK99GAuIenjwD7g8xOhLjXpdNPnHuV7naurJMuIaE37p/ZFS0mrgHcA59k/f/g5BiyqFVsIPFn2O8V3AfMlHVHuHuvlJ841JukI4KVMehwwWZrhEdGOqV5Kb7IdBEkrgI8A77T9bO3QRuDi0pN9GrAEuA+4H1hSer6PouoE2liS7J3Au8r3VwG31861quy/C/haLSl3lDvLiGhFeGAvpZcVFc6lerY5BlxD1ft9NDBS+lzusf1e249IuhX4FlXz/Crb4+U87wM2A/OAtbYfKZf4CLBB0p8A3wBuKvGbgM9JGqW6o7y4X12TLCOivQElyy4rKtzUITZR/jrgug7xTcCmDvHHqHrLJ8efAy5qU9cky4hoL8MdIyL6mHhmOWSSLCOitanuDZ+JkiwjoqXGQxnnlCTLiGjHJFlGRDQyfK3wJMuIaC+T/0ZENJFkGRHRhw3jw9cOT7KMiPZyZxkR0UCSZUREHwYarK8z1yRZRkRLBueZZUREbyYdPBERjeSZZUREA0mWERH9ZCKNiIj+DGSKtoiIBnJnGRHRz3AOd8xSuBHRjsHe32jrR9JaSTslfbMWO0HSiKTt5efxJS5JqyWNSnpY0lm176wq5beXNccn4mdL2la+s1pluchu1+glyTIi2tvvZlt/NwMrJsWuBrbYXgJsKZ8BLqBaK3wJcAVwI1SJj2oJ3TdRreR4TS353VjKTnxvRZ9rdJVkGRHt2c22vqfxXVTrdtetBNaV/XXAhbX4elfuAeZLOgU4Hxixvcf2XmAEWFGOHWf7btsG1k86V6drdJVnlhHRjt2mN3yBpK21z2tsr+nznZNt76gu5R2STirxU4EnauXGSqxXfKxDvNc1ukqyjIj2mveG77K9dEBXVaeaHET8oKQZHhEtGY+PN9oO0lOlCU35ubPEx4BFtXILgSf7xBd2iPe6RldJlhHRzsQUbYPp4OlkIzDRo70KuL0Wv7T0ii8Dni5N6c3AcknHl46d5cDmcuwZSctKL/ilk87V6RpdpRkeEe0NaIo2SbcA51I92xyj6tW+HrhV0uXA48BFpfgm4O3AKPAscBmA7T2SPgHcX8pda3ui0+hKqh73Y4A7ykaPa3SVZBkRrRjwgCb/tX1Jl0PndShr4Kou51kLrO0Q3wqc0SG+u9M1ekmyjIh2nMl/IyIaOYTOm1lLnkED4iX9GPjhdNdjCiwAdk13JaKVufp39mu2X3YoJ5D0f6n+fJrYZXvyCJ1ZaUYly7lK0tYBvmsWh0H+zmKyvDoUEdFAkmVERANJlodHv7GwMfPk7ywOkGR5GDSYOGBKSRqX9KCkb0r6K0kvPoRznSvp/5T9d0rqOrWVpPmS/sNBXOO/SPrQwdZxEKb77yxmniTL4fBPts+0fQbwAvDe+sEyfKz1fwu2N9q+vkeR+UDrZBkxEyVZDp+/BV4pabGkRyV9FngAWCRpuaS7JT1Q7kCPBZC0QtK3JX0d+LcTJ5L0u5L+e9k/WdKXJT1Utt+gGlL2inJX+6lS7o8k3V9muv7j2rk+Luk7kr4KvPqw/WlENJRkOUQkHUE12/S2Eno11WSqbwB+Cvwn4F/bPgvYCvyhpF8C/gL4N8BvAi/vcvrVwN/Yfj1wFvAI1ezT3yt3tX8kaTnVbNXnAGcCZ0v6V5LOBi4G3kCVjN844F894pBlBM9wOEbSg2X/b4GbgF8BflhmnAZYBpwO/F1ZpuQo4G7g14Hv294OIOl/Uk3TP9lbqWZ1wfY48HSHdU2Wl+0b5fOxVMnzJcCXbT9brrHxkH7biCmQZDkc/sn2mfVASYg/rYeopua/ZFK5MzmECVMnEfBfbf+PSdf4wACvETEl0gyPCfcAb5b0SgBJL5b0KuDbwGmSXlHKdZslZgvVdFhImifpOOAZqrvGCZuB99SehZ5apvO/C/htScdIeglVkz9iRkmyDABs/xj4XeAWSQ9TJc9ft/0cVbP7r0sHT7ex++8H3iJpG/D3wGvLNFh/V15Z+pTtrwD/C7i7lLsNeIntB4AvAA8CX6R6VBAxo2RseEREA7mzjIhoIMkyIqKBJMuIiAaSLCMiGkiyjIhoIMkyIqKBJMuIiAb+P5OMZnHYxFWcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "cm = confusion_matrix(y_test, bag_log_reg.predict(X_test))\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "cax = ax.matshow(cm)\n",
    "fig.colorbar(cax)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, '2-class Precision-Recall curve: AP=0.62')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAG2dJREFUeJzt3XuYJHV97/H3x10QlXVRV4wuC6sCUUQUXUFiohiQA0TBox5lvYEh4I2ox0viOSa6YjwaPcZjEkzEYFREEXyMWRUliihoRHc5XBQUsiLICkZAWOQi12/+qBq2HWZqeoat6dnh/Xqeeaa76tfV3/715VP1q67qVBWSJE3mPqMuQJI0txkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgbFZizJYUm+Peo6NrUkFyTZe4o22ye5IcmCWSqrd0kuTbJve3lVkk+NuiYJDIpZl+S+SY5LclmSXyc5J8kBo65rGO0H2c3tB/R/JvnnJFtv6vupqsdV1TenaPOzqtq6qu7Y1Pfffkjf1j7O65L8e5K9NvX93Fsk+XiS25M8Ytz0TdLPSV7cvp9uTPKFJA/uaLsgyV8luWLg/bdNO+/QJGcnuT7J+iTvS7Jw+o94/jEoZt9C4HLgGcBi4C+Bk5IsH2FN0/GcqtoaeBLwFOAvxjdIY3N/bX22fZxLgNOBk0dczyY3Gx+CSR4APB/YALxkgiZj/fxQ4NvA55NkGst/HPAR4GXAw4CbgA933OSdwO8BewEPbG/3m3be/YE30DznewL7AG8etpb5bHN/M292qurGqlpVVZdW1Z1V9SXgp8CTJ7tNkmVJPp/kqiTXJPn7Sdp9KMnl7RrR2Un+YGDeHknWtvP+M8nftNO3SvKpdrnXJVmT5GFDPI6fA18Bdm2X880k707yHZo366OSLG63nq5M8vN2Te6uoaIkRyT5Ubtmd2GSJ7XTB4dgJqt7eZIa+7BL8ogkq5P8Ksm6JEcM3M+qJCcl+WR7XxckWTHVY2wf5+3ACcDSJA8dWOazk5w7sCa828C8CZ+vJI9O8o122tVJThhbm52uJAe39399kp8k2X983w089k+N67PDk/wM+EaSryY5atyyz0vyvPbyY5J8re3Xi5K8cJqlPh+4DjgaOHSyRlV1G/AJ4HeAh0xj+S8BvlhVZ1TVDTQrXs9Lsmh8wyQPogmCI6rqsmr8sKp+09bwD1V1ZlXd2r6+TwCeNo1a5i2DYsTaD+WdgQsmmb8A+BJwGbAcWAqcOMni1gBPBB4MfBo4OclW7bwPAR+qqgcCjwZOaqcfSrNls4zmDfoq4OYh6l4GHAicMzD5ZcCRwKK23k8AtwM7ArsD+wF/0t7+fwCrgJfTrNkdBFwzwV1NVvd4nwHWA48AXgD8nyT7DMw/iKbftgFWAxOG7QSPc8u2xmuAa9tpTwI+BrySps8+AqxOM6zY9XwFeE9b42Np+nzVMHWMq2kP4JPAW9rH83Tg0mks4hnt/f83mtfJyoFl7wLsAHy53Rr4Wttm27bdh9u1+LEhn/OnuK9DaZ6bE4HHjK0MTPCY7gscBqyvqquT/H4bwpP9/X5708cB540tp6p+AtxK854a7/E0r8cXJPlFkouTvLaj9qczyfvyXqeq/BvRH7AF8HXgIx1t9gKuAhZOMO8w4Nsdt70WeEJ7+Qyaze4l49r8MfDvwG5D1HspcAPNGuJlNJv492vnfRM4eqDtw4Bbxua301YCp7eXTwVe33E/+05R93KgaIbylgF3AIsG5r8H+Hh7eRXw9YF5uwA3dzzOVTQfNte1y70G2Htg/j8A7xp3m4toPoAnfb4muJ/nAudM8rhXAZ+a5HYfAT44Vd+NX85Anz1qYP4i4EZgh/b6u4GPtZdfBJw5wX2/Y8jX9/bAncATB57zD03Sz78EvgE8eZrvodOAV42b9vPB52tg+ovbx38ccD9gt/a5etYEbV9Bs+KxZDr1zNc/tyhGJM0Y/vE0b5SjBqZ/Jc3OvRuSvITmQ/CyaoZAplrmm9qhnA1JrqPZUljSzj6cZi3rx+3w0rPb6cfTvIFPTLOD731Jtui4m+dW1TZVtUNVvaaqBrc+Lh+4vANNEF45thZI8yGzbTt/GfCTqR5TR92DHgH8qqp+PTDtMpq1+TG/GLh8E7BVkoVJXjLQ318ZaHNSVW1DE3g/5LeHBncA3jS4hts+nkfQ8Xwl2TbJie0w3PXAp9j4/EzHsH03mbuep7bPvgwc0k46hGbIBZrHuee4x/kSmuGhYbwM+FFVndtePwF48bjX10nt62nbqvrDqjp7mo/lBpot0kEPBH49Qdux1+rRVXVzVZ1Ps6Vz4GCjJM8F3gscUFVXT7Oeeck9+iOQJDRrNQ8DDqxmfBaAqjpgXNu9gO2TLOwKizT7I/6cZgfcBVV1Z5JraYY7qKr/AFa2AfU84HNJHlJVN9Kssb8zzQ71U2jWjo+bwUMbPBXx5TRbFEsmqftymqGk7gVOUve4ZlcAD06yaCAstqdZs5xq+Sew8YNxovlXJ3klsCbJp6vqyrb2d1fVu8e3n+L5eg9NH+1WVde0H0hDDYGN09V3N9LslB0z0Yf6+FNGfwZ4R5IzaNa0Tx+4n29V1bNmUCM0Q3bbJxkL6YU0Q3UH0Az/Tap9PX+lo8kBVXUmzdDQEwZu9yjgvsDFE9xmbJhs0lNmt/t6Pgr8UVX9oKvGexO3KEbjH2jGiJ8zbo18It8HrgTem+QBaXY+T7SDbRHN+OtVwMIkb2dgTSvJS5M8tKrupNnUB7gjyTOTPL4dW78euI1muOUeaT9Q/w34QJIHJrlPmp25z2ib/BPw5iRPTmPHJDuMX85kdY+7r8tphs/e0/bPbjRbIpMGwDQfy49ptrr+rJ30UeBVSfZsa39Akj9qd6B2PV+LaIfukiyl2ccwE8cBr0iyT9uvS5M8pp13LnBIki3S7LB/wRDLO4Vm6+Fomm8h3dlO/xKwc5KXtcvbIslTkjx2qgW2gfloYA+a/WZPpPniw6fp2Kk9ppqdylt3/J3ZNj0BeE6SP2j3qRwNfH7c1uXYMn8CnAm8Lc3+pMfSDK99qa35D9vlPb+qvj9VjfcmBsUsaz8MX0nzxvnFuGGmu6nmOIHn0OwQ/hnNuOmLJmh6Ks0a2MU0wy6/4beHgvYHLkhyA80O4kOq+bbH7wCfowmJHwHfohkS2RReDmwJXEizv+RzwMPbx3UyzXj4p2mGCb5AsxN+vMnqHm8lzRj8FcC/0Iyjf20TPQ6A9wNHJtm2qtYCR9BsDVwLrKPZXzTV8/VOmq8Vb6AZ7vn8TAppP8ReAXywXda3aD7oofnWz6Pbut5J079TLe+WtpZ9B9u3H7b70QxHXUEzfPfXNGvstMN2k+3sPRT416r6QVX9YuyP5jl8djqOdZiOqrqA5gsYJ9Ds51gEvGZsfjuU+78HbrKSpq+uoXkO/rKqTmvn/SXNcO0pkwxH3mulyh8ukiRNzi0KSVIng0KS1MmgkCR1MigkSZ02u+MolixZUsuXLx91GZK0WTn77LOvrqqHTt3y7ja7oFi+fDlr164ddRmStFlJctlMb+vQkySpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnq1FtQJPlYkl8m+eEk85Pkb9P8vvH5meQnEiVJo9XnFsXHaU4RPZkDgJ3avyNpfqNBkjTH9BYUVXUG8KuOJgcDn6zGWcA2SR4+1XJvvXVTVShJGsYo91Es5bd/WGc9v/0bx3dJcmSStUnWXnnltbNSnCSpMcqgyATTJvwVpao6tqpWVNWKxYsf1HNZkqRBowyK9cCygevb0fzcoiRpDhllUKwGXt5+++mpwIaqunKE9UiSJtDb2WOTfAbYG1iSZD3wDmALgKr6R+AU4ECaH6a/iebH4iVJc0xvQVFVK6eYX8Br+7p/SdKm4ZHZkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI69RoUSfZPclGSdUneOsH87ZOcnuScJOcnObDPeiRJ09dbUCRZABwDHADsAqxMssu4Zn8BnFRVuwOHAB/uqx5J0sz0uUWxB7Cuqi6pqluBE4GDx7Up4IHt5cXAFT3WI0magT6DYilw+cD19e20QauAlyZZD5wC/OlEC0pyZJK1SdZu2HBtH7VKkibRZ1Bkgmk17vpK4ONVtR1wIHB8krvVVFXHVtWKqlqxePGDeihVkjSZPoNiPbBs4Pp23H1o6XDgJICq+i6wFbCkx5okSdPUZ1CsAXZK8sgkW9LsrF49rs3PgH0AkjyWJiiu6rEmSdI09RYUVXU7cBRwKvAjmm83XZDk6CQHtc3eBByR5DzgM8BhVTV+eEqSNELZ3D6Xd9xxRa1bt3bUZUjSZiXJ2VW1Yia39chsSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1WjjqAqbrzjvh4otHXYU0fzz4wbDEHyBWh80uKADOOGPUFUjzwy23NEGxcuWoK9FcttkFxcKFsPvuo65Cmh8uuwx+9atRV6G5zn0UkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROQ589NslSYIfB21SVJ/yWpHluqKBI8tfAi4ALgTvayQV0BkWS/YEPAQuAf6qq907Q5oXAqnZ551XVi4ctXpLUv2G3KJ4L/G5V3TLsgpMsAI4BngWsB9YkWV1VFw602Qn4X8DTquraJNsOX7okaTYMu4/iEmCLaS57D2BdVV1SVbcCJwIHj2tzBHBMVV0LUFW/nOZ9SJJ6NuwWxU3AuUlOA+7aqqiq13XcZilw+cD19cCe49rsDJDkOzTDU6uq6qtD1iRJmgXDBsXq9m86MsG0muD+dwL2BrYDzkyya1Vd91sLSo4EjgTYdtvtp1mGJOmeGCooquoTSbak3QIALqqq26a42Xpg2cD17YArJmhzVrusnya5iCY41oy7/2OBYwF23nnF+LCRJPVoqH0USfYG/oNm5/SHgYuTPH2Km60BdkryyDZkDuHuWyVfAJ7Z3scSmiC6ZOjqJUm9G3bo6QPAflV1EUCSnYHPAE+e7AZVdXuSo4BTafY/fKyqLkhyNLC2qla38/ZLMva127dU1TUzfziSpE1t2KDYYiwkAKrq4iRTfguqqk4BThk37e0Dlwt4Y/snSZqDhg2KtUmOA45vr78EOLufkiRJc8mwQfFq4LXA62i+zXQGzb4KSdI8N+y3nm4B/qb9kyTdi3QGRZKTquqFSX7A3Y+BoKp2660ySdKcMNUWxevb/8/uuxBJ0tzUeRxFVV3ZXrwauLyqLgPuCzyBux88J0mah4Y9KeAZwFbtb1KcBrwC+HhfRUmS5o5hgyJVdRPwPODvquq/A7v0V5Ykaa4YOiiS7EVz/MSX22lD/zqeJGnzNWxQvIHmB4b+pT0Nx6OA0/srS5I0Vwx7HMW3gG8NXL+E5uA7SdI8N9VxFP+vqt6Q5ItMfBzFQb1VJkmaE6baohg7t9P/7bsQSdLc1BkUVTV24r+1wM1VdSdAkgU0x1NIkua5Yb+5dBqwL3BDe/1+wL8Bv9dHUZJmzy23wMUXj7oKzWXDBsVWVTUWElTVDUnu31NNkmbJ4sXwi1/AGWeMuhL1b9EDZnrLYYPixiRPqqr/D5DkycDNM71TSXPDNtvAnnuOugrNjgULZnrLYYPiDcDJScbO7/Rw4EUzvVNJ0uZj2OMo1iR5DPC7ND9c9OOquq3XyiRJc8JQR2a3+yP+HHh9Vf0AWJ7EU49L0r3AsKfw+GfgVmCv9vp64K96qUiSNKcMGxSPrqr3AbcBVNXNNENQkqR5btiguDXJ/WhP45Hk0cAtvVUlSZozhv3W0zuArwLLkpwAPA04rK+iJElzx5RBkSTAj2l+tOipNENOr6+qq3uuTZI0B0wZFFVVSb5QVU9m448WSZLuJYbdR3FWkqf0WokkaU4adh/FM4FXJbkUuJFm+Kmqare+CpMkzQ3DBsUBvVYhSZqzpvqFu62AVwE7Aj8Ajquq22ejMEnS3DDVPopPACtoQuIA4AO9VyRJmlOmGnrapaoeD5DkOOD7/ZckSZpLptqiuOsMsQ45SdK901RB8YQk17d/vwZ2G7uc5PqpFp5k/yQXJVmX5K0d7V6QpJKsmO4DkCT1q3Poqapm/ItISRYAxwDPojnb7Jokq6vqwnHtFgGvA7430/uSJPVn2APuZmIPYF1VXVJVtwInAgdP0O5dwPuA3/RYiyRphvoMiqXA5QPX17fT7pJkd2BZVX2pa0FJjkyyNsnaDRuu2vSVSpIm1WdQTPR7FXXXzOQ+wAeBN021oKo6tqpWVNWKxYsfuglLlCRNpc+gWA8sG7i+HXDFwPVFwK7AN9tTgzwVWO0ObUmaW/oMijXATkkemWRL4BBg9djMqtpQVUuqanlVLQfOAg6qqrU91iRJmqbegqI97uIo4FTgR8BJVXVBkqOTHNTX/UqSNq1hTwo4I1V1CnDKuGlvn6Tt3n3WIkmamT6HniRJ84BBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE69BkWS/ZNclGRdkrdOMP+NSS5Mcn6S05Ls0Gc9kqTp6y0okiwAjgEOAHYBVibZZVyzc4AVVbUb8DngfX3VI0mamT63KPYA1lXVJVV1K3AicPBgg6o6vapuaq+eBWzXYz2SpBnoMyiWApcPXF/fTpvM4cBXJpqR5Mgka5Os3bDhqk1YoiRpKn0GRSaYVhM2TF4KrADeP9H8qjq2qlZU1YrFix+6CUuUJE1lYY/LXg8sG7i+HXDF+EZJ9gXeBjyjqm7psR5J0gz0uUWxBtgpySOTbAkcAqwebJBkd+AjwEFV9csea5EkzVBvQVFVtwNHAacCPwJOqqoLkhyd5KC22fuBrYGTk5ybZPUki5MkjUifQ09U1SnAKeOmvX3g8r593r8k6Z7zyGxJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSp16DIsn+SS5Ksi7JWyeYf98kn23nfy/J8j7rkSRNX29BkWQBcAxwALALsDLJLuOaHQ5cW1U7Ah8E/rqveiRJM9PnFsUewLqquqSqbgVOBA4e1+Zg4BPt5c8B+yRJjzVJkqZpYY/LXgpcPnB9PbDnZG2q6vYkG4CHAFcPNkpyJHBke+22FSsedGkvFW92blkM990w6irmBvtiI/tiI/tio+u3m+kt+wyKibYMagZtqKpjgWMBkqytunbFPS9v89f0xU32BfbFIPtiI/tioyRrZ3rbPoee1gPLBq5vB1wxWZskC4HFwK96rEmSNE19BsUaYKckj0yyJXAIsHpcm9XAoe3lFwDfqKq7bVFIkkant6Gndp/DUcCpwALgY1V1QZKjgbVVtRo4Djg+yTqaLYlDhlj0sX3VvBmyLzayLzayLzayLzaacV/EFXhJUhePzJYkdTIoJEmd5mxQePqPjYboizcmuTDJ+UlOS7LDKOqcDVP1xUC7FySpJPP2q5HD9EWSF7avjQuSfHq2a5wtQ7xHtk9yepJz2vfJgaOos29JPpbkl0l+OMn8JPnbtp/OT/KkoRZcVXPuj2bn90+ARwFbAucBu4xr8xrgH9vLhwCfHXXdI+yLZwL3by+/+t7cF227RcAZwFnAilHXPcLXxU7AOcCD2uvbjrruEfbFscCr28u7AJeOuu6e+uLpwJOAH04y/0DgKzTHsD0V+N4wy52rWxSe/mOjKfuiqk6vqpvaq2fRHLMyHw3zugB4F/A+4DezWdwsG6YvjgCOqaprAarql7Nc42wZpi8KeGB7eTF3P6ZrXqiqM+g+Fu1g4JPVOAvYJsnDp1ruXA2KiU7/sXSyNlV1OzB2+o/5Zpi+GHQ4zRrDfDRlXyTZHVhWVV+azcJGYJjXxc7Azkm+k+SsJPvPWnWza5i+WAW8NMl64BTgT2entDlnup8nQL+n8LgnNtnpP+aBoR9nkpcCK4Bn9FrR6HT2RZL70JyF+LDZKmiEhnldLKQZftqbZivzzCS7VtV1Pdc224bpi5XAx6vqA0n2ojl+a9equrP/8uaUGX1uztUtCk//sdEwfUGSfYG3AQdV1S2zVNtsm6ovFgG7At9McinNGOzqebpDe9j3yL9W1W1V9VPgIprgmG+G6YvDgZMAquq7wFbAklmpbm4Z6vNkvLkaFJ7+Y6Mp+6IdbvkITUjM13FomKIvqmpDVS2pquVVtZxmf81BVTXjk6HNYcO8R75A80UHkiyhGYq6ZFarnB3D9MXPgH0AkjyWJiiumtUq54bVwMvbbz89FdhQVVdOdaM5OfRU/Z3+Y7MzZF+8H9gaOLndn/+zqjpoZEX3ZMi+uFcYsi9OBfZLciFwB/CWqrpmdFX3Y8i+eBPw0ST/k2ao5bD5uGKZ5DM0Q41L2v0x7wC2AKiqf6TZP3MgsA64CXjFUMudh30lSdqE5urQkyRpjjAoJEmdDApJUieDQpLUyaCQJHUyKKRxktyR5NwkP0zyxSTbbOLlH5bk79vLq5K8eVMuX9rUDArp7m6uqidW1a40x+i8dtQFSaNkUEjdvsvASdOSvCXJmvZc/u8cmP7ydtp5SY5vpz2n/a2Uc5J8PcnDRlC/dI/NySOzpbkgyQKa0z4c117fj+ZcSXvQnFxtdZKnA9fQnGfraVV1dZIHt4v4NvDUqqokfwL8Gc0RwtJmxaCQ7u5+Sc4FlgNnA19rp+/X/p3TXt+aJjieAHyuqq4GqKqxk1NuB3y2Pd//lsBPZ6V6aRNz6Em6u5ur6onADjQf8GP7KAK8p91/8cSq2rGqjmunT3QunL8D/r6qHg+8kuZEdNJmx6CQJlFVG4DXAW9OsgXNSef+OMnWAEmWJtkWOA14YZKHtNPHhp4WAz9vLx+KtJly6EnqUFXnJDkPOKSqjm9PUf3d9iy9NwAvbc9U+m7gW0nuoBmaOozmV9VOTvJzmlOeP3IUj0G6pzx7rCSpk0NPkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6vRf1NiSzGYsJYwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(y_test, bag_log_reg_pred)\n",
    "step_kwargs = ({'step': 'post'})\n",
    "plt.step(recall, precision, color='b', alpha=0.2,\n",
    "         where='post')\n",
    "plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)\n",
    "\n",
    "from sklearn.metrics import average_precision_score\n",
    "average_precision = average_precision_score(y_test, bag_log_reg_pred)\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision bagging logistic:  0.6949910381555363\n",
      "Recall bagging logistic:  0.6367521619961249\n",
      "F1 score bagging logistic:  0.6645981688708036\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "print('Precision bagging logistic: ', precision_score(y_test, bag_log_reg_pred))\n",
    "print('Recall bagging logistic: ', recall_score(y_test, bag_log_reg_pred))\n",
    "print('F1 score bagging logistic: ', f1_score(y_test, bag_log_reg_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oob_score is  0.674191662544408\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rnd_frst = RandomForestClassifier(n_estimators=200, min_samples_leaf=20, random_state=31, n_jobs = -1, oob_score=True)\n",
    "rnd_frst.fit(X_train, y_train)\n",
    "print('oob_score is ', rnd_frst.oob_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=20, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=200, n_jobs=-1,\n",
       "            oob_score=True, random_state=31, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_frst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6751740207732222"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frst_pred = rnd_frst.predict(X_test)\n",
    "\n",
    "accuracy_score(y_test, frst_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XlcVXX+x/HXFxAXQJDNBUEQQUUxF8Itl9QUM7XMSk3TrHFqpl/NNFPavljTNtXUTJuVZauWW5ia7WqlKZqC4r6xuIAbCsh27/f3x8EiQ7nqvffc5fN8PHzE5Z57z+fE9e2Xc77n81Vaa4QQQngWH7MLEEIIYX8S7kII4YEk3IUQwgNJuAshhAeScBdCCA8k4S6EEB5Iwl0IITyQhLsQQnggCXchhPBAfmbtODw8XMfGxpq1eyGEcEvr1q07rLWOqGs708I9NjaWjIwMs3YvhBBuSSm1z5bt5LSMEEJ4IAl3IYTwQBLuQgjhgSTchRDCA0m4CyGEB6oz3JVSM5VSBUqpTWd5XimlXlZK7VRKZSqlutq/TCGEEOfDlpH7u0DaOZ4fCiRU/5kCvHbxZQkhhIfKXQMrnzf+60B1znPXWq9QSsWeY5ORwHvaWK9vtVIqRCnVXGt9wE41CiGERyjPSsd/wc0obQXf+jAxHaJTHbIve5xzjwJyazzOq/7eHyilpiilMpRSGYWFhXbYtRBCuDirFXZ8xdG3rqXevJvAWgXaCpYK2LvSYbu1xx2qqpbv1brqttZ6BjADICUlRVbmFkJ4ruIC+OUDrBnv4FOUg0U35ku/fgxmFcpaBb7+ENvHYbu3R7jnAdE1HrcE9tvhfYUQwr1oDXt/gIyZsGURWCvZ6JvM25V3Et3rOu4a3AGfg+uMEXtsH4edkgH7hHs6cIdSajbQHSiS8+1CCK9y6hhsnG2E+uHtWOsHoy69FZUymUOHGjMlpAGdWoYY20anOjTUT6sz3JVSHwP9gXClVB7wCFAPQGv9OrAEuBLYCZQCNzuqWCGEcBlaQ/56yHgbNs2DqjJ0y0tZ3+VJ/rKhFX8Lu4SxETGk1dm/0TFsmS0zto7nNfBXu1UkhBCurLwYsj41RukHM6FeAFwyloK245j6I3y3qpAuMSGktGpiapmmtfwVQgi3cnCTEeiZn0DFSWjaEYa9AMnX8dnWkzzw4SYsVs3DVyUxsVcsvj61zTVxHgl3IYQ4m8oyyF5ohHruz8bc9I6jIGUytLwUlBHgwQ3L6BwdwlOjkokObWRy0QYJdyGEONPhnbDuHdjwoXGxNDQeBj8JncdBo1CqLFbeXrGbSouVOwYk0L9tJP0SI1DK3NF6TRLuQggBYKmErYuNUfqe5eDjB+2uMkbpcX1/HaVn7z/B1HmZZOUXMaxTc7TWKKVcKthBwl0I4e2O58K6d+GX96H4EARHw4AHoctNENT0183Kqyz879udvPb9LkIa1ePVG7sytGMzlwv10yTchRDex2qBnV8bo/QdXxrTGhOHGKP0NoPAx/cPL9l7uJTXl+9iROcWPDQsiSYB/iYUbjsJdyGE9zh5yBihr5sFRTkQ2BT6/AO63gQhMX/YvKS8iq+yD3F1lyjaNgvim7v7ExPmGhdM6yLhLoTwbFrDnhXGKH3r50bjrrh+MHg6tBsGvvVqfdnKHYXcNz+L/OOn6BjVmDaRQW4T7CDhLoTwVKVHYePHRqgf2QkNQqD7bdBtEoQnnPVlRaWVPLkkm08y8mgdHsCcKT1pExnkvLrtRMJdCOE5tIa8tUagb5oPlnKI7g5974GkkVCv4TlfbrFqrn39J/YcLuEv/eO5c2ACDer98fy7O5BwF0K4v/KTxp2jGe/AoSzwD4Qu4yHlZmiWXOfLj5ZUENKwHr4+inuGtCUqpCEdo4KdULjjSLgLIdzXwSxY+7bR66Wi2Ajyq/4DyaOhft2nUrTWzF+fz+OfZzM1rR3juscwpEMzJxTueBLuQgj3UnkKNi8wTr3krQW/BtDxWmMaY1S3X282qkvesVLuX7CJFdsL6daqCalxoQ4u3Lkk3IUQ7uHwDiPQN3wEZcchLAGGPAWdx0LD8+vAuOCXPB5csAkNPDaiAxN6tMLH5EZf9ibhLoRwXVUVxvTFjJnG6kU+9aD9cGOUHnuZzaP0M4UG1KdbbCj/uqYjLZu4z/TG8yHhLoRwPcf2/dYSoKTQuMFo4CPGRdLAyPN+u0qLlTdX7qbKorlzYAL9EiPomxDusq0D7EHCXQjhGqwWoxVAxkzY8ZUxKk9MM0bp8QNqbQlgi035RUydl8nm/ScYfkkLl230ZW8S7kIIc508COvfN0bqJ/IgsBn0u9doCRDc8oLftqzSwsvf7OCNFbtp0sif18d3Ja1jc/vV7eIk3IUQzme1Gm11M2bCtiVGS4DWl0PaU9B26FlbApyPfUdKeXPlbkZ1ieLBYUkEN7r493QnEu5CCOcpPWosgJHxDhzdBQ1Docft0O1mCIu/6LcvKa9i2eaDjOrakrbNgvj2H/1dZmUkZ5NwF0I4ltbGEnUZM2HzQqMlQExP6D8N2o+Aeg3sspvl2wu5f34W+4tO0allMG0ig7w22EHCXQjhKGUnIHOOMUov2Az+QcZ59JSboWkHu+3mWEkF0xdnM399PvERAXz6Z/ds9GVvEu5CCPs6sLG6JcBcqCyB5pfA8JeNu0jrB9p1V6cbfe07Usodl7fhjgFt3LbRl71JuAshLl5FKWyeb5x6yV8Hfg0huUZLADs7UlxOk0b++PoopqW1I6pJQzq0cO9GX/Ym4S6EuHCF26pbAnwM5UUQ3haGPgudboCGIXbfndaaT9fl8cTn2Uwd2o4bu7disIc0+rI3CXchxPmpKocti4xz6ft+MFoCJI00Rumtel1wS4C65B4t5f4FWazccZjU2FB6tg5zyH48hYS7EMI2R/dUtwT4AEoPQ5NYGPQYdL4RAiMcuuv56/N4cOEmFDD96o7cmBrjcY2+7E3CXQhxdpYq2LHMOPWy8xtjVN72SmPGS+sB4OPjlDLCA+uTGhfKk9ckExVy7tWUhEHCXQjxRyf2Gy0B1s+CE/kQ1MKYl95lAgRHOXz3lRYrbyzfhcUKdw1KoG9iBH0THfvbgaeRcBdCGKxW2P1ddUuApaAtED/QuECamAa+zomLTflF3DM3ky0HTjCy82+NvsT5semnpZRKA14CfIG3tNZPn/F8DDALCKneZprWeomdaxVCOELJ4d9aAhzbA43CoNcd0G0ShLZ2WhlllRb+8/UO3ly5m9AAf96Y0M1jlrwzQ53hrpTyBV4BrgDygLVKqXStdXaNzR4EPtFav6aUSgKWALEOqFcIYQ9aQ84qY5Se/RlYKqBVbxjwoLEYhl99p5eUc7SUt3/YzeiuLbn/yvZe1+jL3mwZuacCO7XWuwGUUrOBkUDNcNdA4+qvg4H99ixSCGEnZUWwcY4R6oVboH6w0bQr5WaIbO/0ck6WVfLFpoNclxJNYtMgvvtnf49dGcnZbAn3KCC3xuM8oPsZ2zwKfKmU+j8gABhkl+qEEPaRv94I9E3zoLIUWnSFEf+DjqPAP8CUkr7bWsADC7I4eKKMLjEhtIkMkmC3I1vCvbYrGfqMx2OBd7XWzyulegLvK6U6aq2tv3sjpaYAUwBiYmIupF4hhK0qSowwz5gJ+3+Beo0gebRxs1GLLqaVdbSkgumfZ7Pgl3wSIgOZe3svafTlALaEex4QXeNxS/542uUWIA1Aa71KKdUACAcKam6ktZ4BzABISUk58x8IIYQ9FGwxAn3jbCg/ARHt4cp/Q6froYG5/VcsVs3o134i52gpdw5M4K+Xx1PfTxp9OYIt4b4WSFBKxQH5wBhg3Bnb5AADgXeVUu2BBkChPQsVQpxDVTlkpxuhnvMT+PpD0tXGKD2mh8NaAtiq8GQ5YQFGo6/7r2xPVJOGtG/euO4XigtWZ7hrrauUUncAyzCmOc7UWm9WSj0OZGit04F/AG8qpf6OccpmktZaRuZCONqRXUZLgA0fQukRaBIHV0w3WgIEmN97RWvNJxm5PLF4C1PT2jG+RysGJTU1uyyvYNM89+o560vO+N7DNb7OBnrbtzQhRK0sVbB9qTFK3/UtKF9od6UxSo/r77SWAHXJOVLKtPmZ/LTrCN3jQrmsTbjZJXkVuUNVCHdRlA/r3zNaApw8AI2j4PIHjJYAjZubXd3vzF2Xx0MLN+Hro3jymo6MvVQafTmbhLsQrsxqNUbnGTON0brW0GYQDHsBEgY7rSXA+WrauD694sN44pqONA+WRl9mcM1PhhDerrgQNnxgtAQ4vg8ahUPvu4yWAE1iza7uDyqqrLz2/S6sWvP3KxLpkxBBnwRp9GUmCXchXIXWsO/H6pYA6WCthNg+MOgRaDcc/PzNrrBWG3OPc+/cTLYdOsmoLlHS6MtFSLgLYbZTx35rCXB4mzEX/dJbjZYAEW3Nru6sTlVYeOGrbbz9wx4igxrw1k0pMhPGhUi4C2EGrX/fEqDqFESlwMhXocM14O/6t+HnHitl1k/7GJMaw7Sh7WjcQBp9uRIJdyGcqbwYNs01Qv3ARqgXAJfcYExjbH6J2dXV6UR1o6/rqxt9fX9Pf1rIykguScJdCGc4tLm6JcAcqDgJkR1g2POQfD00cI87Nb/deoj752+i4GQZXWOa0CYyUILdhUm4C+EolWVGr/SMmZC7GnzrG6dcUiZDdKrpLQFsdaS4nMc/z+azDftp2zSI1yd0o01koNlliTpIuAthb0d2GYG+4UPjYmloPAx+EjqPg0ahZld3XixWzXWvryL3WCl/H5TI7f3j8fdzjTtgxblJuAthD5ZK2LbECPXd34OPH7QbZozSY/u6TEsAWxWcLCM8oD6+PooHhrWnZZNGtG0mbXndiYS7EBfjeK7RDmD9e1B8CIKjjaXqukyAIPdb/9Nq1Xy8Noenlmxl6tB2TOjRioHtZXqjO5JwF+J8WS2w8xtjlL5jmTGtMWGwMUpPuAJ83LM/+d7DJUybn8nq3UfpFR9GP7nD1K1JuAthq+IC+OV9o8Xu8RwIiITL/m60BAhx75XFPsnI5aGFm/D39eHpUcnccGm03GXq5iTchTgXrWHvSmOUvmURWKsgri9c8Ti0HeayLQHOV1RIQ/omRjB9ZEeaBTcwuxxhBxLuQtSm9Chs/Nho3HVkBzQIgdQ/Gy0BwhPMru6ilVdZePW7XWituXtwW3q3Cae39Fv3KBLuQpymNeRlGKP0zfOhqgxapsLVr0OHq6GeZ9yw80vOMabOy2T7oWKu7dpSGn15KAl3IcpPQtansHYmHMoC/0BjTnrKZGiWbHZ1dlNaUcXzX25n5o97aNa4ATMnpTCgncyE8VQS7sJ7HcwyRumZn0BFMTRNhqtehOTroL7nzenOP3aK91fv48buMUxNa0eQNPryaBLuwrtUnoLNC41Qz1sDfg2gwyhjlN4yxW1aAtiq6FQlS7MOMCY1hoSmQSy/p7+sjOQlJNyFdzi8w7g4uuFDKDsOYQkw5Cm4ZIzbtQSw1ZebD/Lgwk0cKakgJTaUNpGBEuxeRMJdeK6qCti22Bil71lhtARoP7y6JUAfjxuln3a4uJxH0zfzeeYB2jUL4q2JKdLoywtJuAvPczzHuNFo/ftQUgDBMTDwYeg8HoI8+wKixaoZ/dpP7D9exj8HJ/LnfvHU83WvvjbCPiTchWewWmDHV9UtAb40RuUJQ4xRepuBbtsSwFaHTpQREWg0+npkeAdaNmlIQlPPuygsbCfhLtzbyYPGCH39LCjKhcCm0Pef0HUihESbXZ3DWa2aD9fk8MzSrUxNa8uEnrFc3i7S7LKEC5BwF+7HaoW9K4xR+tbFRkuA1v1hyJPQ9krw9Y4pfrsLi5k2P4s1e45yWZtw+reVUBe/kXAX7qP0qDHbJeMdOLoLGjaB7rcZp17C4s2uzqnmrM3h4c82U9/Ph2dHd+K6bi3lLlPxOxLuwrVpDblrqlsCLABLOUT3gH5TIWkk1PPOJlctmzSif1uj0VdkY+/8fyDOTcJduKayE5D1idESoGAz+AdB1wnGKL1pB7Orc7ryKgv//WYnAP8cIo2+RN0k3IVrObCxuiXAp1BZAs06wfCXoONoqO+dc7XX7TvKvXMz2VVYwvUp0uhL2EbCXZivotQ45ZIxE/IzwK8hdLwWLp0MLbp67M1GdSkpr+K5ZduYtWovLYIbMmtyKv0SZXUkYRubwl0plQa8BPgCb2mtn65lm+uBRwENbNRaj7NjncITFW4zLo5u/AjKiiC8LaQ9A5fcYFws9XL7j5/iozU53NSjFfektSOwvozFhO3q/LQopXyBV4ArgDxgrVIqXWudXWObBOA+oLfW+phSSuZkidpVVcDWRUao710JPvUgaYRxLr1Vb68dpZ9WVFrJ4qwDjOtuNPpaee/lNJULpuIC2DIUSAV2aq13AyilZgMjgewa2/wJeEVrfQxAa11g70KFmzu212gJ8MsHUFIIIa1g0KNGS4BAOdUA8MWmgzz02SaOllTQvXUo8RGBEuzigtkS7lFAbo3HeUD3M7ZJBFBK/Yhx6uZRrfUXZ76RUmoKMAUgJsa9FxQWNrBUGa0AMmbCzq+NUXniUGOUHj8AfKTnCUDByTIeTd/MkqyDJDVvzDuTLiU+wjsvHgv7sSXca/s9WdfyPglAf6AlsFIp1VFrffx3L9J6BjADICUl5cz3EJ7ixAFY/57REuBEPgQ1N+ald70JgqPMrs6lWKya619fxf6iMu4Z0pYpfVtLoy9hF7aEex5Qs0lHS2B/Ldus1lpXAnuUUtswwn6tXaoUrs9qhT3fV7cEWALaYozOhz4DiWle0xLAVgeKTtE0qIHR6GtEB6KbNJK2vMKubAn3tUCCUioOyAfGAGfOhFkIjAXeVUqFY5ym2W3PQoWLKjkCGz4wLpAe2wMNQ6HnX6HbJK9rCWALq1Xz3qq9PLtsG9OGtuOmnrFcLj1hhAPUGe5a6yql1B3AMozz6TO11puVUo8DGVrr9OrnBiulsgELcI/W+ogjCxcm0hpyVhuj9OyFYKmAmF5w+QPGzBe/+mZX6JJ2FhQzbV4mGfuO0TcxggHSvVE4kNLanFPfKSkpOiMjw5R9iwtUVgQb5xihXrgF6jc2lqlLmQyR7c2uzqXNXpPDw+mbaVjPl4evSmJU1yi5y1RcEKXUOq11Sl3byV0Rom77fzECPWsuVJZCiy4w4r/GXaT+AWZX5xZiwhoxqH0kj43oSESQ/GYjHE/CXfxR7hrY9S1UlcPu72H/eqMlQPJoY5Qe1dXsCl1eWaWFl7/ZAcC9ae3oFR9Or3hp9CWcR8Jd/N6+VTDrKmMBDDBuNhr6HHS6HhqGmFubm8jYe5R752Wyu7CEMZdGS6MvYQoJd/F73z7+W7ArH2O5uu5TzK3JTRSXV/HcF1t5b/U+okIa8t7kVPpKoy9hEgl38ZusubDvJ1DVi0n7+kNcH3NrciMHi04xe20uE3vGcs+QtgRIoy9hIvn0CcOBjfDZHRDTEwY8BLmrIbYPRKeaXZlLO1ZSwedZB5jQoxVtIo1GX7IyknAFEu4CSg7D7BuNNrvXvweBkRDb2+yqXJrWmqWbDvLwZ5s4XlpJr/gw4iMCJdiFy5Bw93aWSvh0EhQXwOSlRrCLcyo4UcZDn21i2eZDJEcF897k7tLoS7gcCXdv9+VDRl/1q1+HqG5mV+PyLFbNdW+s4mBRGfcNbcctl8XhJ42+hAuScPdmGz6Cn1+DHn+BzmPNrsal7T9+imaNjUZfj4/sSHSThrSW0bpwYTLk8FZ562DR3yCuL1wx3exqXJbFqnnnxz0MfH45H/y8D4B+iRES7MLlycjdG508BHPGQ2BTGP0u+MrHoDY7C05y79xM1uccp3/bCAa2b2p2SULYTP5We5uqCvjkJjh1DG75EgLCzK7IJX30cw6Ppm8moL4vL95wCVd3lkZfwr1IuHubL6Yac9hHz4TmncyuxmXFhjdicIemPDqiA+GB0uhLuB8Jd2+S8Y7R3bH334yOjuJXZZUWXvx6OwrFtKHS6Eu4P7mg6i1yVsOSeyB+IAx82OxqXMrPu48w9KWVvLF8NyfLKjFrjQMh7ElG7t7gxH6YMwGCW8Lot8HH1+yKXMLJskqe+WIrH6zOISa0ER/d2p1ebWS0LjyDhLunqywzZsZUlsLEdKPFgADg0Ily5q7L49bL4rh7cCKN/OWvg/Ac8mn2ZFrD4rshfx3c8IEshQccLalgceZ+JvSMpU1kICvvHSArIwmPJOHuydbMgA0fQr+p0H642dWYSmvN55kHeDR9MyfKKundJpzWEYES7MJjSbh7qj0r4Yv7IHEo9JtmdjWmOnSijAcWbOLrLYfo1DKYD0d3lztMhceTcPdEx3Pg04kQFg+jZoCP906Kslg111c3+nrgyvbc3DtWGn0JryDh7mkqSo3e7JZKGPMRNGhsdkWmyDtWSvPghvj6KKaP7EhMaCNiwwPMLksIp5EhjCfRGhbdCQez4Nq3IDzB7IqczmLVvLVyN4NeWM4Hq41GX30TIyTYhdeRkbsn+em/kPWpsUxe4hCzq3G6bQdPcu+8TDbmHmdgu0gGd5BGX8J7Sbh7ip3fwNePQNJI6PMPs6txug9W7+OxRZsJalCPl8Z0ZsQlLaTRl/BqEu6e4OhumDsZItrDyFfBi0JNa41SijaRgVyZ3JyHr0oiTBp9CSHh7vbKi40LqABjPoT63jHF71SFhRe+2oaPj+K+oe3p0TqMHq2lfbEQp8kFVXemNSy8HQq3wnXvQGic2RU5xapdR0h7aQVvrtxDablFGn0JUQsZubuzlf+GLekw+AmIH2B2NQ53oqySp5Zs5eM1ObQKa8RHf+oubXmFOAubRu5KqTSl1Dal1E6l1Flvd1RKjVZKaaVUiv1KFLXa9gV8+yQkXwc97zC7GqcoOFHOwl/ymdK3NV/c1VeCXYhzqHPkrpTyBV4BrgDygLVKqXStdfYZ2wUBdwI/O6JQUcPhHTD/T9AsGYa/7NEXUI8Ul7No434m9Y6jTWQgP0y9XC6YCmEDW0buqcBOrfVurXUFMBsYWct204FngTI71ifOVFYEH48FX3/jDlT/RmZX5BBaaz7bkM+gF5bz5JIt7C4sBpBgF8JGtoR7FJBb43Fe9fd+pZTqAkRrrT+3Y23iTFYrzP+zMfXx+lkQEm12RQ6x//gpbpmVwV2zN9AqLIDFd/aRRl9CnCdbLqjW9jv/r9MTlFI+wIvApDrfSKkpwBSAmJgY2yoUv/n+Kdi+FIY+B7GXmV2NQ1RZrIyZsZrCk+U8dFUSk3rF4uvjuaedhHAUW8I9D6g5RGwJ7K/xOAjoCHxffUdgMyBdKTVCa51R84201jOAGQApKSkyf+18ZKfDimeh83hI/ZPZ1dhd7tFSWoQ0xM/Xh39dk0xMaCNiwjzzlJMQzmDLaZm1QIJSKk4p5Q+MAdJPP6m1LtJah2utY7XWscBq4A/BLi7CoWxYcBtEdYNhz3vUBdQqi5UZK3Yx6IXlvL9qLwCXJYRLsAtxkeocuWutq5RSdwDLAF9gptZ6s1LqcSBDa51+7ncQF+XUMZg9zrjz9IYPoF4Dsyuymy0HTjB1XiaZeUVckdSUocnNzS5JCI9h001MWuslwJIzvvfwWbbtf/FlCQCsFqNnTFEeTFoMjVuYXZHdvL9qL48tyia4YT3+N64Lw5KbS6MvIexI7lB1Zd88Bru+heEvQUx3s6uxi9ONvhKbBjH8khY8dFUSoQH+ZpclhMeRcHdVWXPhx5cgZTJ0m2R2NRettKKKfy/bjp+v4v4r29O9dRjdpdGXEA4jjcNc0YFM+OwOiOkJac+YXc1F+3HnYYb8ZwUzf9xDRZVVGn0J4QQycnc1JUeMFr4Nm8D174Gf+56yKDpVyb8Wb2FORi5x4QF88ueepMaFml2WEF5Bwt2VWCrh04lQfAgmL4XASLMruiiHi8tZlLmf2/rF87dBCTSo52t2SUJ4DQl3V/LlQ7B3JVz9ujGn3Q0VnjQafU2+LI74iEB+mDpALpgKYQIJd1ex4SP4+TXofjt0Hmt2NedNa83CDfk8tiib0nILl7eLJC48QIJdCJNIuLuC/HWw6G8Q19dYeMPN5B8/xQMLsvh+WyFdY0J4dnQn4sIDzC5LCK8m4W624gKYPR4Cm8Lod8HXvX4kRqOvVRwpruDR4UlM6CmNvoRwBe6VJJ6mqgLmTDBaDNzyJQS4z7zvnCOlRDUxGn09PaoTMaGNiA6VfjBCuAqZ526mL6ZC7moY+T9o3snsamxSZbHy2ve7GPTict5btReA3m3CJdiFcDEycjdLxjuQMRN63wXJo82uxiab9xcxdV4mm/JPMKRDU4ZJoy8hXJaEuxlyfoYl90D8QBj4iNnV2GTWT3uZ/nk2IY38ee3GrtLBUQgXJ+HubCf2wycTILgljH4bfFz7xp7Tjb7aNQtiZOcoHrqqPSGNZHqjEK5Owt2ZKstgzngoL4abPjNaDLiokvIqnlu2jXq+igeGJUmjLyHcjFxQdRatYfHdxpz2UW9AZHuzKzqrFdsLGfziCmat2kulRUujLyHckIzcnWXNm7DhQ+g3FdoPN7uaWhWVVjJ9cTZz1+XROsJo9HVprDT6EsIdSbg7w56V8MU0SBwK/aaZXc1ZHS4pZ2nWAf7SP547B0qjLyHcmYS7ox3PNTo9hsXDqBng41pnwgpOlpG+YT+39mn9a6OvJtIPRgi3J+HuSBWlxuLWlkoY8xE0aGx2Rb/SWjNvfT7TP8/mVKWFge2bEhceIMEuhIeQcHcUrWHRnXAwC8bNgfAEsyv6Ve7RUu5fkMXKHYdJadWEp6+VRl9CeBoJd0dZ9T/I+hQGPASJQ8yu5ldVFitj31zNsZIKpo/swI3dW+Ejjb6E8DgS7o6w61v46mFIGgl9/mF2NQDsPVxCdGgj/Hx9eHa00eirZRPpByOEp3Ktq3ue4Ohu+PRmiGgHI18FZe6ouNJi5ZXvdjL4xRW/NvrqFR8uwS6Eh5ORuz2VFxuLWwOM+RDqB5pazqb8Iu6dm0n2gRMMS27OVZ1amFqPEMJ5JNztRWtYeDsUboXx8yC0tanlvPPjHp5YvIXQAH9eH9+NtI7NTK1HCOHK6rljAAANnElEQVRcEu72svJ52JJuLJMXP8C0Mk43+urQIphRXaJ4cFgSwY3qmVaPEMIcEu72sH0ZfPsEJF8HPe8wpYTi8iqe/WIr/r4+PHhVEqlxoaTGSesAIbyVXFC9WId3wLxboVkyDH/ZlAuo328rYMiLK3h/9T40SKMvIYSM3C9KWRF8PBZ86xkXUP2dOwPlWEkF0xdnM399Pm0iA5l7Wy+6tXLdNsJCCOeRcL9QVivM/7Mx9XFiOoTEOL2EY6UVfLn5EHcOaMNfB7Shvp80+hJCGGw6LaOUSlNKbVNK7VRK/aGtoVLqbqVUtlIqUyn1jVKqlf1LdTHLn4btSyHtaYi9zGm7LThRxowVu9Ba0zoikB+nDuDuwW0l2IUQv1NnuCulfIFXgKFAEjBWKZV0xma/ACla607AXOBZexfqUrYsguXPQOfxkPonp+xSa80na3MZ+MJynv9yO3uPlALITBghRK1sOS2TCuzUWu8GUErNBkYC2ac30Fp/V2P71cB4exbpUgq2wILbIKobDHveKRdQc4+Wct/8LH7YeZjUuFCeHpUsjb6EEOdkS7hHAbk1HucB3c+x/S3A0tqeUEpNAaYAxMQ4/xz1RTt1zLiA6h8AN3wA9Ro4fJenG30dL63kias7Mi41Rhp9CSHqZEu415Yktc61U0qNB1KAfrU9r7WeAcwASElJca/5elYLzL0FivJg0mJo7Nhb+fccLiGmutHXc6MvoVVYI1qENHToPoUQnsOWC6p5QHSNxy2B/WdupJQaBDwAjNBal9unPBfyzeOw6xsY9m+IOdcvLhen0mLlv9/sYMiLK5j1014AesaHSbALIc6LLSP3tUCCUioOyAfGAONqbqCU6gK8AaRprQvsXqXZNs2DH/8DKZOh2ySH7SYz7zj3zs1k68GTDL+kBSM6S6MvIcSFqTPctdZVSqk7gGWALzBTa71ZKfU4kKG1TgeeAwKBT5VxgTFHaz3CgXU7z4FMWPhXiO4Bac84bDczf9jDE4uziQiqz5s3pXBFUlOH7UsI4flsuolJa70EWHLG9x6u8fUgO9flGkqOGC18GzaB698DP/uvL3q60VenlsHccGk004a2J7ihTG8UQlwcuUP1bCxV8OlEKD4Ek5dCkH1H0ifLKnl66Vbq+/ny8PAkUmJDSYmVRl9CCPuQxmFn89VDsHclDH/JmNNuR99tLWDwiyv4eE0Ofr5KGn0JIexORu612fAxrH4Vut8Oncfa7W2PllTw+KLNLNywn8Smgbx6Yy+6xEijLyGE/Um4nyl/HSy6C2L7wODpdn3rolOVfLOlgLsGJvDXy9vg7ye/OAkhHEPCvabiApg9HgKbwnWzjFa+F+lgURkLN+Tz576tiQsP4IdpA+SCqRDC4STcT6uqgE9uMloM3PIlBIRd1NtprZm9Npd/Ld5CpdVKWodmxIYHSLALIZxCwv20L6ZBziq49m1o3umi3mrfkRKmzcti1e4j9GgdytOjOhErjb6EEE4k4Q6w7l3IeBt63wXJoy/qraosVsa9+TNFpyr51zXJjLk0Whp9CSGcTsI952dY/E+IHwgDH7ngt9lVWEyr6kZfz19vNPpqHiz9YIQQ5vDu6Ron9sMnEyC4JYx+G3zOfzWjiior//l6O2n/WcF7q/YB0KN1mAS7EMJU3jtyryyDOeOhvBhu+sxoMXCeNuQeZ+rcTLYdOsnIzi24ukuUAwoVQojz553hrjUs+Ycxp/369yGy/Xm/xds/7OHJxdlEBjXg7YkpDGwvjb6EEK7DO8N9zZvwywfQ915IOr/mlacbfXWODmZMagzThrajcQOZ3iiEcC3eF+57VhrTHhOHQv/7bH7ZibJKnlqylQb1fHhkeAe6tQqlWytp9CWEcE3edUH1eK7R6TEsHkbNAB/bDv/r7ENc8cJy5qzNwd/PRxp9CSFcnveM3CtKYfY4sFTCmI+gQeM6X3KkuJzHFmWTvnE/7ZoFMWNCCpdEhzihWCGEuDjeEe5aG83ADmbBuDkQnmDTy06WVfHdtgL+PiiR2/vHS6MvIYTb8I5wX/U/yPoEBjwIiUPOuen+46dY8Es+f+kfT2x4AD9OGyAXTIUQbsfzw33Xt/DVw5A0Evr886ybWa2aj9bk8PTSrVismmHJzYkND5BgF0K4Jc8O96N74NObIaIdjHwVVO09XvYcLmHavEx+3nOU3m3CeOqaTsSENXJysUIIYT+eG+7lxcbi1gBjPoT6gbVuVmWxMv6tnzlRVsmz13biupSWqLP8IyCEEO7CM8Nda/jsL1C4BW6cC6Gt/7DJzoKTxIYF4Ofrw4s3dKZVWCOaNm5gQrFCCGF/njn9Y+XzkP0ZDHoM2gz83VPlVRZe+Go7af9ZyazqRl+pcaES7EIIj+J5I/fty+DbJyD5Ouj1f797an3OMabOzWRHQTGjukQxShp9CSE8lGeF++EdMO9WaJYMw1/+3QXUN1fs5l9Lt9C8cQPeuflSLm8baWKhQgjhWJ4T7mUnjDtQfesZF1D9jdkuVqvGx0fRtVUIN3aPYWpaO4JkeqMQwsN5RrhbrTB/ChzZZfRmD4mh6FQlTy7OpmE9Xx4b2VEafQkhvIpnXFBd/jRsXwppT0FcH5ZtPsgVLyxn3vp8Aur7SaMvIYTXcf+R+5ZFsPwZ6Dyew0kTeeTD9SzOOkBS88bMnHQpHaOCza5QCCGczr3DvWALLLgNorrBsOcpLrKwckch9wxpy5S+rann6xm/mAghxPmyKf2UUmlKqW1KqZ1KqWm1PF9fKTWn+vmflVKx9i70D04dg9njsPg15J2W09F+9YkND+Cn+wby18vbSLALIbxanQmolPIFXgGGAknAWKVU0hmb3QIc01q3AV4EnrF3ob9jtaDn3orlWA4Ti+/g2Z9Osu9IKQCB9d37lxEhhLAHW4a3qcBOrfVurXUFMBsYecY2I4FZ1V/PBQYqRzVoyV3DqbeuRO36mgcrJqJa9eTLv/clNjzAIbsTQgh3ZMswNwrIrfE4D+h+tm201lVKqSIgDDhsjyJ/lbsG/e6VNLRUUoUPg/pfzoBBqdLoSwghzmDLyL225DxzbqEt26CUmqKUylBKZRQWFtpS3+/tXYmyWADwVYqBDbZLsAshRC1sCfc8ILrG45bA/rNto5TyA4KBo2e+kdZ6htY6RWudEhERcf7VxvYBv/qgfFG+/sZjIYQQf2DLaZm1QIJSKg7IB8YA487YJh2YCKwCRgPfakfcORSdChPTYe9KI9ijU+2+CyGE8AR1hnv1OfQ7gGWALzBTa71ZKfU4kKG1TgfeBt5XSu3EGLGPcVjF0akS6kIIUQeb5g1qrZcAS8743sM1vi4DrrNvaUIIIS6U3OkjhBAeSMJdCCE8kIS7EEJ4IAl3IYTwQBLuQgjhgZRZC1kopQqBfRf48nDs3drA9ckxewc5Zu9wMcfcSmtd512gpoX7xVBKZWitU8yuw5nkmL2DHLN3cMYxy2kZIYTwQBLuQgjhgdw13GeYXYAJ5Ji9gxyzd3D4MbvlOXchhBDn5q4jdyGEEOfg0uHukgtzO5gNx3y3UipbKZWplPpGKdXKjDrtqa5jrrHdaKWUVkq5/cwKW45ZKXV99c96s1LqI2fXaG82fLZjlFLfKaV+qf58X2lGnfailJqplCpQSm06y/NKKfVy9f+PTKVUV7sWoLV2yT8Y7YV3Aa0Bf2AjkHTGNn8BXq/+egwwx+y6nXDMlwONqr++3RuOuXq7IGAFsBpIMbtuJ/ycE4BfgCbVjyPNrtsJxzwDuL366yRgr9l1X+Qx9wW6ApvO8vyVwFKMlex6AD/bc/+uPHJ3rYW5naPOY9Zaf6e1Lq1+uBpjZSx3ZsvPGWA68CxQ5sziHMSWY/4T8IrW+hiA1rrAyTXamy3HrIHG1V8H88cV39yK1noFtaxIV8NI4D1tWA2EKKWa22v/rhzutS3MHXW2bbTWVcDphbndlS3HXNMtGP/yu7M6j1kp1QWI1lp/7szCHMiWn3MikKiU+lEptVoplea06hzDlmN+FBivlMrDWD/i/5xTmmnO9+/7ebFpsQ6T2G1hbjdi8/EopcYDKUA/h1bkeOc8ZqWUD/AiMMlZBTmBLT9nP4xTM/0xfjtbqZTqqLU+7uDaHMWWYx4LvKu1fl4p1RNjdbeOWmur48szhUPzy5VH7nZbmNuN2HLMKKUGAQ8AI7TW5U6qzVHqOuYgoCPwvVJqL8a5yXQ3v6hq62f7M611pdZ6D7ANI+zdlS3HfAvwCYDWehXQAKMHi6ey6e/7hXLlcP91YW6llD/GBdP0M7Y5vTA3OHJhbuep85irT1G8gRHs7n4eFuo4Zq11kdY6XGsdq7WOxbjOMEJrnWFOuXZhy2d7IcbFc5RS4RinaXY7tUr7suWYc4CBAEqp9hjhXujUKp0rHbipetZMD6BIa33Abu9u9hXlOq42Xwlsx7jK/kD19x7H+MsNxg//U2AnsAZobXbNTjjmr4FDwIbqP+lm1+zoYz5j2+9x89kyNv6cFfACkA1kAWPMrtkJx5wE/Igxk2YDMNjsmi/yeD8GDgCVGKP0W4DbgNtq/Ixfqf7/kWXvz7XcoSqEEB7IlU/LCCGEuEAS7kII4YEk3IUQwgNJuAshhAeScBdCCA8k4S6EEB5Iwl0IITyQhLsQQnig/wd4+u3yoeCYnwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "# calculate roc curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, frst_pred)\n",
    "# plot no skill\n",
    "plt.plot([0, 1], [0, 1], linestyle='--')\n",
    "# plot the roc curve for the model\n",
    "plt.plot(fpr, tpr, marker='.')\n",
    "# show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHo5JREFUeJzt3XucjeX+//HXZ2YcO6BMJSYiKkrFSkcRKlRE5VA5lEPttk679m/XPpa9q117d/hp64AkKkInSilCUmImRdgKm8ymTNFBZByu7x/3aI+ZYdaYteZa617v5+Mxj9Za991a79uMt3uu+76v25xziIhIuKT5DiAiIrGnchcRCSGVu4hICKncRURCSOUuIhJCKncRkRBSuYuIhJDKXUQkhFTuIiIhlOHrg2vXru0aNGjg6+NFRJJSTk7ON865zNLW81buDRo0IDs729fHi4gkJTNbG816GpYREQkhlbuISAip3EVEQkjlLiISQip3EZEQKrXczWy0mW00s8/2sdzMbJiZrTSzxWbWIvYxRUSkLKLZcx8DdNzP8k5A44KvwcAT5Y+1H+sWwNyHgv+KiEiJSj3P3Tn3npk12M8qXYGxLrhf33wzq2lmdZxzG2KU8X/WLYBnL4Gd+ZBRFfpNgaxWMf8YEZFkF4sx97rAukLPcwteK8bMBptZtpll5+Xllf2T1swNih0HO7fB59MPJK+ISOjFotythNdKvOu2c26Ecy7inItkZpZ69WxxDVoHe+x7Yn88DjYsLvv7iIiEXCzKPRfIKvS8HrA+Bu9bXFarYCim/R/h0scgLR2evhCWTI7Lx4mIJKtYzC0zBRhiZhOAM4Dv4zLevkdWq/+Nsze5CCb2hZcGwIZPocPdQeGLiKS4UsvdzMYDbYHaZpYL/AWoBOCcexKYBnQGVgJbgWvjFbaYQ46EflPhrd/BB8Pg68/g8qeh+mEVFkFEJBFZcJJLxYtEIi6ms0LmjIE37oAadaHXC3Bks9i9t4hIgjCzHOdcpLT1wnOFasv+cO002PEzjLoAlr7qO5GIiDfhKXcIxuIHz4Yjm8KkfjBzKOze5TuViEiFC1e5AxxaB/q/Aaf1Ca5kfaEnbPvOdyoRkQoVvnIHyKgCXR6Dix+C1bNgZDvY+G/fqUREKkw4yx3ADE4fGJxNs/0HGNUelr/uO5WISIUIb7nvUf9sGDwHajeGF6+GWffB7t2+U4mIxFX4yx2C0yOvfQtOuQrmPAATroKff/CdSkQkblKj3AEqVYXLHodOD8IXbwfDNN984TuViEhcpE65QzAOf8b10Pc12PptcKB1xVu+U4mIxFxqlfsex7YOzoev1QDG94I5/9A4vIiESmqWO0DNY+C66XDylTDrbzCxD2z/0XcqEZGYSN1yB6hcHbqPgAvvhRXTYFQH+HaV71QiIuWW2uUOwTj82UPgmpdhy9cw8nz4YobvVCIi5aJy36PR+cE4fI0seP4KmPsweJoxU0SkvFTuhdVqAAPehmbdYOY9MPlayP/JdyoRkTJTuRdV+SC4YjR0uCeYNnjUBbDpP75TiYiUicq9JGZw7q1wzWT4IRdGtIVV7/pOJSISNZX7/hzXAQbNgkPqwHOXw7xhGocXkaSgci/N4Y1g4DtwwsXwzp/g5UGQv9V3KhGR/VK5R6PKIdBjHLT7IyyZDKMvhO++9J1KRGSfVO7RMoPzfgtXvQib1wbj8P95z3cqEZESqdzLqslFMOhdqH44jL0M5j+hcXgRSTgq9wNRuzEMnBkU/Vt3wqu/gh3bfKcSEfmFyv1AVT0Uej4Pbe+CT8fDM53g+1zfqUREAJV7+aSlQds7odcL8M3KYBx+zTzfqUREVO4xccLFMGgmVDkUxnaBBSM1Di8iXqncYyXz+OBAa6P2MO0OmHIT7NzuO5WIpCiVeyxVqwm9J0DrO2DROHimM/yw3ncqEUlBKvdYS0uD9n+CHmNh4/JgHP7Lj3ynEpEUo3KPl6ZdYeAMqFQdxlwM2c/4TiQiKSSqcjezjma2wsxWmtmdJSyvb2YzzWyxmc02s3qxj5qEjmwKg2fBsefB67fC1FthZ77vVCKSAkotdzNLB4YDnYCmQG8za1pktX8CY51zzYGhwP2xDpq0qtWCqyfBObdCzjPw7KXw49e+U4lIyEWz594KWOmcW+2cywcmAF2LrNMUmFnweFYJy1NbWjpccE9wE5ANn8KINpCb7TuViIRYNOVeF1hX6HluwWuFfQpcXvC4G3CImR1e/nghc9LlwfTB6ZWCK1oXPec7kYiEVDTlbiW8VvQKnTuANma2CGgD/BfYWeyNzAabWbaZZefl5ZU5bCgcdTIMngPHnAWv/RreuAN27fCdSkRCJppyzwWyCj2vB+x18rZzbr1zrrtz7jTgDwWvfV/0jZxzI5xzEedcJDMzsxyxk1z1w+Cal+GsIbBwJIztCltS9B87EYmLaMp9IdDYzI41s8pAL2BK4RXMrLaZ7Xmvu4DRsY0ZQukZcNG90H0k/DcnOB9+/SLfqUQkJEotd+fcTmAIMB1YDkx0zi01s6Fm1qVgtbbACjP7HDgSuDdOecOneQ+4bnrw+OmL4JPxfvOISCiY8zTBVSQScdnZOmPkF1vyYFJ/WPs+nHkjXPDXYO9eRKQQM8txzkVKW09XqCaKgzOh76twxg0w/3F4rhv89K3vVCKSpFTuiSS9EnR6ALo+HsxHM6ItbFjsO5WIJCGVeyI67Wq47k3YvROevhCWTPadSESSjMo9UdVtCdfPgaNPhZcGwNt/hF3FLh0QESmRyj2RHXwE9J0CkQHwwWPw/BWwdZPvVCKSBFTuiS6jMlzyMFw6DNbOg5Hnw9dLfacSkQSnck8WLftB/zdgx88wqgMsfdV3IhFJYCr3ZJLVCgbPhiNPgkn9YMY9sHuX71QikoBU7snm0DrQ/3Vo0Rfefxhe6AnbNvtOJSIJRuWejDKqBGPwFz8Mq2fByHbB/VpFRAqo3JOVGZw+APq9Dtu3BOPwy6f6TiUiCULlnuzqnxWMw9duAi9eA7Pug927facSEc9U7mFQoy5c+yacejXMeQAmXAU/F5tOX0RSiMo9LCpVha7DodM/4Iu3YWR7yPvcdyoR8UTlHiZmcMZg6PsabNsUHGhd8abvVCLigco9jI5tHdyn9fCGML4XzHlQ4/AiKUblHlY1s4I7PDXvCbPuhYl9YPuPvlOJSAVRuYdZpWrQ7Sm46L5geGZUB/h2le9UIlIBVO5hZwZn/Rr6vAxbvoYR58MX7/hOJSJxpnJPFQ3bBufD1zwGnr8S5j4Mnu6fKyLxp3JPJbUawIDp0KwbzLwnuCH39i2+U4lIHKjcU03lg+CK0XDBUFg+JbiN36b/+E4lIjGmck9FZnDOLXD1JPghN7gR96p3facSkRhSuaey4zoE4/CHHg3PXQ7zhmkcXiQkVO6p7rCGMOAdOOESeOdP8NJAyN/qO5WIlJPKXaDKwdBjLLT7E3z2Eoy+EDav9Z1KRMpB5S4BMzjvDrhqImz+MhiHXz3HdyoROUAqd9lbkwth0LtwUCaM6wYfPq5xeJEkpHKX4mofBwNnQJOOMP0ueOUG2LHNdyoRKQOVu5Ss6qHQ8zloexcsngCjO8L3ub5TiUiUVO6yb2lp0PZO6DU+mHDsqTawZp7vVCIShajK3cw6mtkKM1tpZneWsPwYM5tlZovMbLGZdY59VPHmhM7BOHy1mjC2CywYqXF4kQRXarmbWTowHOgENAV6m1nTIqv9EZjonDsN6AU8Huug4llmk6DgG7WHaXfAlCGw42ffqURkH6LZc28FrHTOrXbO5QMTgK5F1nHAoQWPawDrYxdREkbVGtB7Apz3W1j0HIzpDD/oWy2SiKIp97rAukLPcwteK+xu4BozywWmATeV9EZmNtjMss0sOy8v7wDiindpadDuj9BjHGz8dzAO/+V836lEpIhoyt1KeK3ogGtvYIxzrh7QGRhnZsXe2zk3wjkXcc5FMjMzy55WEkfTLsHpkpUPgjGXQPZo34lEpJBoyj0XyCr0vB7Fh10GABMBnHMfAlWB2rEIKAnsyKYweBY0bAOv3wZTb4Gd+b5TiQjRlftCoLGZHWtmlQkOmE4pss6XQHsAMzuRoNw17pIKqtUKpiw49zbIGQPPXgI/fuU7lUjKK7XcnXM7gSHAdGA5wVkxS81sqJl1KVjtdmCQmX0KjAf6O6dz5VJGWjp0uBuueAa+WhLMS7NuoedQIqnNfHVwJBJx2dnZXj5b4uirJTDhqmDv/eKHoUUf34lEQsXMcpxzkdLW0xWqEltHnQyD50D9s4Nz4d+4A3bt8J1KJOWo3CX2qh8GV78EZw2BhSPh2S6wRYdgRCqSyl3iIz0DLroXuo+C9R/DiDbw3499pxJJGSp3ia/mV8J108HSgpklPxnvO5FISlC5S/wdfWpwI+6sVvDqDfDmnRqHF4kzlbtUjINqQ59X4IxfwUdPBHd5+ukb36lEQkvlLhUnvRJ0+jtc9gSsWwAjzocNn/pOJRJKKnepeKdeBde9CW4XPH0RLJnsO5FI6KjcxY+6LYNx+KNPhZcGwPQ/wK6dvlOJhIbKXfw5+AjoOwVOHwgf/guevxy2bvKdSiQUVO7iV0ZluPgh6PIYrP0gmJfmq898pxJJeip3SQwt+kL/abBzOzx9ASx9xXcikaSmcpfEkXU6XD8HjjwJJvWHGXfD7l2+U4kkJZW7JJZDjoL+r0OLfvD+I/BCD9i22XcqkaSjcpfEk1EFugyDSx6B1XOC8+E3LvedSiSpqNwlcUWuC/bi83+CUR1gWdEbgInIvqjcJbEdc2YwDp95PEzsA+/eC7t3+04lkvBU7pL4Dj06OJPm1GvgvQdhQm/4+XvfqUQSmspdkkOlqtD1X9D5n7ByBoxsD3mf+04lkrBU7pI8zKDVIOj7WnAGzch2sOJN36lEEpLKXZJPg3ODeWkObwTje8HsBzQOL1KEyl2SU80suO4taN4TZt8XHGzd/qPvVCIJQ+UuyatSNej2FFx0fzA8M6oDfLvKdyqRhKByl+RmBmfdCH1ehi0bgwuePn/bdyoR71TuEg4N2wbj8DWPCaYsmPsQOOc3k4hHKncJj1r1YcDbcFJ3mDkUJvWD7Vt8pxLxQuUu4VK5Olz+NFzwV1g+NZg+eNNq36lEKpzKXcLHDM65Ga6eDD+sD8bhV870nUqkQqncJbyOaw+DZwXTFzx/BcwbpnF4SRkqdwm3wxrCgHfgxEvhnT/BSwMhf6vvVCJxF1W5m1lHM1thZivN7M4Slj9iZp8UfH1uZt/FPqrIAapyMFz5LLT/M3z2Eoy+EDav9Z1KJK5KLXczSweGA52ApkBvM2taeB3n3G3OuVOdc6cCjwEvxyOsyAEzg9a3w9WTYPOXwY24V8/xnUokbqLZc28FrHTOrXbO5QMTgK77Wb83MD4W4URirvEFwTj8QZkwrht8+LjG4SWUoin3usC6Qs9zC14rxszqA8cC75Y/mkicHN4IBs2E4zvB9LvglRtgxzbfqURiKppytxJe29euTi9gsnOuxFvWm9lgM8s2s+y8vLxoM4rEXpVDoMc4aPt7WDwBRneE79aV/v+JJIloyj0XyCr0vB6wfh/r9mI/QzLOuRHOuYhzLpKZmRl9SpF4SEuDtr+DXuODCcdGtIU17/tOJRIT0ZT7QqCxmR1rZpUJCrzYnYrN7HigFvBhbCOKxNkJnWHQu1CtJoztCh+N0Di8JL1Sy905txMYAkwHlgMTnXNLzWyomXUptGpvYIJz+lshSSizSVDwx3WAN38Lrw2BHT/7TiVywMxXF0ciEZedne3ls0X2afdumH1/cCPuui2h53PBFa4iCcLMcpxzkdLW0xWqIoWlpUG7PwQHW/NWwFNt4Mv5vlOJlJnKXaQkTbvAwBnB1a1jLoHs0b4TiZSJyl1kX444MRiHb9gGXr8Npt4CO7f7TiUSFZW7yP5UqwVXTYRzb4OcMcFe/I9f+U4lUiqVu0hp0tKhw91wxTPw9WfBOPy6hb5TieyXyl0kWid1D6YPzqgCYzrDx2N9JxLZJ5W7SFkcdVJwI+7658CUm+CN22Fnvu9UIsWo3EXKqvphwS38zr4JFo4KrmrdstF3KpG9qNxFDkR6Blz4N+g+CtYvCual+W+O71Qiv1C5i5RH8ythwHSwNBjdCT7RrQwkMajcRcqrzinBOHxWK3j1BpjYH+b8A9Yt8BxMUpnKXSQWDqoNfV6FZt1h2Ssw628w5mL48iPfySRFqdxFYiU9IzibZs/9bXblw+T+sPRV2F3i/WtE4kblLhJLDVpDRlWwdEirBG43TOoHj7WEBSMhf6vvhJIiNOWvSKytWwBr5gZFX7clLJ8KHwwLzqapfjicPghaDQqGckTKKNopf1XuIhXBOVj7QVDyn78FGdXg1KvgrF8HN+wWiVK05Z5REWFEUp4ZNDgn+Nr4b/jwMVg0LphKuGkXOPsWqNfSd0oJEY25i1S0I06ArsPh1iVw7q2wajaMagfPdIYVbwV3gxIpJ5W7iC+HHBXMNvmbpXDRfbB5LYzvCY+fCR+P09zxUi4qdxHfqhwSjL3f8gl0HwnplWHKEHi0Ocx9GLZ95zuhJCGVu0iiSK8EzXvADXOhzyvB8M3Me+CRZvDW7+G7db4TShJRuYskGjNo1A76vgbXvwfHd4KPnoT/fwq8NAi+WuI7oSQBlbtIIqtzClw+KhiyOeN6+Pcb8OS5MPYyWPVucIqlSAlU7iLJoOYx0PH+4OBr+z/DxmUwrhs81RoWT4RdO3wnlASjchdJJtVqQevbg9MouzwWnFHz8iAYdhp8OBy2/+g7oSQIlbtIMsqoAi36wo0fQe8JUCMLpv8+OPg642748SvfCcUzlbtIMktLCw64XvcmDJwJx7aB9x+FR0+G134NeSt8JxRPVO4iYVEvAj3HwU05cFofWDIZhreCF3rCmnk6+JpiVO4iYXN4I7jkYbhtKbS5M5ilckxnGNVec8unEJW7SFgdVBvOvyso+c7/hK3fam75FKJyFwm7ytWD+eNv+hiufBaqHwbT7oBHT4JZ98NP3/hOKHEQVbmbWUczW2FmK83szn2s08PMlpnZUjN7IbYxRaTc0tKh2WXBgdf+06De6TDn7/DISfD6b+DbVb4TSgyVOp+7maUDw4ELgFxgoZlNcc4tK7ROY+Au4Bzn3GYzOyJegUWknDS3fEqIZs+9FbDSObfaOZcPTAC6FllnEDDcObcZwDm3MbYxRSQuNLd8aEVT7nWBwtPR5Ra8VlgToImZzTOz+WbWsaQ3MrPBZpZtZtl5eXkHllhEYk9zy4dONOVuJbxW9ITZDKAx0BboDYwys5rF/ifnRjjnIs65SGZmZlmziki8aW750Iim3HOBrELP6wHrS1jnNefcDufcf4AVBGUvIslIc8snvWjKfSHQ2MyONbPKQC9gSpF1XgXOBzCz2gTDNKtjGVREPNDc8kmr1HJ3zu0EhgDTgeXAROfcUjMbamZdClabDnxrZsuAWcBvnXPfxiu0iHigueWTijlP35BIJOKys7O9fLaIxMC2zcHpkx89BVu+hqNOhrNvhmbdgmEdiQszy3HORUpbT1eoisiB0dzyCU3lLiLlo7nlE5LKXURiQ3PLJxSVu4jE3j7nlu8Faz/QwdcKoHIXkfgpNrf8R/BMJxjVAZa9prnl40jlLiLxV2xu+W9gYl/4VwQWjoId23wnDB2Vu4hUnKJzy1erBW/cHhx8nf13+EmXx8SKyl1EKl5Jc8vPvj8o+Tduh026wL28Sp3PXUQkbkqaW/7jscHFUSdeqrnly0F77iKSGArPLX/OLZpbvpxU7iKSWDS3fEyo3EUkMWlu+XJRuYtIYis2t/yJmls+CjqgKiLJYc/c8o3awYbF8MFjwdzyC56CZt3hnJuDmSkF0J67iCSjOs3h8pFwy6fQ6npYMU1zyxehcheR5FUzCzreB7d9Bu3/AhuXwbhu8FRrWDwRdu3wndAblbuIJL9qtaD1bwrmlv8X7MxP+bnlVe4iEh4ZVaBFH7hxPvR+EWoeU2hu+XtSam55lbuIhE9aGhzfEa6dFkxx0LAtzNszt/yQlJhbXuUuIuFWLwI9xgZzy7foC0smpcTc8ip3EUkNhzWEix9KmbnlVe4iklqKzS3/bSjnlle5i0hq+mVu+ZxQzi2vcheR1BbSueU1/YCICIRubnntuYuIFBWCueVV7iIi+5LEc8ur3EVESpOEc8ur3EVEopVEc8vrgKqISFklwdzyUe25m1lHM1thZivN7M4Slvc3szwz+6Tga2Dso4qIJKAEnVveXCkfbGbpwOfABUAusBDo7ZxbVmid/kDEOTck2g+ORCIuOzv7QDKLiCSubZsh+5lgT37L18Ee/Nk3Q7NuwbBOOZlZjnMuUtp60ey5twJWOudWO+fygQlA1/IGFBEJpdLmll81G+Y+BOsWxDVGNGPudYHCRwlygTNKWO9yMzuPYC//Nudc3I4s9Hzqw2KvXdK8Dn3OasC2/F30f6b4H9oVLetxZSSLTT/l86vncootv+bM+lx6ytGs/24bt734SbHlg1o3pEPTI1mVt4Xfv7yk2PKb2jXm3Ma1Wbr+e4ZOXVZs+f/reDwt6x9GztpNPPhW8elG/3xpU5odXYP3v/iGx979otjy+7qfTKPMg5mx7GtGzi1+xdwjPU/l6JrVmPrpep6bv7bY8ieuaclhB1VmUvY6JufkFls+5tpWVKuczrgP1/D64g3Flr94/VkAjHhvFTOXb9xrWdVK6Tx7XSsAhs38gnkrv9lrea3qlXmyT3DxxwNv/ZuP127ea3mdGlV5tNdpANwzdSnL1v+w1/KGmQdxf/fmANz18mJW5/201/KmRx/KXy5tBsCtExax4fuf91reon4tftfxBABuGJfD5q35ey0/57ja3Ny+MQD9Ri/g5x17TyDV/sQjGHxeI0A/e/rZK+vP3un87sb58MXbfP7KvTSZ/nscYBhkVIV+UyCrVbFtjoVo9tythNeKjuVMBRo455oDM4BnS3wjs8Fmlm1m2Xl5eWVLKiKSjArmln+47qNMr35xwYsOduXDmrlx+9hoxtzPAu52zl1U8PwuAOfc/ftYPx3Y5Jyrsb/31Zi7iKScdQvg2S5BsadXPqA992jH3KMZllkINDazY4H/Ar2Aq4p8WB3n3J7fp7oAy8uUVkQkFWS1Cgp9zVxo0DpuQzIQRbk753aa2RBgOpAOjHbOLTWzoUC2c24KcLOZdQF2ApuA/nFLLCKSzLJaxbXU9yh1WCZeNCwjIlJ2sTwVUkREkozKXUQkhFTuIiIhpHIXEQkhlbuISAh5O1vGzPKA4tcqR6c28E2pa4WLtjk1aJtTQ3m2ub5zLrO0lbyVe3mYWXY0pwKFibY5NWibU0NFbLOGZUREQkjlLiISQsla7iN8B/BA25watM2pIe7bnJRj7iIisn/JuucuIiL7kdDlHsWNuauY2YsFyz8yswYVnzK2otjm35jZMjNbbGYzzay+j5yxVNo2F1rvCjNzZpb0Z1ZEs81m1qPge73UzF6o6IyxFsXP9jFmNsvMFhX8fHf2kTNWzGy0mW00s8/2sdzMbFjBn8diM2sR0wDOuYT8IpheeBXQEKgMfAo0LbLOjcCTBY97AS/6zl0B23w+UL3g8a9SYZsL1jsEeA+YT3Azdu/Z4/x9bgwsAmoVPD/Cd+4K2OYRwK8KHjcF1vjOXc5tPg9oAXy2j+WdgTcJ7nZ3JvBRLD8/kffco7kxd1f+d0u/yUB7MyvptoDJotRtds7Ncs5tLXg6H6hXwRljLdobsP8VeBD4uYRlySaabR4EDHfObQZwzm0kuUWzzQ44tOBxDWB9BeaLOefcewT3t9iXrsBYF5gP1DSzOrH6/EQu95JuzF13X+s453YC3wOHV0i6+IhmmwsbQPAvfzIrdZvN7DQgyzn3ekUGi6Novs9NgCZmNs/M5ptZxwpLFx/RbPPdwDVmlgtMA26qmGjelPXve5lEc5s9X6K5MXc06ySTqLfHzK4BIkCbuCaKv/1us5mlAY8Qrrt7RfN9ziAYmmlL8NvZXDM7yTn3XZyzxUs029wbGOOce6jg3s3jCrZ5d/zjeRHX/krkPfdcIKvQ83oU/zXtl3XMLIPgV7n9/RqU6KLZZsysA/AHoItzbnsFZYuX0rb5EOAkYLaZrSEYm5yS5AdVo/3Zfs05t8M59x9gBUHZJ6totnkAMBHAOfchUJVgDpawiurv+4FK5HL/5cbcZlaZ4IDplCLrTAH6FTy+AnjXFRypSFKlbnPBEMVTBMWe7OOwUMo2O+e+d87Vds41cM41IDjO0MU5l8z3aIzmZ/tVgoPnmFltgmGa1RWaMrai2eYvgfYAZnYiQbnnVWjKijUF6Ftw1syZwPfOuQ0xe3ffR5RLOdrcGfic4Cj7HwpeG0rwlxuCb/4kYCWwAGjoO3MFbPMM4Gvgk4KvKb4zx3ubi6w7myQ/WybK77MBDwPLgCVAL9+ZK2CbmwLzCM6k+QS40Hfmcm7veGADsINgL30AcANwQ6Hv8fCCP48lsf651hWqIiIhlMjDMiIicoBU7iIiIaRyFxEJIZW7iEgIqdxFREJI5S4iEkIqdxGREFK5i4iE0P8BaDVaAJKERoUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, frst_pred)\n",
    "# plot no skill\n",
    "plt.plot([0, 1], [0.5, 0.5], linestyle='--')\n",
    "# plot the precision-recall curve for the model\n",
    "plt.plot(recall, precision, marker='.')\n",
    "# show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, '2-class Precision-Recall curve: AP=0.59')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAG1hJREFUeJzt3X24JGV95vHv7QyIyjioI64OA4MCUUQEmaCuieJKWGAVXOMqBF8wRDQJ0WyMibtJdMQYE13jmoiJJBgVUQQvYyYGQxTRUSNxhgVUQMiIICMY3gd5cXj77R9Vx2mbc+r0OUydPnP4fq6rr9NV9XT1r5/T3XfXU13VqSokSZrKQ8ZdgCRpfjMoJEmdDApJUieDQpLUyaCQJHUyKCRJnQyKbVySY5N8bdx1bG1JLk5y0DRtdk1yW5JFc1RW75JcmeTg9vrqJB8fd02SQTEGSR6a5JQkVyX5cZILkhw27rpG0b6R3dm+Qf9Hkr9LsuPWvp+qempVfXmaNj+oqh2r6t6tff/tm/Td7eO8Jcm/Jnn21r6fB4skH0lyT5InDM3fKv2c5Ffa19PtST6b5NEdbattd1t7+duBZTsl+WiS69rL6pnWshAZFOOxGLgaeB6wFPgj4IwkK8dY00y8qKp2BJ4B/Dzwh8MN0tjWn1+fah/nMuBc4Mwx17PVJVk8B/fxCOCXgU3AMZM0mejnxwJfAz6TJDNY/1OBDwGvBB4H3AF8cJqbPb39kLFjVf3awPz3AQ8HVgIHAq9M8ppRa1motvUX8japqm6vqtVVdWVV3VdVnwO+Dxww1W2SrEjymSTXJ7kxyQemaPf+JFcnuTXJ+Ul+cWDZgUnWt8v+I8mft/N3SPLxdr23JFmX5HEjPI4fAp8H9mnX8+Uk70zydZoX6xOTLG23nq5N8sMkfzw4VJTktUkubbesLknyjHb+4BDMVHWvbD8dLm6nn5BkTZKbkmxI8tqB+1md5IwkH2vv6+Ikq6Z7jO3jvAc4DVie5LED63xhkgsHPgnvO7Bs0v9Xkicl+VI774YkpyXZaZQ6hiU5sr3/W5N8L8mhw3038Ng/PtRnxyX5AfClJP+c5IShdV+U5CXt9Scn+ULbr5cledkMS/1l4BbgRODVUzWqqruBjwL/CXjMDNZ/DPCPVbW2qm6j+eD1kiRLZlgnwIuAd1fVHVV1JXAK8KuzWM+CYlDMA+2b8l7AxVMsXwR8DriK5pPOcuD0KVa3DtgPeDTwCeDMJDu0y94PvL+qHgk8CTijnf9qmi2bFTQv0NcDd45Q9wrgcOCCgdmvBI4HlrT1fhS4B9gD2B84BPi19vb/A1gNvAp4JHAEcOMkdzVV3cM+CWwEngC8FPiTJC8YWH4ETb/tBKwBJg3bSR7n9m2NNwI3t/OeAXwYeB1Nn30IWJNmWLHr/xXgXW2NT6Hp89Wj1DFU04HAx4A3t4/nucCVM1jF89r7/680z5OjB9a9N7Ab8E/t1sAX2jY7t+0+2H6Knxjy+dY09/Vqmv/N6cCTJz4MTPKYHgocC2ysqhuS/EIbwlNdfqG96VOBiybWU1XfA+6ieU1NZW2SH7VhvnK4lKHr+0zz+Ba+qvIyxguwHfBF4EMdbZ4NXA8snmTZscDXOm57M81mNsBa4O3AsqE2vwr8K7DvCPVeCdxG8wnxKppN/Ie1y74MnDjQ9nHA5onl7byjgXPb62cDb+y4n4OnqXslUDRDeSuAe4ElA8vfBXykvb4a+OLAsr2BOzse52qaN5tb2vXeCBw0sPyvgHcM3eYymjfgKf9fk9zPi4ELpnjcq4GPT3G7DwHvm67vhtcz0GdPHFi+BLgd2K2dfifw4fb6y4GvTnLfbxvx+b0rcB+w38D//P1T9PN1wJeAA2b4GjoHeP3QvB8O/r+Glj0X2J4mYD8AfGfifwV8HPhM2yd7AN8DNs+knoV4cYtijNKM4Z9K80I5YWD+57NlR9sxNG+CV1UzBDLdOt/UDuVsSnILzZbCsnbxcTSfsr7bDi+9sJ1/Ks0L+PQk1yR5d5LtOu7mxVW1U1XtVlW/UVWDWx9XD1zfjSYIr534FEjzJrNzu3wFzQtxOlPVPegJwE1V9eOBeVfRfJqf8KOB63cAOyRZnOSYgf7+/ECbM6pqJ5rA+w4/OzS4G/CmwU+47eN5Ah3/ryQ7Jzm9HYa7leaNadlwuxGM2ndT+en/qe2zfwKOamcdRTPUBs3jfObQ4zyGZnhoFK8ELq2qC9vp04BfGXp+ndE+n3auqv9SVefP8LHcRrNFOuiRwI8naUs1Q1R3VdUtwBuB3Wm2rgDeQLM1/e/AP7BlK/VBrfcdWZpcktCMfz4OOLya8VkAquqwobbPBnZNsrgrLNLsj/h94AXAxVV1X5KbaTelq+rfgaPbgHoJ8Okkj6mq22k+sb+93Qw/i+bT8SmzeGiDpyO+mmaLYtkUdV9NM5TUvcIp6h5qdg3w6CRLBsJiV5pPltOt/zS2vDFOtvyGJK8D1iX5RFVd29b+zqp653D7af5f76Lpo32r6sYkL2bEIbAhXX13O80O2QmTvakPnzb6k8DbkqwFHkaz837ifr5SVb80ixqhGbLbNclESC+mGao7jGb4b0rt8/nzHU0Oq6qv0gzZPn3gdk8EHgpcPmKNxZbXyE0M7HBP8ifAN0dcz4LlFsX4/BXNp5gXDX0in8w3gWuBP03yiDQ7n58zSbslNPsDrgcWJ3krA5+0krwiyWOr6j6aTX2Ae5M8P8nT2rH1W4G7aYZbHpD2DfVfgPcmeWSSh7Q7c5/XNvlb4HeTHJDGHkl2G17PVHUP3dfVNMNn72r7Z1+aLZEpA2CGj+W7NFtdv9fO+hvg9Ume2db+iCT/rd2B2vX/WkI7dJdkOc0+htk4BXhNkhe0/bo8yZPbZRcCRyXZLs0O+5eOsL6zaLYeTqT5FtJ97fzPAXsleWW7vu2S/HySp0y5plYbmE+i+fbQfu1lH5r9HVPu1J5QVV+tLd9Mmuzy1bbpacCLkvxiu0/lROAzQ1uXEzU9Ncl+SRal+Vr3e2k+TFzaLn9Skse0yw+j2d/2x9PVutAZFGPQvhm+juaF86OhYab7qeY4gRfRjJn+gGZT+OWTND2b5hPY5TTDLj/hZ4eCDgUuTnIbzQ7io6rqJzSfOD9NExKXAl+hGRLZGl5FMx58Cc3+kk8Dj28f15k04+GfoBkm+CzNTvhhU9U97GiaMfhrgL+nGUf/wlZ6HADvAY5PsnNVrQdeS7M1cDOwgWZ/0XT/r7fTfK14E81wz2dmU0hVfRN4Dc3XOTfR/M8mQvaPaN6gb27v7xMjrG9zW8vBg+3bN9tDaIajrqEZvvszmk/stMN2k34JgyYM/qGqvl1VP5q40PwPX5iOYx1moqoupvkCxmk0+zmWAL8xsbwdyv3f7eTjgE/RPNevoHm+vHBgi/4A4Ns0z8d3Ace0639QS5U/XCRJmppbFJKkTgaFJKmTQSFJ6mRQSJI6bXPHUSxbtqxWrlw57jIkaZty/vnn31BVj52+5f1tc0GxcuVK1q9fP+4yJGmbkuSq2d7WoSdJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1Km3oEjy4STXJfnOFMuT5C/S/LbxtzLFzyNKksarzy2Kj9CcHnoqhwF7tpfjaX6fQZI0z/QWFFW1Fripo8mRwMeqcR6wU5LHT7feu+7aWhVKkkYxzn0Uy/nZH9XZyM/+vvFPJTk+yfok66+99uY5KU6S1BhnUGSSeZP+ilJVnVxVq6pq1dKlj+q5LEnSoHEGxUZgxcD0LjQ/tShJmkfGGRRrgFe13356FrCpqq4dYz2SpEn0dvbYJJ8EDgKWJdkIvA3YDqCq/ho4Czic5kfp76D5oXhJ0jzTW1BU1dHTLC/gN/u6f0nS1uGR2ZKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOvUaFEkOTXJZkg1J3jLJ8l2TnJvkgiTfSnJ4n/VIkmaut6BIsgg4CTgM2Bs4OsneQ83+EDijqvYHjgI+2Fc9kqTZ6XOL4kBgQ1VdUVV3AacDRw61KeCR7fWlwDU91iNJmoU+g2I5cPXA9MZ23qDVwCuSbATOAn5rshUlOT7J+iTrN226uY9aJUlT6DMoMsm8Gpo+GvhIVe0CHA6cmuR+NVXVyVW1qqpWLV36qB5KlSRNpc+g2AisGJjehfsPLR0HnAFQVd8AdgCW9ViTJGmG+gyKdcCeSXZPsj3Nzuo1Q21+ALwAIMlTaILi+h5rkiTNUG9BUVX3ACcAZwOX0ny76eIkJyY5om32JuC1SS4CPgkcW1XDw1OSpDHKtva+vMceq2rDhvXjLkOStilJzq+qVbO5rUdmS5I6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKnT4nEXMFP33QeXXz7uKqR+PfrRsGzZuKuQGttcUACsXTvuCqT+bN7cBMXRR4+7EqmxzQXF4sWw//7jrkLqz1VXwU03jbsKaQv3UUiSOhkUkqROBoUkqZNBIUnqZFBIkjqN/K2nJMuB3QZvU1V+UVWSFriRgiLJnwEvBy4B7m1nF9AZFEkOBd4PLAL+tqr+dJI2LwNWt+u7qKp+ZdTiJUn9G3WL4sXAz1XV5lFXnGQRcBLwS8BGYF2SNVV1yUCbPYH/BTynqm5OsvPopUuS5sKo+yiuALab4boPBDZU1RVVdRdwOnDkUJvXAidV1c0AVXXdDO9DktSzUbco7gAuTHIO8NOtiqp6Q8dtlgNXD0xvBJ451GYvgCRfpxmeWl1V/zxiTZKkOTBqUKxpLzORSebVJPe/J3AQsAvw1ST7VNUtP7Oi5HjgeICdd951hmVIkh6IkYKiqj6aZHvaLQDgsqq6e5qbbQRWDEzvAlwzSZvz2nV9P8llNMGxbuj+TwZOBthrr1XDYSNJ6tFI+yiSHAT8O83O6Q8Clyd57jQ3WwfsmWT3NmSO4v5bJZ8Fnt/exzKaILpi5OolSb0bdejpvcAhVXUZQJK9gE8CB0x1g6q6J8kJwNk0+x8+XFUXJzkRWF9Va9plhySZ+Nrtm6vqxtk/HEnS1jZqUGw3ERIAVXV5kmm/BVVVZwFnDc1768D1An6nvUiS5qFRg2J9klOAU9vpY4Dz+ylJkjSfjBoUvw78JvAGmm8zraXZVyFJWuBG/dbTZuDP24sk6UGkMyiSnFFVL0vybe5/DARVtW9vlUmS5oXptije2P59Yd+FSJLmp87jKKrq2vbqDcDVVXUV8FDg6dz/4DlJ0gI06kkB1wI7tL9JcQ7wGuAjfRUlSZo/Rg2KVNUdwEuAv6yq/w7s3V9ZkqT5YuSgSPJsmuMn/qmdN/Kv40mStl2jBsVv0/zA0N+3p+F4InBuf2VJkuaLUY+j+ArwlYHpK2gOvpMkLXDTHUfxf6vqt5P8I5MfR3FEb5VJkuaF6bYoJs7t9H/6LkSSND91BkVVTZz4bz1wZ1XdB5BkEc3xFJKkBW7Uby6dAxwM3NZOPwz4F+A/91GU9GC3eTNcfvm4q5AaowbFDlU1ERJU1W1JHt5TTdKD2tKl8KMfwdq1465EC8uSR8z2lqMGxe1JnlFV/w8gyQHAnbO9U0lT22kneOYzx12FFp5Fi2Z7y1GD4reBM5NMnN/p8cDLZ3unkqRtx6jHUaxL8mTg52h+uOi7VXV3r5VJkuaFkY7MbvdH/D7wxqr6NrAyiacel6QHgVFP4fF3wF3As9vpjcAf91KRJGleGTUonlRV7wbuBqiqO2mGoCRJC9yoQXFXkofRnsYjyZOAzb1VJUmaN0b91tPbgH8GViQ5DXgOcGxfRUmS5o9pgyJJgO/S/GjRs2iGnN5YVTf0XJskaR6YNiiqqpJ8tqoOYMuPFkmSHiRG3UdxXpKf77USSdK8NOo+iucDr09yJXA7zfBTVdW+fRUmSZofRg2Kw3qtQpI0b033C3c7AK8H9gC+DZxSVffMRWGSpPlhun0UHwVW0YTEYcB7e69IkjSvTDf0tHdVPQ0gySnAN/svSZI0n0y3RfHTM8Q65CRJD07TBcXTk9zaXn4M7DtxPcmt0608yaFJLkuyIclbOtq9NEklWTXTByBJ6lfn0FNVzfoXkZIsAk4CfonmbLPrkqypqkuG2i0B3gD822zvS5LUn1EPuJuNA4ENVXVFVd0FnA4cOUm7dwDvBn7SYy2SpFnqMyiWA1cPTG9s5/1Ukv2BFVX1ua4VJTk+yfok6zdtun7rVypJmlKfQTHZ71XUTxcmDwHeB7xpuhVV1clVtaqqVi1d+titWKIkaTp9BsVGYMXA9C7ANQPTS4B9gC+3pwZ5FrDGHdqSNL/0GRTrgD2T7J5ke+AoYM3EwqraVFXLqmplVa0EzgOOqKr1PdYkSZqh3oKiPe7iBOBs4FLgjKq6OMmJSY7o634lSVvXqCcFnJWqOgs4a2jeW6doe1CftUiSZqfPoSdJ0gJgUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKlTr0GR5NAklyXZkOQtkyz/nSSXJPlWknOS7NZnPZKkmestKJIsAk4CDgP2Bo5OsvdQswuAVVW1L/Bp4N191SNJmp0+tygOBDZU1RVVdRdwOnDkYIOqOreq7mgnzwN26bEeSdIs9BkUy4GrB6Y3tvOmchzw+ckWJDk+yfok6zdtun4rlihJmk6fQZFJ5tWkDZNXAKuA90y2vKpOrqpVVbVq6dLHbsUSJUnTWdzjujcCKwamdwGuGW6U5GDgD4DnVdXmHuuRJM1Cn1sU64A9k+yeZHvgKGDNYIMk+wMfAo6oqut6rEWSNEu9BUVV3QOcAJwNXAqcUVUXJzkxyRFts/cAOwJnJrkwyZopVidJGpM+h56oqrOAs4bmvXXg+sF93r8k6YHzyGxJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSp16DIsmhSS5LsiHJWyZZ/tAkn2qX/1uSlX3WI0maud6CIski4CTgMGBv4Ogkew81Ow64uar2AN4H/Flf9UiSZqfPLYoDgQ1VdUVV3QWcDhw51OZI4KPt9U8DL0iSHmuSJM3Q4h7XvRy4emB6I/DMqdpU1T1JNgGPAW4YbJTkeOD4duruVasedWUvFW9zNi+Fh24adxXzg32xhX2xhX2xxa27zPaWfQbFZFsGNYs2VNXJwMkASdZX3bzqgZe37Wv64g77AvtikH2xhX2xRZL1s71tn0NPG4EVA9O7ANdM1SbJYmApcFOPNUmSZqjPoFgH7Jlk9yTbA0cBa4barAFe3V5/KfClqrrfFoUkaXx6G3pq9zmcAJwNLAI+XFUXJzkRWF9Va4BTgFOTbKDZkjhqhFWf3FfN2yD7Ygv7Ygv7Ygv7YotZ90X8AC9J6uKR2ZKkTgaFJKnTvA0KT/+xxQh98TtJLknyrSTnJNltHHXOhen6YqDdS5NUkgX71chR+iLJy9rnxsVJPjHXNc6VEV4juyY5N8kF7evk8HHU2bckH05yXZLvTLE8Sf6i7advJXnGSCuuqnl3odn5/T3gicD2wEXA3kNtfgP46/b6UcCnxl33GPvi+cDD2+u//mDui7bdEmAtcB6watx1j/F5sSdwAfCodnrncdc9xr44Gfj19vrewJXjrrunvngu8AzgO1MsPxz4PM0xbM8C/m2U9c7XLQpP/7HFtH1RVedW1R3t5Hk0x6wsRKM8LwDeAbwb+MlcFjfHRumL1wInVdXNAFV13RzXOFdG6YsCHtleX8r9j+laEKpqLd3Hoh0JfKwa5wE7JXn8dOudr0Ex2ek/lk/VpqruASZO/7HQjNIXg46j+cSwEE3bF0n2B1ZU1efmsrAxGOV5sRewV5KvJzkvyaFzVt3cGqUvVgOvSLIROAv4rbkpbd6Z6fsJ0O8pPB6IrXb6jwVg5MeZ5BXAKuB5vVY0Pp19keQhNGchPnauChqjUZ4Xi2mGnw6i2cr8apJ9quqWnmuba6P0xdHAR6rqvUmeTXP81j5VdV//5c0rs3rfnK9bFJ7+Y4tR+oIkBwN/ABxRVZvnqLa5Nl1fLAH2Ab6c5EqaMdg1C3SH9qivkX+oqrur6vvAZTTBsdCM0hfHAWcAVNU3gB2AZXNS3fwy0vvJsPkaFJ7+Y4tp+6IdbvkQTUgs1HFomKYvqmpTVS2rqpVVtZJmf80RVTXrk6HNY6O8Rj5L80UHkiyjGYq6Yk6rnBuj9MUPgBcAJHkKTVBcP6dVzg9rgFe13356FrCpqq6d7kbzcuip+jv9xzZnxL54D7AjcGa7P/8HVXXE2IruyYh98aAwYl+cDRyS5BLgXuDNVXXj+Krux4h98Sbgb5L8T5qhlmMX4gfLJJ+kGWpc1u6PeRuwHUBV/TXN/pnDgQ3AHcBrRlrvAuwrSdJWNF+HniRJ84RBIUnqZFBIkjoZFJKkTgaFJKmTQSENSXJvkguTfCfJPybZaSuv/9gkH2ivr07yu1tz/dLWZlBI93dnVe1XVfvQHKPzm+MuSBong0Lq9g0GTpqW5M1J1rXn8n/7wPxXtfMuSnJqO+9F7W+lXJDki0keN4b6pQdsXh6ZLc0HSRbRnPbhlHb6EJpzJR1Ic3K1NUmeC9xIc56t51TVDUke3a7ia8CzqqqS/BrwezRHCEvbFINCur+HJbkQWAmcD3yhnX9Ie7mgnd6RJjieDny6qm4AqKqJk1PuAnyqPd//9sD356R6aStz6Em6vzuraj9gN5o3+Il9FAHe1e6/2K+q9qiqU9r5k50L5y+BD1TV04DX0ZyITtrmGBTSFKpqE/AG4HeTbEdz0rlfTbIjQJLlSXYGzgFeluQx7fyJoaelwA/b669G2kY59CR1qKoLklwEHFVVp7anqP5Ge5be24BXtGcqfSfwlST30gxNHUvzq2pnJvkhzSnPdx/HY5AeKM8eK0nq5NCTJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOv1/Si55+YaHtlgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(y_test, frst_pred)\n",
    "step_kwargs = ({'step': 'post'})\n",
    "plt.step(recall, precision, color='b', alpha=0.2,\n",
    "         where='post')\n",
    "plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)\n",
    "\n",
    "from sklearn.metrics import average_precision_score\n",
    "average_precision = average_precision_score(y_test, frst_pred)\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEGCAYAAADscbcsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGc9JREFUeJzt3X+QVWd9x/H3J5DEaIwQSVIEIqjrD8JUEmhkauNEcZJNxkrsmBY6I2tkSpOSjo62I2pnYmOdxnHUGVrFxoEJ2DQ/KsYwlhSRZkxtIQlJKARJZIOarNkBgTQykpCw++0f59l62Nx7zzlwL3d37+c188w993uec86zbPLd85znnPMoIjAzs8ZOa3cDzMxGAydLM7MSnCzNzEpwsjQzK8HJ0sysBCdLM7MSnCxbSFK3pCcl9Upa3u72WDFJqyXtl/R4u9tiI4uTZYtIGgd8HbgKmAkskjSzva2yEm4DutvdCBt5nCxb51KgNyL2RsRLwJ3Agja3yQpExAPAoXa3w0YeJ8vWmQI8k/vel2JmNgo5WbaOasT8bKnZKOVk2Tp9wLTc96nAs21qi5mdJCfL1nkY6JI0Q9IZwEJgfZvbZGYnyMmyRSLiGHAjsBHYDdwdEbva2yorIukOYAvwNkl9kpa0u002MsivaDMzK+YzSzOzEpwszcxKcLI0MyvBydLMrAQny1NA0tJ2t8Gq8e/MhnOyPDX8P97o49+ZHcfJ0syshBF1n+Wkc8fF9Gmnt7sZTfergwOc9/px7W5GS/x0x6vb3YSWeJmjnM6Z7W5G073Ib3gpjtZ6b0FpV773NXHw0ECpuo/sOLoxIsbEK+/Gt7sBedOnnc5DG6cVV7QR48o3zG53E6yCB2PzSe/j4KEBHtp4Yam64ybvmXTSBxwhRlSyNLORL4BBBtvdjFPOydLMKgmCl6NcN3wscbI0s8p8ZmlmViAIBkbQwPCp4mRpZpUNduBL/50szaySAAacLM3MivnM0sysQAAv+5qlmVljQXRkN9zPhptZNQEDJUsjkqZJul/Sbkm7JH08xT8v6ZeStqdydW6bz0jqlfSkpCtz8e4U65W0PBefIelBSXsk3ZUmD0TSmel7b1o/vejHdrI0s0qyJ3jKlQLHgE9FxDuAecAySTPTuq9FxOxUNgCkdQuBi4Bu4BuSxkkaB3wduAqYCSzK7edLaV9dwHPA0AR0S4DnIuItwNdSvYacLM2sIjFQsjQSEf0R8WhaPkw2C+qUBpssAO6MiKMR8TOgF7g0ld6I2BsRLwF3AgskCXgf8J20/Rrgmty+1qTl7wDzU/26nCzNrJJsgEelCjBJ0rZcqfme0NQNvhh4MIVulLRD0mpJE1NsCvBMbrO+FKsXfz3wv2la6nz8uH2l9c+n+nV5gMfMKsnusyz9lrcDETG3UQVJZwPrgE9ExK8lrQS+kA71BeArwMeg5kGD2id90aA+BetqcrI0s8oG46Reifn/JJ1Olihvj4jvAkTEvtz6bwHfT1/7gPw7HKcCz6blWvEDwARJ49PZY77+0L76JI0HXgccatRWd8PNrJKhM8uTvWaZrhGuAnZHxFdz8cm5ah8CHk/L64GFaSR7BtAFPAQ8DHSlke8zyAaB1kf2ZvP7gQ+n7XuAe3P76knLHwb+IwrehO4zSzOrJBADzTnPejfwEWCnpO0p9lmy0ezZZHn558CfA0TELkl3Az8hG0lfFpG9K07SjcBGYBywOiJ2pf19GrhT0t8Bj5ElZ9LntyX1kp1RLixqrJOlmVXWjG54RPyY2tcONzTY5ovAF2vEN9TaLiL2ko2WD4+/CFxbpb1OlmZWSSBeirE5p1QjTpZmVkl2U3rnDXc4WZpZZRVuHRoznCzNrJIIMRA+szQzKzToM0szs8ayAZ7OSx2d9xOb2UnxAI+ZWUkDTXrccTRxsjSzSpr4BM+o4mRpZpUNejTczKyx7EUaTpZmZg0F4mU/7mhm1lgEvindzKyYfFO6mVmRwGeWZmaleIDHzKxAoKbNwTOaOFmaWSXZVLidlzo67yc2s5NUPBnZWORkaWaVBJ35BE/n/cRmdtKaNBXuNEn3S9otaZekj6f4lyU9IWmHpHskTUjx6ZJekLQ9lW/m9jVH0k5JvZJWpGl2kXSupE2S9qTPiSmuVK83HeeSop/ZydLMKokQg3FaqVLgGPCpiHgHMA9YJmkmsAmYFRG/C/wU+Exum6ciYnYq1+fiK4GlZHOJdwHdKb4c2BwRXcDm9B3gqlzdpWn7hpwszaySbIBnXKnScD8R/RHxaFo+DOwGpkTEDyLiWKq2FZjaaD+SJgPnRMSWiAhgLXBNWr0AWJOW1wyLr43MVmBC2k9dTpZmVlE2B0+ZAkyStC1XltbcozQduBh4cNiqjwH35b7PkPSYpB9JuizFpgB9uTp9KQZwQUT0Q5acgfNz2zxTZ5uaPMBjZpVkAzylR8MPRMTcRhUknQ2sAz4REb/OxT9H1lW/PYX6gQsj4qCkOcD3JF0ENS+ORkG7Km/jZGlmlTXrCR5Jp5Mlytsj4ru5eA/wAWB+6loTEUeBo2n5EUlPAW8lOyvMd9WnAs+m5X2SJkdEf+pm70/xPmBanW1qcjfczCoZeoKnTGkkjVivAnZHxFdz8W7g08AHI+JILn6epHFp+U1kgzN7U/f6sKR5aZ+LgXvTZuuBnrTcMyy+OI2KzwOeH+qu1+MzSzOrrEkTlr0b+AiwU9L2FPsssAI4E9iU7gDamka+3wPcLOkYMABcHxGH0nY3ALcBZ5Fd4xy6znkLcLekJcDTwLUpvgG4GugFjgDXFTXWydLMKomAlwdPPllGxI+pfe1wQ53668i67LXWbQNm1YgfBObXiAewrEp7nSzNrJKsG955V/CcLM2sMj8bbmZWoOKtQ2NGS8+lJXVLejI9f7m8eAszG/ma9rjjqNKynyYN8X+d7BnMmcCi9NynmY1yg2kenqIylrSyG34p0BsRewEk3Un2POZPWnhMM2uxbDS886bCbeV5cqlnLyUtHXpu9FcHB1rYHDNrhmbdlD7atDJZlnr2MiJujYi5ETH3vNd33l8rs9HI3fDmqvzspZmNfJ06Gt7KZPkw0CVpBvBLYCHwpy08npmdImNtpLuMliXLiDgm6UZgIzAOWB0Ru1p1PDM7NSLEMSfL5oqIDdR5ztPMRi93w83MCviapZlZSU6WZmYFhu6z7DROlmZW2Vi7h7IMJ0szqyQCjjXh5b+jjZOlmVXmbriZWQFfszQzKymcLM3MinXiAE/nXaU1s5MSQbPmDZ8m6X5JuyXtkvTxFD9X0iZJe9LnxBSXpBVp5oUdki7J7asn1d8jqScXnyNpZ9pmRZpXvO4xGnGyNLOKxMDgaaVKgWPApyLiHcA8YFmaTWE5sDkiuoDN6Ttksy50pbIUWAlZ4gNuAt5F9tLxm3LJb2WqO7Rdd4rXO0ZdTpZmVlmESpXG+4j+iHg0LR8GdpO9IHwBsCZVWwNck5YXAGsjsxWYIGkycCWwKSIORcRzwCagO607JyK2pHnC1w7bV61j1OVrlmZWScVnwydJ2pb7fmtE3Dq8kqTpwMXAg8AFEdEPWUKVdH6qVm/2hUbxvhpxGhyjLidLM6smsuuWJR2IiLmNKkg6G1gHfCIifp0uK9asWrs1leMnxN1wM6usWdNKSDqdLFHeHhHfTeF9qQtN+tyf4vVmX2gUn1oj3ugYdTlZmlkl0aQBnjQyvQrYHRFfza1aDwyNaPcA9+bii9Oo+Dzg+dSV3ghcIWliGti5AtiY1h2WNC8da/GwfdU6Rl3uhptZZRW64Y28G/gIsFPS9hT7LHALcLekJcDTwLVp3QbgaqAXOAJcl7UlDkn6AtlUNgA3R8ShtHwDcBtwFnBfKjQ4Rl1OlmZWWTOe4ImIH1P7uiLA/Br1A1hWZ1+rgdU14tuAWTXiB2sdoxEnSzOrJMKPO5qZleIXaZiZldCka5ajipOlmVUSiEG//NfMrFgHnlg6WZpZRR7gMTMrqQNPLZ0szawyn1mamRUIYHDQydLMrLEAfGZpZlbM91mamZXhZGlmVqR4yoixyMnSzKrzmaWZWYGA8Gi4mVkZTpZmZsXcDTczK8HJ0sysgG9KNzMrpxNvSi/9Bk9JZ7ayIWY2igyqXCkgabWk/ZIez8XukrQ9lZ8PzfwoabqkF3LrvpnbZo6knZJ6Ja1IU98i6VxJmyTtSZ8TU1ypXq+kHZIuKWprYbKUdKmkncCe9P2dkv6h8F/BzMYsRblSwm1Adz4QEX8SEbMjYjawDvhubvVTQ+si4vpcfCWwFOhKZWify4HNEdEFbE7fAa7K1V2atm+ozJnlCuADwMH0g/wP8N4S25nZWBQVStGuIh4ADtVal84O/xi4o9E+JE0GzomILWm63LXANWn1AmBNWl4zLL42MluBCWk/dZVJlqdFxC+GxQZKbGdmY5KyAZ4yBSZJ2pYrSysc6DJgX0TsycVmSHpM0o8kXZZiU4C+XJ2+FAO4ICL6AdLn+bltnqmzTU1lBniekXQpEJLGAX8J/LTEdmY2VpUf4DkQEXNP8CiLOP6ssh+4MCIOSpoDfE/SRdS+Q76ohZW3KZMsbyDril8I7AN+mGJm1qkGW7t7SeOBPwLmDMUi4ihwNC0/Iukp4K1kZ4VTc5tPBZ5Ny/skTY6I/tTN3p/ifcC0OtvUVNgNj4j9EbEwIialsjAiDhRtZ2Zj1NB9luW64Sfq/cATEfH/3WtJ56XeLZLeRDY4szd1rw9Lmpeucy4G7k2brQd60nLPsPjiNCo+D3h+qLteT+GZpaRvUeP0NCKqXHswszGk5Eh38X6kO4DLya5t9gE3RcQqYCGvHNh5D3CzpGNk4ybXR8TQ4NANZCPrZwH3pQJwC3C3pCXA08C1Kb4BuBroBY4A1xW1tUw3/Ie55VcBH+L4C6Nm1mmalCwjYlGd+EdrxNaR3UpUq/42YFaN+EFgfo14AMuqtLUwWUbEXfnvkr4NbKpyEDOz0e5EHnecAbyx2Q0B2LX/PGat+ItW7Npa5MV/OdLuJlgFRz/7303ZT7O64aNJmWuWz/Hbk+7TyG4gXV5/CzMb04JSjzKONQ2TZRpZeifwyxQaTH19M+tkHZgFGt46lBLjPRExkEoH/hOZ2XBNfDZ81CjzuONDZd7IYWYdpEnPho8mdbvhksZHxDHgD4A/S3fL/4bsMaGICCdQs041xhJhGY2uWT4EXMJv39JhZjYmu9hlNEqWAoiIp05RW8xstPBo+HHOk/TJeisj4qstaI+ZjQI+szzeOOBsOnGCYDNrzMnyOP0RcfMpa4mZjQ6+ZvkKPqM0s9qcLI/zijd1mJkBqMUv/x2J6t6UnntPnJlZxzuRtw6ZWadzN9zMrIAHeMzMSnKyNDMrwcnSzKwx0Zmj4U6WZlZNh16zLPM+SzOz4zXpfZaSVkvaL+nxXOzzkn4paXsqV+fWfUZSr6QnJV2Zi3enWK+k5bn4DEkPStoj6S5JZ6T4mel7b1o/vaitTpZmVl3zXv57G9BdI/61iJidygYASTPJ5hO/KG3zDUnjJI0Dvg5cBcwEFqW6AF9K++oCngOWpPgS4LmIeAvwtVSvISdLM6usWdNKRMQDZJMglrEAuDMijkbEz4Be4NJUeiNib0S8BNwJLEhziL0P+E7afg2/fT/vgvSdtH5+ql+Xk6WZVVf+zHKSpG25srTkEW6UtCN10yem2BTgmVydvhSrF3898L9pxod8/Lh9pfXPp/p1OVmaWTWRjYaXKcCBiJibK7eWOMJK4M3AbKAf+EqK1zrzixOIN9pXXU6WZlZdCycsi4h9aTbZQeBbZN1syM4Mp+WqTgWebRA/AEyQNH5Y/Lh9pfWvo+BygJOlmVXWyqlwJU3Off0QMDRSvh5YmEayZwBdZHOFPQx0pZHvM8gGgdanqbvvBz6ctu8B7s3tqyctfxj4j6Kpvn2fpZlV16T7LCXdAVxOdm2zD7gJuFzS7HSUnwN/DhARuyTdDfwEOAYsi4iBtJ8bgY1kMzysjohd6RCfBu6U9HfAY8CqFF8FfFtSL9kZ5cKitjpZmlk1TZwTPCIW1QivqhEbqv9F4Is14huADTXie/ltNz4ffxG4tkpbnSzNrBLRmU/wOFmaWWVOlmZmZThZmpmV4GRpZlagQ9865GRpZtU5WZqZFfPLf83MSnA33MysSBNvSh9NnCzNrDonSzOzxvwEj5lZSRrsvGzpZGlm1fiapZlZOe6Gm5mV4WRpZlbMZ5ZmZmU4WZqZFQg/7mhmVsj3WZqZldV4IsQxyVPhmlllzZoKV9JqSfslPZ6LfVnSE5J2SLpH0oQUny7pBUnbU/lmbps5knZK6pW0QpJS/FxJmyTtSZ8TU1ypXm86ziVFbXWyNLNqokIpdhvQPSy2CZgVEb8L/BT4TG7dUxExO5Xrc/GVwFKyucS7cvtcDmyOiC5gc/oOcFWu7tK0fUMtS5a1/mKY2digwXKlSEQ8QDZvdz72g4g4lr5uBaY2bIs0GTgnIrZERABrgWvS6gXAmrS8Zlh8bWS2AhPSfupq5ZnlbbzyL4aZjQEVkuUkSdtyZWnFQ30MuC/3fYakxyT9SNJlKTYF6MvV6UsxgAsioh8gfZ6f2+aZOtvU1LIBnoh4QNL0Vu3fzNokqDLAcyAi5p7IYSR9DjgG3J5C/cCFEXFQ0hzge5IuIhugr9XKhruvuk3bR8PTX5qlAOPPmdjm1phZGa2+dUhSD/ABYH7qWhMRR4GjafkRSU8BbyU7K8x31acCz6blfZImR0R/6mbvT/E+YFqdbWpq+wBPRNwaEXMjYu74V7+m3c0xszKaN8DzCpK6gU8DH4yII7n4eZLGpeU3kQ3O7E3d68OS5qVR8MXAvWmz9UBPWu4ZFl+cRsXnAc8PddfrafuZpZmNLs28KV3SHcDlZNc2+4CbyEa/zwQ2pTuAtqaR7/cAN0s6BgwA10fE0ODQDWTjJGeRXeMcus55C3C3pCXA08C1Kb4BuBroBY4A1xW11cnSzKqJaNrLfyNiUY3wqjp11wHr6qzbBsyqET8IzK8RD2BZlba28tahO4AtwNsk9aXMbmZjQQu74SNVK0fDa/3FMLMxwM+Gm5kVCcBz8JiZldB5udLJ0syqczfczKwET4VrZlZkDI50l+FkaWaVZDeld162dLI0s+o8B4+ZWTGfWZqZFfE1SzOzMpr3bPho4mRpZtW5G25mViDKza8z1jhZmll1PrM0Myuh83Klk6WZVafBzuuHO1maWTWBb0o3Mysiwjelm5mV4mRpZlZCBybLts8bbmajzNA1yzKlgKTVkvZLejwXO1fSJkl70ufEFJekFZJ6Je2QdElum55Uf4+knlx8jqSdaZsVaV7xusdoxMnSzCrT4GCpUsJtQPew2HJgc0R0AZvTd4CrgK5UlgIrIUt8ZPONvwu4FLgpl/xWprpD23UXHKMuJ0szqyiybniZUrSniAeAQ8PCC4A1aXkNcE0uvjYyW4EJkiYDVwKbIuJQRDwHbAK607pzImJLmid87bB91TpGXb5maWbVBFWuWU6StC33/daIuLVgmwsioh8gIvolnZ/iU4BncvX6UqxRvK9GvNEx6nKyNLPqyt9neSAi5jbpqKoRixOInxB3w82sMkWUKidoX+pCkz73p3gfMC1XbyrwbEF8ao14o2PU5WRpZtU16ZplHeuBoRHtHuDeXHxxGhWfBzyfutIbgSskTUwDO1cAG9O6w5LmpVHwxcP2VesYdbkbbmbVRMBAc553lHQHcDnZtc0+slHtW4C7JS0BngauTdU3AFcDvcAR4LqsOXFI0heAh1O9myNiaNDoBrIR97OA+1KhwTHqcrI0s+qadFN6RCyqs2p+jboBLKuzn9XA6hrxbcCsGvGDtY7RiJOlmVXXgU/wOFmaWTUBeA4eM7MiAdF572hzsjSzaoKmDfCMJk6WZladr1mamZXgZGlmVuSkbjgftZwszayaADxhmZlZCT6zNDMr0rzHHUcTJ0szqyYgfJ+lmVkJfoLHzKwEX7M0MysQ4dFwM7NSfGZpZlYkiIGBdjfilHOyNLNq/Io2M7OSfOuQmVljAYTPLM3MCoRf/mtmVkonDvAoRtAtAJJ+Bfyi3e1ogUnAgXY3wioZq7+zN0bEeSezA0n/TvbvU8aBiOg+meONFCMqWY5VkrZFxNx2t8PK8+/Mhjut3Q0wMxsNnCzNzEpwsjw1bm13A6wy/87sOE6Wp0BEtPV/PEkDkrZLelzSv0p69Uns63JJ30/LH5S0vEHdCZL+4gSO8XlJf3WibWyGdv/ObORxsuwML0TE7IiYBbwEXJ9fqUzl/xYiYn1E3NKgygSgcrI0G4mcLDvPfwJvkTRd0m5J3wAeBaZJukLSFkmPpjPQswEkdUt6QtKPgT8a2pGkj0r6x7R8gaR7JP1PKr8P3AK8OZ3VfjnV+2tJD0vaIelvc/v6nKQnJf0QeNsp+9cwK8nJsoNIGg9cBexMobcBayPiYuA3wN8A74+IS4BtwCclvQr4FvCHwGXA79TZ/QrgRxHxTuASYBewHHgqndX+taQrgC7gUmA2MEfSeyTNARYCF5Ml499r8o9udtL8BE9nOEvS9rT8n8Aq4A3ALyJia4rPA2YC/yUJ4AxgC/B24GcRsQdA0j8DS2sc433AYoCIGACelzRxWJ0rUnksfT+bLHm+FrgnIo6kY6w/qZ/WrAWcLDvDCxExOx9ICfE3+RCwKSIWDas3m+zdCc0g4O8j4p+GHeMTTTyGWUu4G25DtgLvlvQWAEmvlvRW4AlghqQ3p3qL6my/GbghbTtO0jnAYbKzxiEbgY/lroVOkXQ+8ADwIUlnSXotWZffbERxsjQAIuJXwEeBOyTtIEueb4+IF8m63f+WBnjqPbv/ceC9knYCjwAXRcRBsm7945K+HBE/AP4F2JLqfQd4bUQ8CtwFbAfWkV0qMBtR/Gy4mVkJPrM0MyvBydLMrAQnSzOzEpwszcxKcLI0MyvBydLMrAQnSzOzEv4PE3En5g6FhI8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test, rnd_frst.predict(X_test))\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "cax = ax.matshow(cm)\n",
    "fig.colorbar(cax)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision random forest:  0.7080958731049972\n",
      "Recall random forest:  0.5959636831907755\n",
      "F1 score random forest:  0.647208821944523\n"
     ]
    }
   ],
   "source": [
    "print('Precision random forest: ', precision_score(y_test, frst_pred))\n",
    "print('Recall random forest: ', recall_score(y_test, frst_pred))\n",
    "print('F1 score random forest: ', f1_score(y_test, frst_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-error:0.441334\tvalidation_0-logloss:0.690516\tvalidation_1-error:0.443683\tvalidation_1-logloss:0.688035\n",
      "[1]\tvalidation_0-error:0.403217\tvalidation_0-logloss:0.682476\tvalidation_1-error:0.407721\tvalidation_1-logloss:0.686971\n",
      "[2]\tvalidation_0-error:0.396783\tvalidation_0-logloss:0.680276\tvalidation_1-error:0.400518\tvalidation_1-logloss:0.678433\n",
      "[3]\tvalidation_0-error:0.393629\tvalidation_0-logloss:0.677416\tvalidation_1-error:0.397562\tvalidation_1-logloss:0.675579\n",
      "[4]\tvalidation_0-error:0.39651\tvalidation_0-logloss:0.672791\tvalidation_1-error:0.398851\tvalidation_1-logloss:0.672321\n",
      "[5]\tvalidation_0-error:0.392067\tvalidation_0-logloss:0.669742\tvalidation_1-error:0.394994\tvalidation_1-logloss:0.674406\n",
      "[6]\tvalidation_0-error:0.387064\tvalidation_0-logloss:0.666391\tvalidation_1-error:0.391425\tvalidation_1-logloss:0.671445\n",
      "[7]\tvalidation_0-error:0.384323\tvalidation_0-logloss:0.66534\tvalidation_1-error:0.388949\tvalidation_1-logloss:0.670383\n",
      "[8]\tvalidation_0-error:0.383256\tvalidation_0-logloss:0.664286\tvalidation_1-error:0.388771\tvalidation_1-logloss:0.663249\n",
      "[9]\tvalidation_0-error:0.384314\tvalidation_0-logloss:0.662491\tvalidation_1-error:0.388813\tvalidation_1-logloss:0.661604\n",
      "[10]\tvalidation_0-error:0.380789\tvalidation_0-logloss:0.660137\tvalidation_1-error:0.385803\tvalidation_1-logloss:0.659535\n",
      "[11]\tvalidation_0-error:0.379263\tvalidation_0-logloss:0.658409\tvalidation_1-error:0.384435\tvalidation_1-logloss:0.658349\n",
      "[12]\tvalidation_0-error:0.379389\tvalidation_0-logloss:0.657038\tvalidation_1-error:0.384863\tvalidation_1-logloss:0.661324\n",
      "[13]\tvalidation_0-error:0.378924\tvalidation_0-logloss:0.655578\tvalidation_1-error:0.384718\tvalidation_1-logloss:0.660234\n",
      "[14]\tvalidation_0-error:0.376247\tvalidation_0-logloss:0.653769\tvalidation_1-error:0.382276\tvalidation_1-logloss:0.658739\n",
      "[15]\tvalidation_0-error:0.3761\tvalidation_0-logloss:0.653313\tvalidation_1-error:0.382756\tvalidation_1-logloss:0.658135\n",
      "[16]\tvalidation_0-error:0.377695\tvalidation_0-logloss:0.651564\tvalidation_1-error:0.382917\tvalidation_1-logloss:0.657009\n",
      "[17]\tvalidation_0-error:0.377647\tvalidation_0-logloss:0.650606\tvalidation_1-error:0.38284\tvalidation_1-logloss:0.655939\n",
      "[18]\tvalidation_0-error:0.375162\tvalidation_0-logloss:0.650534\tvalidation_1-error:0.381258\tvalidation_1-logloss:0.650618\n",
      "[19]\tvalidation_0-error:0.374991\tvalidation_0-logloss:0.64966\tvalidation_1-error:0.3811\tvalidation_1-logloss:0.650059\n",
      "[20]\tvalidation_0-error:0.371894\tvalidation_0-logloss:0.6488\tvalidation_1-error:0.378388\tvalidation_1-logloss:0.649476\n",
      "[21]\tvalidation_0-error:0.371258\tvalidation_0-logloss:0.648802\tvalidation_1-error:0.378109\tvalidation_1-logloss:0.649794\n",
      "[22]\tvalidation_0-error:0.371162\tvalidation_0-logloss:0.648663\tvalidation_1-error:0.377903\tvalidation_1-logloss:0.649428\n",
      "[23]\tvalidation_0-error:0.370919\tvalidation_0-logloss:0.645575\tvalidation_1-error:0.377757\tvalidation_1-logloss:0.647113\n",
      "[24]\tvalidation_0-error:0.370784\tvalidation_0-logloss:0.645037\tvalidation_1-error:0.377646\tvalidation_1-logloss:0.646775\n",
      "[25]\tvalidation_0-error:0.370688\tvalidation_0-logloss:0.644159\tvalidation_1-error:0.377611\tvalidation_1-logloss:0.649393\n",
      "[26]\tvalidation_0-error:0.368521\tvalidation_0-logloss:0.642603\tvalidation_1-error:0.375968\tvalidation_1-logloss:0.648554\n",
      "[27]\tvalidation_0-error:0.368659\tvalidation_0-logloss:0.642467\tvalidation_1-error:0.376356\tvalidation_1-logloss:0.648254\n",
      "[28]\tvalidation_0-error:0.368512\tvalidation_0-logloss:0.642075\tvalidation_1-error:0.376388\tvalidation_1-logloss:0.64786\n",
      "[29]\tvalidation_0-error:0.367223\tvalidation_0-logloss:0.642994\tvalidation_1-error:0.374999\tvalidation_1-logloss:0.648574\n",
      "[30]\tvalidation_0-error:0.366911\tvalidation_0-logloss:0.642469\tvalidation_1-error:0.37456\tvalidation_1-logloss:0.648081\n",
      "[31]\tvalidation_0-error:0.366152\tvalidation_0-logloss:0.641826\tvalidation_1-error:0.373838\tvalidation_1-logloss:0.647497\n",
      "[32]\tvalidation_0-error:0.36561\tvalidation_0-logloss:0.641178\tvalidation_1-error:0.373494\tvalidation_1-logloss:0.647037\n",
      "[33]\tvalidation_0-error:0.365472\tvalidation_0-logloss:0.640769\tvalidation_1-error:0.373538\tvalidation_1-logloss:0.646713\n",
      "[34]\tvalidation_0-error:0.364896\tvalidation_0-logloss:0.640004\tvalidation_1-error:0.3731\tvalidation_1-logloss:0.646343\n",
      "[35]\tvalidation_0-error:0.364519\tvalidation_0-logloss:0.639433\tvalidation_1-error:0.372416\tvalidation_1-logloss:0.64588\n",
      "[36]\tvalidation_0-error:0.363808\tvalidation_0-logloss:0.638885\tvalidation_1-error:0.371988\tvalidation_1-logloss:0.645435\n",
      "[37]\tvalidation_0-error:0.36406\tvalidation_0-logloss:0.638407\tvalidation_1-error:0.371936\tvalidation_1-logloss:0.645125\n",
      "[38]\tvalidation_0-error:0.363892\tvalidation_0-logloss:0.636177\tvalidation_1-error:0.371764\tvalidation_1-logloss:0.643278\n",
      "[39]\tvalidation_0-error:0.363361\tvalidation_0-logloss:0.635528\tvalidation_1-error:0.371262\tvalidation_1-logloss:0.642748\n",
      "[40]\tvalidation_0-error:0.362936\tvalidation_0-logloss:0.635583\tvalidation_1-error:0.371127\tvalidation_1-logloss:0.642555\n",
      "[41]\tvalidation_0-error:0.362687\tvalidation_0-logloss:0.635136\tvalidation_1-error:0.370985\tvalidation_1-logloss:0.642238\n",
      "[42]\tvalidation_0-error:0.362462\tvalidation_0-logloss:0.634707\tvalidation_1-error:0.370926\tvalidation_1-logloss:0.641913\n",
      "[43]\tvalidation_0-error:0.361386\tvalidation_0-logloss:0.634262\tvalidation_1-error:0.37025\tvalidation_1-logloss:0.641579\n",
      "[44]\tvalidation_0-error:0.36102\tvalidation_0-logloss:0.633868\tvalidation_1-error:0.370014\tvalidation_1-logloss:0.641198\n",
      "[45]\tvalidation_0-error:0.360858\tvalidation_0-logloss:0.632868\tvalidation_1-error:0.369937\tvalidation_1-logloss:0.640615\n",
      "[46]\tvalidation_0-error:0.360243\tvalidation_0-logloss:0.633552\tvalidation_1-error:0.369469\tvalidation_1-logloss:0.641198\n",
      "[47]\tvalidation_0-error:0.359518\tvalidation_0-logloss:0.633162\tvalidation_1-error:0.368995\tvalidation_1-logloss:0.640887\n",
      "[48]\tvalidation_0-error:0.358577\tvalidation_0-logloss:0.632727\tvalidation_1-error:0.367976\tvalidation_1-logloss:0.640549\n",
      "[49]\tvalidation_0-error:0.35846\tvalidation_0-logloss:0.632428\tvalidation_1-error:0.367794\tvalidation_1-logloss:0.640256\n",
      "[50]\tvalidation_0-error:0.358235\tvalidation_0-logloss:0.632013\tvalidation_1-error:0.36757\tvalidation_1-logloss:0.639966\n",
      "[51]\tvalidation_0-error:0.35798\tvalidation_0-logloss:0.631612\tvalidation_1-error:0.367343\tvalidation_1-logloss:0.639639\n",
      "[52]\tvalidation_0-error:0.357147\tvalidation_0-logloss:0.631279\tvalidation_1-error:0.366603\tvalidation_1-logloss:0.639363\n",
      "[53]\tvalidation_0-error:0.356679\tvalidation_0-logloss:0.630845\tvalidation_1-error:0.366268\tvalidation_1-logloss:0.6356\n",
      "[54]\tvalidation_0-error:0.356538\tvalidation_0-logloss:0.630857\tvalidation_1-error:0.366275\tvalidation_1-logloss:0.635642\n",
      "[55]\tvalidation_0-error:0.356298\tvalidation_0-logloss:0.630625\tvalidation_1-error:0.366068\tvalidation_1-logloss:0.635412\n",
      "[56]\tvalidation_0-error:0.356064\tvalidation_0-logloss:0.63025\tvalidation_1-error:0.365754\tvalidation_1-logloss:0.635218\n",
      "[57]\tvalidation_0-error:0.355725\tvalidation_0-logloss:0.629931\tvalidation_1-error:0.365541\tvalidation_1-logloss:0.634894\n",
      "[58]\tvalidation_0-error:0.355003\tvalidation_0-logloss:0.629578\tvalidation_1-error:0.364693\tvalidation_1-logloss:0.63463\n",
      "[59]\tvalidation_0-error:0.35446\tvalidation_0-logloss:0.628623\tvalidation_1-error:0.364352\tvalidation_1-logloss:0.636384\n",
      "[60]\tvalidation_0-error:0.354238\tvalidation_0-logloss:0.628245\tvalidation_1-error:0.364098\tvalidation_1-logloss:0.636124\n",
      "[61]\tvalidation_0-error:0.353726\tvalidation_0-logloss:0.627987\tvalidation_1-error:0.36367\tvalidation_1-logloss:0.635891\n",
      "[62]\tvalidation_0-error:0.353357\tvalidation_0-logloss:0.627601\tvalidation_1-error:0.363722\tvalidation_1-logloss:0.635655\n",
      "[63]\tvalidation_0-error:0.353105\tvalidation_0-logloss:0.627315\tvalidation_1-error:0.36348\tvalidation_1-logloss:0.635366\n",
      "[64]\tvalidation_0-error:0.35253\tvalidation_0-logloss:0.627454\tvalidation_1-error:0.362972\tvalidation_1-logloss:0.635446\n",
      "[65]\tvalidation_0-error:0.351819\tvalidation_0-logloss:0.625639\tvalidation_1-error:0.362585\tvalidation_1-logloss:0.633918\n",
      "[66]\tvalidation_0-error:0.351792\tvalidation_0-logloss:0.625377\tvalidation_1-error:0.362554\tvalidation_1-logloss:0.633776\n",
      "[67]\tvalidation_0-error:0.351531\tvalidation_0-logloss:0.625106\tvalidation_1-error:0.362238\tvalidation_1-logloss:0.633584\n",
      "[68]\tvalidation_0-error:0.351034\tvalidation_0-logloss:0.624763\tvalidation_1-error:0.362\tvalidation_1-logloss:0.633335\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[69]\tvalidation_0-error:0.350779\tvalidation_0-logloss:0.62455\tvalidation_1-error:0.361852\tvalidation_1-logloss:0.633251\n",
      "[70]\tvalidation_0-error:0.350512\tvalidation_0-logloss:0.624296\tvalidation_1-error:0.361634\tvalidation_1-logloss:0.633111\n",
      "[71]\tvalidation_0-error:0.350047\tvalidation_0-logloss:0.624034\tvalidation_1-error:0.361178\tvalidation_1-logloss:0.63288\n",
      "[72]\tvalidation_0-error:0.349639\tvalidation_0-logloss:0.623757\tvalidation_1-error:0.360897\tvalidation_1-logloss:0.632624\n",
      "[73]\tvalidation_0-error:0.349409\tvalidation_0-logloss:0.624276\tvalidation_1-error:0.360822\tvalidation_1-logloss:0.633185\n",
      "[74]\tvalidation_0-error:0.349019\tvalidation_0-logloss:0.623981\tvalidation_1-error:0.360557\tvalidation_1-logloss:0.632952\n",
      "[75]\tvalidation_0-error:0.347988\tvalidation_0-logloss:0.623797\tvalidation_1-error:0.359942\tvalidation_1-logloss:0.632797\n",
      "[76]\tvalidation_0-error:0.347634\tvalidation_0-logloss:0.62342\tvalidation_1-error:0.359471\tvalidation_1-logloss:0.632525\n",
      "[77]\tvalidation_0-error:0.347364\tvalidation_0-logloss:0.623187\tvalidation_1-error:0.359322\tvalidation_1-logloss:0.632311\n",
      "[78]\tvalidation_0-error:0.346851\tvalidation_0-logloss:0.622907\tvalidation_1-error:0.358912\tvalidation_1-logloss:0.631997\n",
      "[79]\tvalidation_0-error:0.34636\tvalidation_0-logloss:0.621983\tvalidation_1-error:0.358506\tvalidation_1-logloss:0.631423\n",
      "[80]\tvalidation_0-error:0.346144\tvalidation_0-logloss:0.621766\tvalidation_1-error:0.35835\tvalidation_1-logloss:0.631307\n",
      "[81]\tvalidation_0-error:0.345667\tvalidation_0-logloss:0.62147\tvalidation_1-error:0.357894\tvalidation_1-logloss:0.63105\n",
      "[82]\tvalidation_0-error:0.345622\tvalidation_0-logloss:0.621242\tvalidation_1-error:0.357839\tvalidation_1-logloss:0.630919\n",
      "[83]\tvalidation_0-error:0.34519\tvalidation_0-logloss:0.620961\tvalidation_1-error:0.357499\tvalidation_1-logloss:0.63072\n",
      "[84]\tvalidation_0-error:0.344954\tvalidation_0-logloss:0.620704\tvalidation_1-error:0.35733\tvalidation_1-logloss:0.630621\n",
      "[85]\tvalidation_0-error:0.344927\tvalidation_0-logloss:0.620753\tvalidation_1-error:0.357179\tvalidation_1-logloss:0.630584\n",
      "[86]\tvalidation_0-error:0.344717\tvalidation_0-logloss:0.620524\tvalidation_1-error:0.357061\tvalidation_1-logloss:0.630454\n",
      "[87]\tvalidation_0-error:0.344564\tvalidation_0-logloss:0.620277\tvalidation_1-error:0.356934\tvalidation_1-logloss:0.630296\n",
      "[88]\tvalidation_0-error:0.344015\tvalidation_0-logloss:0.620095\tvalidation_1-error:0.356274\tvalidation_1-logloss:0.630227\n",
      "[89]\tvalidation_0-error:0.344315\tvalidation_0-logloss:0.619877\tvalidation_1-error:0.356328\tvalidation_1-logloss:0.629948\n",
      "[90]\tvalidation_0-error:0.344258\tvalidation_0-logloss:0.619606\tvalidation_1-error:0.356356\tvalidation_1-logloss:0.629827\n",
      "[91]\tvalidation_0-error:0.344033\tvalidation_0-logloss:0.61938\tvalidation_1-error:0.356249\tvalidation_1-logloss:0.629638\n",
      "[92]\tvalidation_0-error:0.343832\tvalidation_0-logloss:0.619152\tvalidation_1-error:0.356102\tvalidation_1-logloss:0.629449\n",
      "[93]\tvalidation_0-error:0.343479\tvalidation_0-logloss:0.618937\tvalidation_1-error:0.355881\tvalidation_1-logloss:0.629269\n",
      "[94]\tvalidation_0-error:0.342984\tvalidation_0-logloss:0.618743\tvalidation_1-error:0.355336\tvalidation_1-logloss:0.629131\n",
      "[95]\tvalidation_0-error:0.342507\tvalidation_0-logloss:0.618532\tvalidation_1-error:0.354905\tvalidation_1-logloss:0.628946\n",
      "[96]\tvalidation_0-error:0.341773\tvalidation_0-logloss:0.618336\tvalidation_1-error:0.354416\tvalidation_1-logloss:0.628774\n",
      "[97]\tvalidation_0-error:0.341635\tvalidation_0-logloss:0.618071\tvalidation_1-error:0.354209\tvalidation_1-logloss:0.628605\n",
      "[98]\tvalidation_0-error:0.341422\tvalidation_0-logloss:0.617798\tvalidation_1-error:0.353998\tvalidation_1-logloss:0.628438\n",
      "[99]\tvalidation_0-error:0.340933\tvalidation_0-logloss:0.617554\tvalidation_1-error:0.353669\tvalidation_1-logloss:0.628274\n",
      "[100]\tvalidation_0-error:0.34078\tvalidation_0-logloss:0.617372\tvalidation_1-error:0.353541\tvalidation_1-logloss:0.6281\n",
      "[101]\tvalidation_0-error:0.340645\tvalidation_0-logloss:0.617146\tvalidation_1-error:0.353397\tvalidation_1-logloss:0.627977\n",
      "[102]\tvalidation_0-error:0.340391\tvalidation_0-logloss:0.61729\tvalidation_1-error:0.353245\tvalidation_1-logloss:0.628036\n",
      "[103]\tvalidation_0-error:0.340259\tvalidation_0-logloss:0.617071\tvalidation_1-error:0.3533\tvalidation_1-logloss:0.627837\n",
      "[104]\tvalidation_0-error:0.34019\tvalidation_0-logloss:0.616354\tvalidation_1-error:0.353146\tvalidation_1-logloss:0.627424\n",
      "[105]\tvalidation_0-error:0.339995\tvalidation_0-logloss:0.616277\tvalidation_1-error:0.353\tvalidation_1-logloss:0.627377\n",
      "[106]\tvalidation_0-error:0.339767\tvalidation_0-logloss:0.616088\tvalidation_1-error:0.352879\tvalidation_1-logloss:0.627273\n",
      "[107]\tvalidation_0-error:0.339476\tvalidation_0-logloss:0.615879\tvalidation_1-error:0.352761\tvalidation_1-logloss:0.627111\n",
      "[108]\tvalidation_0-error:0.339185\tvalidation_0-logloss:0.61565\tvalidation_1-error:0.352631\tvalidation_1-logloss:0.627018\n",
      "[109]\tvalidation_0-error:0.339155\tvalidation_0-logloss:0.615505\tvalidation_1-error:0.352598\tvalidation_1-logloss:0.626886\n",
      "[110]\tvalidation_0-error:0.338811\tvalidation_0-logloss:0.615185\tvalidation_1-error:0.35224\tvalidation_1-logloss:0.626674\n",
      "[111]\tvalidation_0-error:0.338502\tvalidation_0-logloss:0.615727\tvalidation_1-error:0.351962\tvalidation_1-logloss:0.627193\n",
      "[112]\tvalidation_0-error:0.338232\tvalidation_0-logloss:0.615539\tvalidation_1-error:0.351647\tvalidation_1-logloss:0.627016\n",
      "[113]\tvalidation_0-error:0.338058\tvalidation_0-logloss:0.615345\tvalidation_1-error:0.351487\tvalidation_1-logloss:0.626851\n",
      "[114]\tvalidation_0-error:0.337929\tvalidation_0-logloss:0.615151\tvalidation_1-error:0.35141\tvalidation_1-logloss:0.626681\n",
      "[115]\tvalidation_0-error:0.337576\tvalidation_0-logloss:0.614961\tvalidation_1-error:0.351303\tvalidation_1-logloss:0.626583\n",
      "[116]\tvalidation_0-error:0.336958\tvalidation_0-logloss:0.614742\tvalidation_1-error:0.350949\tvalidation_1-logloss:0.626443\n",
      "[117]\tvalidation_0-error:0.336607\tvalidation_0-logloss:0.614581\tvalidation_1-error:0.350633\tvalidation_1-logloss:0.626325\n",
      "[118]\tvalidation_0-error:0.336658\tvalidation_0-logloss:0.614386\tvalidation_1-error:0.350791\tvalidation_1-logloss:0.626212\n",
      "[119]\tvalidation_0-error:0.336313\tvalidation_0-logloss:0.614175\tvalidation_1-error:0.350549\tvalidation_1-logloss:0.626098\n",
      "[120]\tvalidation_0-error:0.336262\tvalidation_0-logloss:0.612978\tvalidation_1-error:0.350285\tvalidation_1-logloss:0.624997\n",
      "[121]\tvalidation_0-error:0.33596\tvalidation_0-logloss:0.612823\tvalidation_1-error:0.349977\tvalidation_1-logloss:0.624883\n",
      "[122]\tvalidation_0-error:0.336268\tvalidation_0-logloss:0.612899\tvalidation_1-error:0.350146\tvalidation_1-logloss:0.624949\n",
      "[123]\tvalidation_0-error:0.336373\tvalidation_0-logloss:0.61272\tvalidation_1-error:0.350193\tvalidation_1-logloss:0.624812\n",
      "[124]\tvalidation_0-error:0.335978\tvalidation_0-logloss:0.612406\tvalidation_1-error:0.350053\tvalidation_1-logloss:0.624754\n",
      "[125]\tvalidation_0-error:0.33542\tvalidation_0-logloss:0.6123\tvalidation_1-error:0.349622\tvalidation_1-logloss:0.624644\n",
      "[126]\tvalidation_0-error:0.335498\tvalidation_0-logloss:0.612096\tvalidation_1-error:0.349641\tvalidation_1-logloss:0.624485\n",
      "[127]\tvalidation_0-error:0.335297\tvalidation_0-logloss:0.611884\tvalidation_1-error:0.349483\tvalidation_1-logloss:0.624342\n",
      "[128]\tvalidation_0-error:0.33512\tvalidation_0-logloss:0.611707\tvalidation_1-error:0.349365\tvalidation_1-logloss:0.624237\n",
      "[129]\tvalidation_0-error:0.334973\tvalidation_0-logloss:0.611566\tvalidation_1-error:0.349304\tvalidation_1-logloss:0.624137\n",
      "[130]\tvalidation_0-error:0.334619\tvalidation_0-logloss:0.610926\tvalidation_1-error:0.348992\tvalidation_1-logloss:0.623772\n",
      "[131]\tvalidation_0-error:0.334485\tvalidation_0-logloss:0.610725\tvalidation_1-error:0.348919\tvalidation_1-logloss:0.625133\n",
      "[132]\tvalidation_0-error:0.334041\tvalidation_0-logloss:0.61053\tvalidation_1-error:0.348493\tvalidation_1-logloss:0.625069\n",
      "[133]\tvalidation_0-error:0.333522\tvalidation_0-logloss:0.610399\tvalidation_1-error:0.348219\tvalidation_1-logloss:0.624982\n",
      "[134]\tvalidation_0-error:0.333363\tvalidation_0-logloss:0.610197\tvalidation_1-error:0.348138\tvalidation_1-logloss:0.624795\n",
      "[135]\tvalidation_0-error:0.3333\tvalidation_0-logloss:0.609989\tvalidation_1-error:0.348068\tvalidation_1-logloss:0.624676\n",
      "[136]\tvalidation_0-error:0.332914\tvalidation_0-logloss:0.609832\tvalidation_1-error:0.347749\tvalidation_1-logloss:0.624592\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[137]\tvalidation_0-error:0.33253\tvalidation_0-logloss:0.609652\tvalidation_1-error:0.347585\tvalidation_1-logloss:0.624526\n",
      "[138]\tvalidation_0-error:0.332518\tvalidation_0-logloss:0.609762\tvalidation_1-error:0.34768\tvalidation_1-logloss:0.624563\n",
      "[139]\tvalidation_0-error:0.332245\tvalidation_0-logloss:0.609567\tvalidation_1-error:0.347576\tvalidation_1-logloss:0.624507\n",
      "[140]\tvalidation_0-error:0.332098\tvalidation_0-logloss:0.60939\tvalidation_1-error:0.347504\tvalidation_1-logloss:0.624328\n",
      "[141]\tvalidation_0-error:0.331984\tvalidation_0-logloss:0.609249\tvalidation_1-error:0.347467\tvalidation_1-logloss:0.624222\n",
      "[142]\tvalidation_0-error:0.331894\tvalidation_0-logloss:0.609105\tvalidation_1-error:0.347449\tvalidation_1-logloss:0.624083\n",
      "[143]\tvalidation_0-error:0.331681\tvalidation_0-logloss:0.608898\tvalidation_1-error:0.347198\tvalidation_1-logloss:0.623903\n",
      "[144]\tvalidation_0-error:0.331591\tvalidation_0-logloss:0.608696\tvalidation_1-error:0.346983\tvalidation_1-logloss:0.623712\n",
      "[145]\tvalidation_0-error:0.331448\tvalidation_0-logloss:0.608554\tvalidation_1-error:0.346918\tvalidation_1-logloss:0.623613\n",
      "[146]\tvalidation_0-error:0.331133\tvalidation_0-logloss:0.60839\tvalidation_1-error:0.346918\tvalidation_1-logloss:0.62351\n",
      "[147]\tvalidation_0-error:0.33089\tvalidation_0-logloss:0.608213\tvalidation_1-error:0.346713\tvalidation_1-logloss:0.623388\n",
      "[148]\tvalidation_0-error:0.330797\tvalidation_0-logloss:0.608077\tvalidation_1-error:0.346783\tvalidation_1-logloss:0.62335\n",
      "[149]\tvalidation_0-error:0.330539\tvalidation_0-logloss:0.607927\tvalidation_1-error:0.346627\tvalidation_1-logloss:0.623218\n",
      "[150]\tvalidation_0-error:0.330431\tvalidation_0-logloss:0.607773\tvalidation_1-error:0.346617\tvalidation_1-logloss:0.623141\n",
      "[151]\tvalidation_0-error:0.330305\tvalidation_0-logloss:0.607637\tvalidation_1-error:0.346518\tvalidation_1-logloss:0.623045\n",
      "[152]\tvalidation_0-error:0.330098\tvalidation_0-logloss:0.607493\tvalidation_1-error:0.346315\tvalidation_1-logloss:0.622936\n",
      "[153]\tvalidation_0-error:0.329997\tvalidation_0-logloss:0.607343\tvalidation_1-error:0.346273\tvalidation_1-logloss:0.620608\n",
      "[154]\tvalidation_0-error:0.329796\tvalidation_0-logloss:0.607174\tvalidation_1-error:0.345992\tvalidation_1-logloss:0.622712\n",
      "[155]\tvalidation_0-error:0.329736\tvalidation_0-logloss:0.607011\tvalidation_1-error:0.345931\tvalidation_1-logloss:0.622587\n",
      "[156]\tvalidation_0-error:0.32961\tvalidation_0-logloss:0.606874\tvalidation_1-error:0.345819\tvalidation_1-logloss:0.620262\n",
      "[157]\tvalidation_0-error:0.329556\tvalidation_0-logloss:0.606731\tvalidation_1-error:0.34575\tvalidation_1-logloss:0.620107\n",
      "[158]\tvalidation_0-error:0.329457\tvalidation_0-logloss:0.606587\tvalidation_1-error:0.345742\tvalidation_1-logloss:0.620008\n",
      "[159]\tvalidation_0-error:0.329067\tvalidation_0-logloss:0.60642\tvalidation_1-error:0.34561\tvalidation_1-logloss:0.61986\n",
      "[160]\tvalidation_0-error:0.328839\tvalidation_0-logloss:0.606315\tvalidation_1-error:0.34541\tvalidation_1-logloss:0.61981\n",
      "[161]\tvalidation_0-error:0.328764\tvalidation_0-logloss:0.60617\tvalidation_1-error:0.345171\tvalidation_1-logloss:0.619714\n",
      "[162]\tvalidation_0-error:0.328429\tvalidation_0-logloss:0.606031\tvalidation_1-error:0.345156\tvalidation_1-logloss:0.61968\n",
      "[163]\tvalidation_0-error:0.328288\tvalidation_0-logloss:0.605866\tvalidation_1-error:0.345014\tvalidation_1-logloss:0.619575\n",
      "[164]\tvalidation_0-error:0.328135\tvalidation_0-logloss:0.605722\tvalidation_1-error:0.344899\tvalidation_1-logloss:0.619467\n",
      "[165]\tvalidation_0-error:0.327886\tvalidation_0-logloss:0.60556\tvalidation_1-error:0.344858\tvalidation_1-logloss:0.619391\n",
      "[166]\tvalidation_0-error:0.327793\tvalidation_0-logloss:0.605407\tvalidation_1-error:0.344681\tvalidation_1-logloss:0.6193\n",
      "[167]\tvalidation_0-error:0.327772\tvalidation_0-logloss:0.605717\tvalidation_1-error:0.344694\tvalidation_1-logloss:0.619661\n",
      "[168]\tvalidation_0-error:0.327574\tvalidation_0-logloss:0.605567\tvalidation_1-error:0.344605\tvalidation_1-logloss:0.619646\n",
      "[169]\tvalidation_0-error:0.327484\tvalidation_0-logloss:0.605468\tvalidation_1-error:0.34457\tvalidation_1-logloss:0.619599\n",
      "[170]\tvalidation_0-error:0.327334\tvalidation_0-logloss:0.605299\tvalidation_1-error:0.344512\tvalidation_1-logloss:0.61952\n",
      "[171]\tvalidation_0-error:0.327226\tvalidation_0-logloss:0.605145\tvalidation_1-error:0.344446\tvalidation_1-logloss:0.619454\n",
      "[172]\tvalidation_0-error:0.327097\tvalidation_0-logloss:0.605017\tvalidation_1-error:0.344375\tvalidation_1-logloss:0.619363\n",
      "[173]\tvalidation_0-error:0.326951\tvalidation_0-logloss:0.604882\tvalidation_1-error:0.344312\tvalidation_1-logloss:0.619251\n",
      "[174]\tvalidation_0-error:0.326873\tvalidation_0-logloss:0.604748\tvalidation_1-error:0.344326\tvalidation_1-logloss:0.619198\n",
      "[175]\tvalidation_0-error:0.326576\tvalidation_0-logloss:0.604205\tvalidation_1-error:0.344109\tvalidation_1-logloss:0.618935\n",
      "[176]\tvalidation_0-error:0.326312\tvalidation_0-logloss:0.604076\tvalidation_1-error:0.344032\tvalidation_1-logloss:0.61886\n",
      "[177]\tvalidation_0-error:0.326231\tvalidation_0-logloss:0.604196\tvalidation_1-error:0.343982\tvalidation_1-logloss:0.61893\n",
      "[178]\tvalidation_0-error:0.326138\tvalidation_0-logloss:0.60407\tvalidation_1-error:0.343863\tvalidation_1-logloss:0.61885\n",
      "[179]\tvalidation_0-error:0.326123\tvalidation_0-logloss:0.603925\tvalidation_1-error:0.343842\tvalidation_1-logloss:0.618762\n",
      "[180]\tvalidation_0-error:0.325859\tvalidation_0-logloss:0.603819\tvalidation_1-error:0.34386\tvalidation_1-logloss:0.618732\n",
      "[181]\tvalidation_0-error:0.325742\tvalidation_0-logloss:0.603626\tvalidation_1-error:0.343818\tvalidation_1-logloss:0.618596\n",
      "[182]\tvalidation_0-error:0.325766\tvalidation_0-logloss:0.60351\tvalidation_1-error:0.343898\tvalidation_1-logloss:0.61853\n",
      "[183]\tvalidation_0-error:0.32561\tvalidation_0-logloss:0.603369\tvalidation_1-error:0.343787\tvalidation_1-logloss:0.618459\n",
      "[184]\tvalidation_0-error:0.325416\tvalidation_0-logloss:0.603242\tvalidation_1-error:0.343711\tvalidation_1-logloss:0.618375\n",
      "[185]\tvalidation_0-error:0.325359\tvalidation_0-logloss:0.603129\tvalidation_1-error:0.343736\tvalidation_1-logloss:0.618318\n",
      "[186]\tvalidation_0-error:0.325437\tvalidation_0-logloss:0.603024\tvalidation_1-error:0.343737\tvalidation_1-logloss:0.61824\n",
      "[187]\tvalidation_0-error:0.325242\tvalidation_0-logloss:0.602835\tvalidation_1-error:0.343564\tvalidation_1-logloss:0.618135\n",
      "[188]\tvalidation_0-error:0.325086\tvalidation_0-logloss:0.602753\tvalidation_1-error:0.343535\tvalidation_1-logloss:0.618119\n",
      "[189]\tvalidation_0-error:0.32484\tvalidation_0-logloss:0.602603\tvalidation_1-error:0.34334\tvalidation_1-logloss:0.617993\n",
      "[190]\tvalidation_0-error:0.32478\tvalidation_0-logloss:0.602478\tvalidation_1-error:0.343299\tvalidation_1-logloss:0.617896\n",
      "[191]\tvalidation_0-error:0.32472\tvalidation_0-logloss:0.602334\tvalidation_1-error:0.343226\tvalidation_1-logloss:0.617777\n",
      "[192]\tvalidation_0-error:0.324426\tvalidation_0-logloss:0.602213\tvalidation_1-error:0.343135\tvalidation_1-logloss:0.617741\n",
      "[193]\tvalidation_0-error:0.324261\tvalidation_0-logloss:0.601194\tvalidation_1-error:0.343085\tvalidation_1-logloss:0.616871\n",
      "[194]\tvalidation_0-error:0.324132\tvalidation_0-logloss:0.601063\tvalidation_1-error:0.343015\tvalidation_1-logloss:0.616765\n",
      "[195]\tvalidation_0-error:0.32389\tvalidation_0-logloss:0.600925\tvalidation_1-error:0.342921\tvalidation_1-logloss:0.616719\n",
      "[196]\tvalidation_0-error:0.32368\tvalidation_0-logloss:0.600825\tvalidation_1-error:0.342758\tvalidation_1-logloss:0.616644\n",
      "[197]\tvalidation_0-error:0.32356\tvalidation_0-logloss:0.600675\tvalidation_1-error:0.342711\tvalidation_1-logloss:0.61655\n",
      "[198]\tvalidation_0-error:0.32341\tvalidation_0-logloss:0.600593\tvalidation_1-error:0.342642\tvalidation_1-logloss:0.616492\n",
      "[199]\tvalidation_0-error:0.323212\tvalidation_0-logloss:0.600433\tvalidation_1-error:0.34259\tvalidation_1-logloss:0.616363\n",
      "[200]\tvalidation_0-error:0.323089\tvalidation_0-logloss:0.600208\tvalidation_1-error:0.342546\tvalidation_1-logloss:0.61624\n",
      "[201]\tvalidation_0-error:0.3229\tvalidation_0-logloss:0.6001\tvalidation_1-error:0.342429\tvalidation_1-logloss:0.616178\n",
      "[202]\tvalidation_0-error:0.322822\tvalidation_0-logloss:0.599952\tvalidation_1-error:0.342334\tvalidation_1-logloss:0.616108\n",
      "[203]\tvalidation_0-error:0.322741\tvalidation_0-logloss:0.599833\tvalidation_1-error:0.342253\tvalidation_1-logloss:0.616001\n",
      "[204]\tvalidation_0-error:0.322511\tvalidation_0-logloss:0.599964\tvalidation_1-error:0.342185\tvalidation_1-logloss:0.616049\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[205]\tvalidation_0-error:0.322337\tvalidation_0-logloss:0.599834\tvalidation_1-error:0.342134\tvalidation_1-logloss:0.615991\n",
      "[206]\tvalidation_0-error:0.322301\tvalidation_0-logloss:0.599741\tvalidation_1-error:0.342066\tvalidation_1-logloss:0.615954\n",
      "[207]\tvalidation_0-error:0.322103\tvalidation_0-logloss:0.599594\tvalidation_1-error:0.34197\tvalidation_1-logloss:0.615871\n",
      "[208]\tvalidation_0-error:0.321878\tvalidation_0-logloss:0.599471\tvalidation_1-error:0.341856\tvalidation_1-logloss:0.615771\n",
      "[209]\tvalidation_0-error:0.32183\tvalidation_0-logloss:0.599054\tvalidation_1-error:0.341854\tvalidation_1-logloss:0.615595\n",
      "[210]\tvalidation_0-error:0.321713\tvalidation_0-logloss:0.598929\tvalidation_1-error:0.341792\tvalidation_1-logloss:0.615537\n",
      "[211]\tvalidation_0-error:0.32159\tvalidation_0-logloss:0.598807\tvalidation_1-error:0.341639\tvalidation_1-logloss:0.615445\n",
      "[212]\tvalidation_0-error:0.321407\tvalidation_0-logloss:0.598685\tvalidation_1-error:0.341482\tvalidation_1-logloss:0.615322\n",
      "[213]\tvalidation_0-error:0.321335\tvalidation_0-logloss:0.598578\tvalidation_1-error:0.341456\tvalidation_1-logloss:0.615226\n",
      "[214]\tvalidation_0-error:0.321179\tvalidation_0-logloss:0.598454\tvalidation_1-error:0.341345\tvalidation_1-logloss:0.615148\n",
      "[215]\tvalidation_0-error:0.321158\tvalidation_0-logloss:0.598359\tvalidation_1-error:0.341313\tvalidation_1-logloss:0.615088\n",
      "[216]\tvalidation_0-error:0.320985\tvalidation_0-logloss:0.598215\tvalidation_1-error:0.341146\tvalidation_1-logloss:0.615011\n",
      "[217]\tvalidation_0-error:0.320988\tvalidation_0-logloss:0.598118\tvalidation_1-error:0.341191\tvalidation_1-logloss:0.614966\n",
      "[218]\tvalidation_0-error:0.320841\tvalidation_0-logloss:0.598003\tvalidation_1-error:0.341157\tvalidation_1-logloss:0.614923\n",
      "[219]\tvalidation_0-error:0.320895\tvalidation_0-logloss:0.597885\tvalidation_1-error:0.34094\tvalidation_1-logloss:0.614871\n",
      "[220]\tvalidation_0-error:0.32076\tvalidation_0-logloss:0.597784\tvalidation_1-error:0.340906\tvalidation_1-logloss:0.614837\n",
      "[221]\tvalidation_0-error:0.320625\tvalidation_0-logloss:0.597659\tvalidation_1-error:0.340743\tvalidation_1-logloss:0.614753\n",
      "[222]\tvalidation_0-error:0.320421\tvalidation_0-logloss:0.598004\tvalidation_1-error:0.340577\tvalidation_1-logloss:0.615215\n",
      "[223]\tvalidation_0-error:0.32031\tvalidation_0-logloss:0.597883\tvalidation_1-error:0.34057\tvalidation_1-logloss:0.61517\n",
      "[224]\tvalidation_0-error:0.320292\tvalidation_0-logloss:0.597652\tvalidation_1-error:0.340599\tvalidation_1-logloss:0.615001\n",
      "[225]\tvalidation_0-error:0.320091\tvalidation_0-logloss:0.597555\tvalidation_1-error:0.340542\tvalidation_1-logloss:0.61493\n",
      "[226]\tvalidation_0-error:0.31998\tvalidation_0-logloss:0.597446\tvalidation_1-error:0.340466\tvalidation_1-logloss:0.614867\n",
      "[227]\tvalidation_0-error:0.31983\tvalidation_0-logloss:0.597311\tvalidation_1-error:0.340357\tvalidation_1-logloss:0.614751\n",
      "[228]\tvalidation_0-error:0.319821\tvalidation_0-logloss:0.597269\tvalidation_1-error:0.340274\tvalidation_1-logloss:0.614727\n",
      "[229]\tvalidation_0-error:0.319749\tvalidation_0-logloss:0.597116\tvalidation_1-error:0.340242\tvalidation_1-logloss:0.614629\n",
      "[230]\tvalidation_0-error:0.31971\tvalidation_0-logloss:0.596982\tvalidation_1-error:0.340196\tvalidation_1-logloss:0.614517\n",
      "[231]\tvalidation_0-error:0.319698\tvalidation_0-logloss:0.59694\tvalidation_1-error:0.340227\tvalidation_1-logloss:0.614499\n",
      "[232]\tvalidation_0-error:0.319557\tvalidation_0-logloss:0.596813\tvalidation_1-error:0.340167\tvalidation_1-logloss:0.614446\n",
      "[233]\tvalidation_0-error:0.319465\tvalidation_0-logloss:0.596679\tvalidation_1-error:0.340032\tvalidation_1-logloss:0.614359\n",
      "[234]\tvalidation_0-error:0.319405\tvalidation_0-logloss:0.596595\tvalidation_1-error:0.33995\tvalidation_1-logloss:0.614265\n",
      "[235]\tvalidation_0-error:0.319393\tvalidation_0-logloss:0.596528\tvalidation_1-error:0.339911\tvalidation_1-logloss:0.614249\n",
      "[236]\tvalidation_0-error:0.319345\tvalidation_0-logloss:0.596416\tvalidation_1-error:0.339871\tvalidation_1-logloss:0.614181\n",
      "[237]\tvalidation_0-error:0.319252\tvalidation_0-logloss:0.596314\tvalidation_1-error:0.339795\tvalidation_1-logloss:0.614109\n",
      "[238]\tvalidation_0-error:0.319204\tvalidation_0-logloss:0.596181\tvalidation_1-error:0.339665\tvalidation_1-logloss:0.614036\n",
      "[239]\tvalidation_0-error:0.31906\tvalidation_0-logloss:0.596304\tvalidation_1-error:0.339583\tvalidation_1-logloss:0.614128\n",
      "[240]\tvalidation_0-error:0.318967\tvalidation_0-logloss:0.596169\tvalidation_1-error:0.339527\tvalidation_1-logloss:0.614067\n",
      "[241]\tvalidation_0-error:0.318916\tvalidation_0-logloss:0.596078\tvalidation_1-error:0.339468\tvalidation_1-logloss:0.614031\n",
      "[242]\tvalidation_0-error:0.318781\tvalidation_0-logloss:0.595979\tvalidation_1-error:0.339386\tvalidation_1-logloss:0.614\n",
      "[243]\tvalidation_0-error:0.318709\tvalidation_0-logloss:0.595848\tvalidation_1-error:0.339341\tvalidation_1-logloss:0.613927\n",
      "[244]\tvalidation_0-error:0.318427\tvalidation_0-logloss:0.595737\tvalidation_1-error:0.339161\tvalidation_1-logloss:0.613865\n",
      "[245]\tvalidation_0-error:0.31837\tvalidation_0-logloss:0.595659\tvalidation_1-error:0.339112\tvalidation_1-logloss:0.6138\n",
      "[246]\tvalidation_0-error:0.318205\tvalidation_0-logloss:0.595533\tvalidation_1-error:0.339012\tvalidation_1-logloss:0.613745\n",
      "[247]\tvalidation_0-error:0.318091\tvalidation_0-logloss:0.595357\tvalidation_1-error:0.338962\tvalidation_1-logloss:0.613642\n",
      "[248]\tvalidation_0-error:0.317939\tvalidation_0-logloss:0.595294\tvalidation_1-error:0.338903\tvalidation_1-logloss:0.613614\n",
      "[249]\tvalidation_0-error:0.317831\tvalidation_0-logloss:0.595181\tvalidation_1-error:0.33882\tvalidation_1-logloss:0.613531\n",
      "[250]\tvalidation_0-error:0.317777\tvalidation_0-logloss:0.595005\tvalidation_1-error:0.338835\tvalidation_1-logloss:0.614336\n",
      "[251]\tvalidation_0-error:0.317675\tvalidation_0-logloss:0.594911\tvalidation_1-error:0.338757\tvalidation_1-logloss:0.614266\n",
      "[252]\tvalidation_0-error:0.317591\tvalidation_0-logloss:0.594824\tvalidation_1-error:0.338723\tvalidation_1-logloss:0.614221\n",
      "[253]\tvalidation_0-error:0.317567\tvalidation_0-logloss:0.594726\tvalidation_1-error:0.338728\tvalidation_1-logloss:0.614187\n",
      "[254]\tvalidation_0-error:0.317489\tvalidation_0-logloss:0.594648\tvalidation_1-error:0.338723\tvalidation_1-logloss:0.614193\n",
      "[255]\tvalidation_0-error:0.317324\tvalidation_0-logloss:0.594541\tvalidation_1-error:0.338622\tvalidation_1-logloss:0.614106\n",
      "[256]\tvalidation_0-error:0.317252\tvalidation_0-logloss:0.594463\tvalidation_1-error:0.338584\tvalidation_1-logloss:0.614062\n",
      "[257]\tvalidation_0-error:0.317138\tvalidation_0-logloss:0.594328\tvalidation_1-error:0.338612\tvalidation_1-logloss:0.613998\n",
      "[258]\tvalidation_0-error:0.317087\tvalidation_0-logloss:0.594235\tvalidation_1-error:0.338573\tvalidation_1-logloss:0.613957\n",
      "[259]\tvalidation_0-error:0.316931\tvalidation_0-logloss:0.594157\tvalidation_1-error:0.338368\tvalidation_1-logloss:0.613919\n",
      "[260]\tvalidation_0-error:0.316913\tvalidation_0-logloss:0.594093\tvalidation_1-error:0.338343\tvalidation_1-logloss:0.613901\n",
      "[261]\tvalidation_0-error:0.316793\tvalidation_0-logloss:0.593977\tvalidation_1-error:0.338336\tvalidation_1-logloss:0.613837\n",
      "[262]\tvalidation_0-error:0.316658\tvalidation_0-logloss:0.593572\tvalidation_1-error:0.338293\tvalidation_1-logloss:0.613629\n",
      "[263]\tvalidation_0-error:0.316541\tvalidation_0-logloss:0.593807\tvalidation_1-error:0.338181\tvalidation_1-logloss:0.613755\n",
      "[264]\tvalidation_0-error:0.31629\tvalidation_0-logloss:0.593395\tvalidation_1-error:0.338002\tvalidation_1-logloss:0.613528\n",
      "[265]\tvalidation_0-error:0.316113\tvalidation_0-logloss:0.593295\tvalidation_1-error:0.337913\tvalidation_1-logloss:0.613448\n",
      "[266]\tvalidation_0-error:0.315954\tvalidation_0-logloss:0.593209\tvalidation_1-error:0.337878\tvalidation_1-logloss:0.6134\n",
      "[267]\tvalidation_0-error:0.315948\tvalidation_0-logloss:0.593164\tvalidation_1-error:0.337875\tvalidation_1-logloss:0.613393\n",
      "[268]\tvalidation_0-error:0.315867\tvalidation_0-logloss:0.593056\tvalidation_1-error:0.337946\tvalidation_1-logloss:0.613351\n",
      "[269]\tvalidation_0-error:0.315795\tvalidation_0-logloss:0.592941\tvalidation_1-error:0.337856\tvalidation_1-logloss:0.61329\n",
      "[270]\tvalidation_0-error:0.315603\tvalidation_0-logloss:0.592823\tvalidation_1-error:0.337664\tvalidation_1-logloss:0.613241\n",
      "[271]\tvalidation_0-error:0.315471\tvalidation_0-logloss:0.592738\tvalidation_1-error:0.33755\tvalidation_1-logloss:0.613207\n",
      "[272]\tvalidation_0-error:0.315348\tvalidation_0-logloss:0.592631\tvalidation_1-error:0.337608\tvalidation_1-logloss:0.613148\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[273]\tvalidation_0-error:0.315327\tvalidation_0-logloss:0.592551\tvalidation_1-error:0.337492\tvalidation_1-logloss:0.613072\n",
      "[274]\tvalidation_0-error:0.315195\tvalidation_0-logloss:0.592453\tvalidation_1-error:0.337472\tvalidation_1-logloss:0.613035\n",
      "[275]\tvalidation_0-error:0.31486\tvalidation_0-logloss:0.592333\tvalidation_1-error:0.33733\tvalidation_1-logloss:0.613018\n",
      "[276]\tvalidation_0-error:0.314806\tvalidation_0-logloss:0.592246\tvalidation_1-error:0.337343\tvalidation_1-logloss:0.613009\n",
      "[277]\tvalidation_0-error:0.314569\tvalidation_0-logloss:0.592154\tvalidation_1-error:0.337216\tvalidation_1-logloss:0.612981\n",
      "[278]\tvalidation_0-error:0.314467\tvalidation_0-logloss:0.59204\tvalidation_1-error:0.337234\tvalidation_1-logloss:0.612873\n",
      "[279]\tvalidation_0-error:0.31447\tvalidation_0-logloss:0.592117\tvalidation_1-error:0.337221\tvalidation_1-logloss:0.612881\n",
      "[280]\tvalidation_0-error:0.31441\tvalidation_0-logloss:0.591958\tvalidation_1-error:0.337153\tvalidation_1-logloss:0.612768\n",
      "[281]\tvalidation_0-error:0.314344\tvalidation_0-logloss:0.591868\tvalidation_1-error:0.33712\tvalidation_1-logloss:0.612722\n",
      "[282]\tvalidation_0-error:0.314281\tvalidation_0-logloss:0.591769\tvalidation_1-error:0.33711\tvalidation_1-logloss:0.612655\n",
      "[283]\tvalidation_0-error:0.314188\tvalidation_0-logloss:0.591682\tvalidation_1-error:0.337008\tvalidation_1-logloss:0.612584\n",
      "[284]\tvalidation_0-error:0.31402\tvalidation_0-logloss:0.591581\tvalidation_1-error:0.336976\tvalidation_1-logloss:0.612515\n",
      "[285]\tvalidation_0-error:0.313903\tvalidation_0-logloss:0.591511\tvalidation_1-error:0.33694\tvalidation_1-logloss:0.612441\n",
      "[286]\tvalidation_0-error:0.313909\tvalidation_0-logloss:0.591446\tvalidation_1-error:0.337021\tvalidation_1-logloss:0.612398\n",
      "[287]\tvalidation_0-error:0.313846\tvalidation_0-logloss:0.591357\tvalidation_1-error:0.336987\tvalidation_1-logloss:0.612365\n",
      "[288]\tvalidation_0-error:0.313786\tvalidation_0-logloss:0.591268\tvalidation_1-error:0.336949\tvalidation_1-logloss:0.612361\n",
      "[289]\tvalidation_0-error:0.313732\tvalidation_0-logloss:0.591139\tvalidation_1-error:0.336838\tvalidation_1-logloss:0.612286\n",
      "[290]\tvalidation_0-error:0.313603\tvalidation_0-logloss:0.591044\tvalidation_1-error:0.336748\tvalidation_1-logloss:0.612235\n",
      "[291]\tvalidation_0-error:0.313505\tvalidation_0-logloss:0.590985\tvalidation_1-error:0.336627\tvalidation_1-logloss:0.612174\n",
      "[292]\tvalidation_0-error:0.313457\tvalidation_0-logloss:0.590266\tvalidation_1-error:0.336642\tvalidation_1-logloss:0.611494\n",
      "[293]\tvalidation_0-error:0.313322\tvalidation_0-logloss:0.590808\tvalidation_1-error:0.336503\tvalidation_1-logloss:0.612067\n",
      "[294]\tvalidation_0-error:0.313265\tvalidation_0-logloss:0.590104\tvalidation_1-error:0.336496\tvalidation_1-logloss:0.611411\n",
      "[295]\tvalidation_0-error:0.313139\tvalidation_0-logloss:0.590011\tvalidation_1-error:0.336419\tvalidation_1-logloss:0.611348\n",
      "[296]\tvalidation_0-error:0.313058\tvalidation_0-logloss:0.59057\tvalidation_1-error:0.336369\tvalidation_1-logloss:0.611906\n",
      "[297]\tvalidation_0-error:0.313109\tvalidation_0-logloss:0.589867\tvalidation_1-error:0.336378\tvalidation_1-logloss:0.611249\n",
      "[298]\tvalidation_0-error:0.313082\tvalidation_0-logloss:0.589763\tvalidation_1-error:0.336373\tvalidation_1-logloss:0.61119\n",
      "[299]\tvalidation_0-error:0.312863\tvalidation_0-logloss:0.589678\tvalidation_1-error:0.336267\tvalidation_1-logloss:0.611142\n",
      "[300]\tvalidation_0-error:0.312746\tvalidation_0-logloss:0.589926\tvalidation_1-error:0.336234\tvalidation_1-logloss:0.611428\n",
      "[301]\tvalidation_0-error:0.312698\tvalidation_0-logloss:0.589832\tvalidation_1-error:0.336183\tvalidation_1-logloss:0.611374\n",
      "[302]\tvalidation_0-error:0.312587\tvalidation_0-logloss:0.589729\tvalidation_1-error:0.336141\tvalidation_1-logloss:0.611312\n",
      "[303]\tvalidation_0-error:0.312485\tvalidation_0-logloss:0.58964\tvalidation_1-error:0.336068\tvalidation_1-logloss:0.611253\n",
      "[304]\tvalidation_0-error:0.312392\tvalidation_0-logloss:0.589544\tvalidation_1-error:0.336023\tvalidation_1-logloss:0.611198\n",
      "[305]\tvalidation_0-error:0.312377\tvalidation_0-logloss:0.589436\tvalidation_1-error:0.336019\tvalidation_1-logloss:0.611149\n",
      "[306]\tvalidation_0-error:0.312236\tvalidation_0-logloss:0.589331\tvalidation_1-error:0.335899\tvalidation_1-logloss:0.611113\n",
      "[307]\tvalidation_0-error:0.31214\tvalidation_0-logloss:0.589243\tvalidation_1-error:0.335893\tvalidation_1-logloss:0.61105\n",
      "[308]\tvalidation_0-error:0.312167\tvalidation_0-logloss:0.589149\tvalidation_1-error:0.335846\tvalidation_1-logloss:0.611004\n",
      "[309]\tvalidation_0-error:0.312137\tvalidation_0-logloss:0.589039\tvalidation_1-error:0.335772\tvalidation_1-logloss:0.61094\n",
      "[310]\tvalidation_0-error:0.312074\tvalidation_0-logloss:0.588991\tvalidation_1-error:0.335765\tvalidation_1-logloss:0.610917\n",
      "[311]\tvalidation_0-error:0.312\tvalidation_0-logloss:0.588875\tvalidation_1-error:0.335712\tvalidation_1-logloss:0.610888\n",
      "[312]\tvalidation_0-error:0.311922\tvalidation_0-logloss:0.588794\tvalidation_1-error:0.335704\tvalidation_1-logloss:0.610851\n",
      "[313]\tvalidation_0-error:0.311712\tvalidation_0-logloss:0.588684\tvalidation_1-error:0.335563\tvalidation_1-logloss:0.610808\n",
      "[314]\tvalidation_0-error:0.311658\tvalidation_0-logloss:0.588609\tvalidation_1-error:0.335546\tvalidation_1-logloss:0.610765\n",
      "[315]\tvalidation_0-error:0.311526\tvalidation_0-logloss:0.58853\tvalidation_1-error:0.335403\tvalidation_1-logloss:0.610696\n",
      "[316]\tvalidation_0-error:0.3114\tvalidation_0-logloss:0.588435\tvalidation_1-error:0.335335\tvalidation_1-logloss:0.610668\n",
      "[317]\tvalidation_0-error:0.311196\tvalidation_0-logloss:0.588329\tvalidation_1-error:0.335156\tvalidation_1-logloss:0.610623\n",
      "[318]\tvalidation_0-error:0.311202\tvalidation_0-logloss:0.588241\tvalidation_1-error:0.335149\tvalidation_1-logloss:0.610572\n",
      "[319]\tvalidation_0-error:0.311073\tvalidation_0-logloss:0.588149\tvalidation_1-error:0.334993\tvalidation_1-logloss:0.610505\n",
      "[320]\tvalidation_0-error:0.310998\tvalidation_0-logloss:0.588064\tvalidation_1-error:0.334976\tvalidation_1-logloss:0.610471\n",
      "[321]\tvalidation_0-error:0.310941\tvalidation_0-logloss:0.587991\tvalidation_1-error:0.334922\tvalidation_1-logloss:0.610431\n",
      "[322]\tvalidation_0-error:0.310881\tvalidation_0-logloss:0.587913\tvalidation_1-error:0.334888\tvalidation_1-logloss:0.610387\n",
      "[323]\tvalidation_0-error:0.310845\tvalidation_0-logloss:0.587812\tvalidation_1-error:0.334917\tvalidation_1-logloss:0.610346\n",
      "[324]\tvalidation_0-error:0.3108\tvalidation_0-logloss:0.587745\tvalidation_1-error:0.334879\tvalidation_1-logloss:0.610288\n",
      "[325]\tvalidation_0-error:0.310743\tvalidation_0-logloss:0.587849\tvalidation_1-error:0.334832\tvalidation_1-logloss:0.61036\n",
      "[326]\tvalidation_0-error:0.310641\tvalidation_0-logloss:0.587764\tvalidation_1-error:0.334809\tvalidation_1-logloss:0.610321\n",
      "[327]\tvalidation_0-error:0.310522\tvalidation_0-logloss:0.587668\tvalidation_1-error:0.334799\tvalidation_1-logloss:0.610287\n",
      "[328]\tvalidation_0-error:0.310444\tvalidation_0-logloss:0.587601\tvalidation_1-error:0.334745\tvalidation_1-logloss:0.610251\n",
      "[329]\tvalidation_0-error:0.310342\tvalidation_0-logloss:0.587533\tvalidation_1-error:0.334665\tvalidation_1-logloss:0.610208\n",
      "[330]\tvalidation_0-error:0.310183\tvalidation_0-logloss:0.587445\tvalidation_1-error:0.334595\tvalidation_1-logloss:0.610148\n",
      "[331]\tvalidation_0-error:0.310147\tvalidation_0-logloss:0.587372\tvalidation_1-error:0.334585\tvalidation_1-logloss:0.610113\n",
      "[332]\tvalidation_0-error:0.310093\tvalidation_0-logloss:0.587302\tvalidation_1-error:0.334541\tvalidation_1-logloss:0.610061\n",
      "[333]\tvalidation_0-error:0.310045\tvalidation_0-logloss:0.587195\tvalidation_1-error:0.334501\tvalidation_1-logloss:0.610033\n",
      "[334]\tvalidation_0-error:0.309988\tvalidation_0-logloss:0.586827\tvalidation_1-error:0.33439\tvalidation_1-logloss:0.609866\n",
      "[335]\tvalidation_0-error:0.309916\tvalidation_0-logloss:0.586748\tvalidation_1-error:0.334356\tvalidation_1-logloss:0.609829\n",
      "[336]\tvalidation_0-error:0.309763\tvalidation_0-logloss:0.586675\tvalidation_1-error:0.334276\tvalidation_1-logloss:0.609785\n",
      "[337]\tvalidation_0-error:0.3097\tvalidation_0-logloss:0.586608\tvalidation_1-error:0.334242\tvalidation_1-logloss:0.609743\n",
      "[338]\tvalidation_0-error:0.309595\tvalidation_0-logloss:0.586535\tvalidation_1-error:0.334214\tvalidation_1-logloss:0.609737\n",
      "[339]\tvalidation_0-error:0.309475\tvalidation_0-logloss:0.58646\tvalidation_1-error:0.334145\tvalidation_1-logloss:0.609719\n",
      "[340]\tvalidation_0-error:0.309376\tvalidation_0-logloss:0.586429\tvalidation_1-error:0.33418\tvalidation_1-logloss:0.609701\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[341]\tvalidation_0-error:0.309322\tvalidation_0-logloss:0.586358\tvalidation_1-error:0.334183\tvalidation_1-logloss:0.609654\n",
      "[342]\tvalidation_0-error:0.309298\tvalidation_0-logloss:0.586262\tvalidation_1-error:0.334098\tvalidation_1-logloss:0.609593\n",
      "[343]\tvalidation_0-error:0.309271\tvalidation_0-logloss:0.586167\tvalidation_1-error:0.334084\tvalidation_1-logloss:0.609544\n",
      "[344]\tvalidation_0-error:0.309223\tvalidation_0-logloss:0.586083\tvalidation_1-error:0.33411\tvalidation_1-logloss:0.609529\n",
      "[345]\tvalidation_0-error:0.30916\tvalidation_0-logloss:0.585909\tvalidation_1-error:0.33403\tvalidation_1-logloss:0.609425\n",
      "[346]\tvalidation_0-error:0.309103\tvalidation_0-logloss:0.585838\tvalidation_1-error:0.334058\tvalidation_1-logloss:0.609408\n",
      "[347]\tvalidation_0-error:0.30907\tvalidation_0-logloss:0.585755\tvalidation_1-error:0.333994\tvalidation_1-logloss:0.609387\n",
      "[348]\tvalidation_0-error:0.308981\tvalidation_0-logloss:0.58568\tvalidation_1-error:0.333975\tvalidation_1-logloss:0.609323\n",
      "[349]\tvalidation_0-error:0.30896\tvalidation_0-logloss:0.585609\tvalidation_1-error:0.333897\tvalidation_1-logloss:0.609285\n",
      "[350]\tvalidation_0-error:0.308936\tvalidation_0-logloss:0.585549\tvalidation_1-error:0.333894\tvalidation_1-logloss:0.609256\n",
      "[351]\tvalidation_0-error:0.308876\tvalidation_0-logloss:0.585449\tvalidation_1-error:0.333903\tvalidation_1-logloss:0.609199\n",
      "[352]\tvalidation_0-error:0.308789\tvalidation_0-logloss:0.585362\tvalidation_1-error:0.33389\tvalidation_1-logloss:0.609189\n",
      "[353]\tvalidation_0-error:0.308753\tvalidation_0-logloss:0.585283\tvalidation_1-error:0.333829\tvalidation_1-logloss:0.609141\n",
      "[354]\tvalidation_0-error:0.308669\tvalidation_0-logloss:0.585221\tvalidation_1-error:0.333825\tvalidation_1-logloss:0.609109\n",
      "[355]\tvalidation_0-error:0.308549\tvalidation_0-logloss:0.585151\tvalidation_1-error:0.333795\tvalidation_1-logloss:0.609098\n",
      "[356]\tvalidation_0-error:0.308414\tvalidation_0-logloss:0.585113\tvalidation_1-error:0.333771\tvalidation_1-logloss:0.609074\n",
      "[357]\tvalidation_0-error:0.30836\tvalidation_0-logloss:0.585022\tvalidation_1-error:0.333732\tvalidation_1-logloss:0.609052\n",
      "[358]\tvalidation_0-error:0.308264\tvalidation_0-logloss:0.584956\tvalidation_1-error:0.333727\tvalidation_1-logloss:0.609047\n",
      "[359]\tvalidation_0-error:0.308243\tvalidation_0-logloss:0.58489\tvalidation_1-error:0.33372\tvalidation_1-logloss:0.609022\n",
      "[360]\tvalidation_0-error:0.308132\tvalidation_0-logloss:0.584819\tvalidation_1-error:0.333698\tvalidation_1-logloss:0.609011\n",
      "[361]\tvalidation_0-error:0.308111\tvalidation_0-logloss:0.584748\tvalidation_1-error:0.333671\tvalidation_1-logloss:0.608957\n",
      "[362]\tvalidation_0-error:0.308114\tvalidation_0-logloss:0.584676\tvalidation_1-error:0.333622\tvalidation_1-logloss:0.60893\n",
      "[363]\tvalidation_0-error:0.308057\tvalidation_0-logloss:0.584589\tvalidation_1-error:0.333675\tvalidation_1-logloss:0.607678\n",
      "[364]\tvalidation_0-error:0.308021\tvalidation_0-logloss:0.58453\tvalidation_1-error:0.33365\tvalidation_1-logloss:0.607658\n",
      "[365]\tvalidation_0-error:0.307961\tvalidation_0-logloss:0.584439\tvalidation_1-error:0.333603\tvalidation_1-logloss:0.607641\n",
      "[366]\tvalidation_0-error:0.307901\tvalidation_0-logloss:0.584348\tvalidation_1-error:0.333529\tvalidation_1-logloss:0.607599\n",
      "[367]\tvalidation_0-error:0.307874\tvalidation_0-logloss:0.584275\tvalidation_1-error:0.333572\tvalidation_1-logloss:0.607575\n",
      "[368]\tvalidation_0-error:0.307778\tvalidation_0-logloss:0.584192\tvalidation_1-error:0.333517\tvalidation_1-logloss:0.607541\n",
      "[369]\tvalidation_0-error:0.307661\tvalidation_0-logloss:0.584126\tvalidation_1-error:0.333447\tvalidation_1-logloss:0.607505\n",
      "[370]\tvalidation_0-error:0.30764\tvalidation_0-logloss:0.584037\tvalidation_1-error:0.33339\tvalidation_1-logloss:0.607467\n",
      "[371]\tvalidation_0-error:0.307544\tvalidation_0-logloss:0.583967\tvalidation_1-error:0.333356\tvalidation_1-logloss:0.607445\n",
      "[372]\tvalidation_0-error:0.307494\tvalidation_0-logloss:0.584019\tvalidation_1-error:0.333309\tvalidation_1-logloss:0.607492\n",
      "[373]\tvalidation_0-error:0.307449\tvalidation_0-logloss:0.583905\tvalidation_1-error:0.333258\tvalidation_1-logloss:0.607442\n",
      "[374]\tvalidation_0-error:0.307407\tvalidation_0-logloss:0.583853\tvalidation_1-error:0.333216\tvalidation_1-logloss:0.607404\n",
      "[375]\tvalidation_0-error:0.30738\tvalidation_0-logloss:0.583789\tvalidation_1-error:0.333215\tvalidation_1-logloss:0.607391\n",
      "[376]\tvalidation_0-error:0.307377\tvalidation_0-logloss:0.5837\tvalidation_1-error:0.333203\tvalidation_1-logloss:0.607324\n",
      "[377]\tvalidation_0-error:0.307338\tvalidation_0-logloss:0.583634\tvalidation_1-error:0.333198\tvalidation_1-logloss:0.607314\n",
      "[378]\tvalidation_0-error:0.307299\tvalidation_0-logloss:0.583571\tvalidation_1-error:0.333144\tvalidation_1-logloss:0.607287\n",
      "[379]\tvalidation_0-error:0.307116\tvalidation_0-logloss:0.583509\tvalidation_1-error:0.333002\tvalidation_1-logloss:0.607254\n",
      "[380]\tvalidation_0-error:0.307083\tvalidation_0-logloss:0.583448\tvalidation_1-error:0.332973\tvalidation_1-logloss:0.607207\n",
      "[381]\tvalidation_0-error:0.307035\tvalidation_0-logloss:0.583372\tvalidation_1-error:0.332971\tvalidation_1-logloss:0.60718\n",
      "[382]\tvalidation_0-error:0.306975\tvalidation_0-logloss:0.583295\tvalidation_1-error:0.33299\tvalidation_1-logloss:0.607165\n",
      "[383]\tvalidation_0-error:0.306855\tvalidation_0-logloss:0.583208\tvalidation_1-error:0.332947\tvalidation_1-logloss:0.607115\n",
      "[384]\tvalidation_0-error:0.306828\tvalidation_0-logloss:0.583149\tvalidation_1-error:0.332942\tvalidation_1-logloss:0.607069\n",
      "[385]\tvalidation_0-error:0.306762\tvalidation_0-logloss:0.583075\tvalidation_1-error:0.332881\tvalidation_1-logloss:0.607036\n",
      "[386]\tvalidation_0-error:0.306573\tvalidation_0-logloss:0.582997\tvalidation_1-error:0.3328\tvalidation_1-logloss:0.607004\n",
      "[387]\tvalidation_0-error:0.306468\tvalidation_0-logloss:0.58289\tvalidation_1-error:0.332733\tvalidation_1-logloss:0.606971\n",
      "[388]\tvalidation_0-error:0.306321\tvalidation_0-logloss:0.582835\tvalidation_1-error:0.332609\tvalidation_1-logloss:0.606938\n",
      "[389]\tvalidation_0-error:0.306207\tvalidation_0-logloss:0.582746\tvalidation_1-error:0.332612\tvalidation_1-logloss:0.606906\n",
      "[390]\tvalidation_0-error:0.306153\tvalidation_0-logloss:0.582681\tvalidation_1-error:0.332611\tvalidation_1-logloss:0.606893\n",
      "[391]\tvalidation_0-error:0.306129\tvalidation_0-logloss:0.582618\tvalidation_1-error:0.332572\tvalidation_1-logloss:0.606854\n",
      "[392]\tvalidation_0-error:0.306015\tvalidation_0-logloss:0.582551\tvalidation_1-error:0.332521\tvalidation_1-logloss:0.606811\n",
      "[393]\tvalidation_0-error:0.30592\tvalidation_0-logloss:0.5825\tvalidation_1-error:0.332513\tvalidation_1-logloss:0.606786\n",
      "[394]\tvalidation_0-error:0.30583\tvalidation_0-logloss:0.582406\tvalidation_1-error:0.332468\tvalidation_1-logloss:0.606764\n",
      "[395]\tvalidation_0-error:0.305698\tvalidation_0-logloss:0.582324\tvalidation_1-error:0.332445\tvalidation_1-logloss:0.606746\n",
      "[396]\tvalidation_0-error:0.305686\tvalidation_0-logloss:0.58226\tvalidation_1-error:0.332478\tvalidation_1-logloss:0.606744\n",
      "[397]\tvalidation_0-error:0.305536\tvalidation_0-logloss:0.582448\tvalidation_1-error:0.3324\tvalidation_1-logloss:0.606957\n",
      "[398]\tvalidation_0-error:0.305449\tvalidation_0-logloss:0.582386\tvalidation_1-error:0.332346\tvalidation_1-logloss:0.606935\n",
      "[399]\tvalidation_0-error:0.305383\tvalidation_0-logloss:0.58231\tvalidation_1-error:0.332301\tvalidation_1-logloss:0.606928\n",
      "[400]\tvalidation_0-error:0.305293\tvalidation_0-logloss:0.582251\tvalidation_1-error:0.332276\tvalidation_1-logloss:0.60689\n",
      "[401]\tvalidation_0-error:0.305245\tvalidation_0-logloss:0.582185\tvalidation_1-error:0.332249\tvalidation_1-logloss:0.606858\n",
      "[402]\tvalidation_0-error:0.305167\tvalidation_0-logloss:0.581874\tvalidation_1-error:0.332183\tvalidation_1-logloss:0.606713\n",
      "[403]\tvalidation_0-error:0.305086\tvalidation_0-logloss:0.582061\tvalidation_1-error:0.332143\tvalidation_1-logloss:0.606795\n",
      "[404]\tvalidation_0-error:0.305047\tvalidation_0-logloss:0.581739\tvalidation_1-error:0.332094\tvalidation_1-logloss:0.606626\n",
      "[405]\tvalidation_0-error:0.304975\tvalidation_0-logloss:0.581676\tvalidation_1-error:0.332118\tvalidation_1-logloss:0.606622\n",
      "[406]\tvalidation_0-error:0.304888\tvalidation_0-logloss:0.58162\tvalidation_1-error:0.332091\tvalidation_1-logloss:0.606597\n",
      "[407]\tvalidation_0-error:0.304804\tvalidation_0-logloss:0.581558\tvalidation_1-error:0.332032\tvalidation_1-logloss:0.606554\n",
      "[408]\tvalidation_0-error:0.30463\tvalidation_0-logloss:0.581489\tvalidation_1-error:0.331899\tvalidation_1-logloss:0.606511\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[409]\tvalidation_0-error:0.304522\tvalidation_0-logloss:0.581392\tvalidation_1-error:0.331855\tvalidation_1-logloss:0.606475\n",
      "[410]\tvalidation_0-error:0.304507\tvalidation_0-logloss:0.581362\tvalidation_1-error:0.331778\tvalidation_1-logloss:0.606425\n",
      "[411]\tvalidation_0-error:0.304445\tvalidation_0-logloss:0.5813\tvalidation_1-error:0.33174\tvalidation_1-logloss:0.606389\n",
      "[412]\tvalidation_0-error:0.304397\tvalidation_0-logloss:0.581244\tvalidation_1-error:0.331806\tvalidation_1-logloss:0.606405\n",
      "[413]\tvalidation_0-error:0.304343\tvalidation_0-logloss:0.581167\tvalidation_1-error:0.3318\tvalidation_1-logloss:0.606372\n",
      "[414]\tvalidation_0-error:0.304262\tvalidation_0-logloss:0.581104\tvalidation_1-error:0.331735\tvalidation_1-logloss:0.606326\n",
      "[415]\tvalidation_0-error:0.304217\tvalidation_0-logloss:0.581022\tvalidation_1-error:0.331772\tvalidation_1-logloss:0.60632\n",
      "[416]\tvalidation_0-error:0.30416\tvalidation_0-logloss:0.580949\tvalidation_1-error:0.331704\tvalidation_1-logloss:0.606288\n",
      "[417]\tvalidation_0-error:0.304136\tvalidation_0-logloss:0.580896\tvalidation_1-error:0.331651\tvalidation_1-logloss:0.60624\n",
      "[418]\tvalidation_0-error:0.304115\tvalidation_0-logloss:0.580828\tvalidation_1-error:0.331676\tvalidation_1-logloss:0.606213\n",
      "[419]\tvalidation_0-error:0.304061\tvalidation_0-logloss:0.580786\tvalidation_1-error:0.331666\tvalidation_1-logloss:0.606202\n",
      "[420]\tvalidation_0-error:0.304034\tvalidation_0-logloss:0.580692\tvalidation_1-error:0.331672\tvalidation_1-logloss:0.606191\n",
      "[421]\tvalidation_0-error:0.303959\tvalidation_0-logloss:0.580608\tvalidation_1-error:0.331645\tvalidation_1-logloss:0.606178\n",
      "[422]\tvalidation_0-error:0.303923\tvalidation_0-logloss:0.580559\tvalidation_1-error:0.33162\tvalidation_1-logloss:0.606151\n",
      "[423]\tvalidation_0-error:0.303908\tvalidation_0-logloss:0.580483\tvalidation_1-error:0.331613\tvalidation_1-logloss:0.606137\n",
      "[424]\tvalidation_0-error:0.303809\tvalidation_0-logloss:0.580406\tvalidation_1-error:0.331576\tvalidation_1-logloss:0.606092\n",
      "[425]\tvalidation_0-error:0.30374\tvalidation_0-logloss:0.580349\tvalidation_1-error:0.331567\tvalidation_1-logloss:0.606054\n",
      "[426]\tvalidation_0-error:0.30368\tvalidation_0-logloss:0.580286\tvalidation_1-error:0.331574\tvalidation_1-logloss:0.606029\n",
      "[427]\tvalidation_0-error:0.303605\tvalidation_0-logloss:0.580251\tvalidation_1-error:0.331545\tvalidation_1-logloss:0.606027\n",
      "[428]\tvalidation_0-error:0.30359\tvalidation_0-logloss:0.580195\tvalidation_1-error:0.331511\tvalidation_1-logloss:0.605993\n",
      "[429]\tvalidation_0-error:0.30356\tvalidation_0-logloss:0.580131\tvalidation_1-error:0.33153\tvalidation_1-logloss:0.605938\n",
      "[430]\tvalidation_0-error:0.303461\tvalidation_0-logloss:0.580054\tvalidation_1-error:0.331444\tvalidation_1-logloss:0.605895\n",
      "[431]\tvalidation_0-error:0.30332\tvalidation_0-logloss:0.579989\tvalidation_1-error:0.331425\tvalidation_1-logloss:0.605919\n",
      "[432]\tvalidation_0-error:0.303194\tvalidation_0-logloss:0.57993\tvalidation_1-error:0.33137\tvalidation_1-logloss:0.605836\n",
      "[433]\tvalidation_0-error:0.303152\tvalidation_0-logloss:0.579879\tvalidation_1-error:0.331362\tvalidation_1-logloss:0.605798\n",
      "[434]\tvalidation_0-error:0.30308\tvalidation_0-logloss:0.579883\tvalidation_1-error:0.331333\tvalidation_1-logloss:0.605819\n",
      "[435]\tvalidation_0-error:0.303026\tvalidation_0-logloss:0.57981\tvalidation_1-error:0.331363\tvalidation_1-logloss:0.605819\n",
      "[436]\tvalidation_0-error:0.302952\tvalidation_0-logloss:0.579737\tvalidation_1-error:0.331311\tvalidation_1-logloss:0.605747\n",
      "[437]\tvalidation_0-error:0.30285\tvalidation_0-logloss:0.579667\tvalidation_1-error:0.331261\tvalidation_1-logloss:0.605732\n",
      "[438]\tvalidation_0-error:0.302775\tvalidation_0-logloss:0.579597\tvalidation_1-error:0.331264\tvalidation_1-logloss:0.605713\n",
      "[439]\tvalidation_0-error:0.302838\tvalidation_0-logloss:0.579533\tvalidation_1-error:0.331257\tvalidation_1-logloss:0.605705\n",
      "[440]\tvalidation_0-error:0.302856\tvalidation_0-logloss:0.579466\tvalidation_1-error:0.331226\tvalidation_1-logloss:0.605649\n",
      "[441]\tvalidation_0-error:0.302862\tvalidation_0-logloss:0.579395\tvalidation_1-error:0.33124\tvalidation_1-logloss:0.605625\n",
      "[442]\tvalidation_0-error:0.302814\tvalidation_0-logloss:0.57936\tvalidation_1-error:0.331211\tvalidation_1-logloss:0.605616\n",
      "[443]\tvalidation_0-error:0.302754\tvalidation_0-logloss:0.579305\tvalidation_1-error:0.331207\tvalidation_1-logloss:0.605594\n",
      "[444]\tvalidation_0-error:0.3027\tvalidation_0-logloss:0.579244\tvalidation_1-error:0.331171\tvalidation_1-logloss:0.605576\n",
      "[445]\tvalidation_0-error:0.302688\tvalidation_0-logloss:0.579159\tvalidation_1-error:0.331178\tvalidation_1-logloss:0.605526\n",
      "[446]\tvalidation_0-error:0.30264\tvalidation_0-logloss:0.579096\tvalidation_1-error:0.331121\tvalidation_1-logloss:0.605493\n",
      "[447]\tvalidation_0-error:0.302577\tvalidation_0-logloss:0.579076\tvalidation_1-error:0.331046\tvalidation_1-logloss:0.605493\n",
      "[448]\tvalidation_0-error:0.302532\tvalidation_0-logloss:0.57901\tvalidation_1-error:0.330991\tvalidation_1-logloss:0.605463\n",
      "[449]\tvalidation_0-error:0.302475\tvalidation_0-logloss:0.578962\tvalidation_1-error:0.330969\tvalidation_1-logloss:0.605401\n",
      "[450]\tvalidation_0-error:0.302424\tvalidation_0-logloss:0.578896\tvalidation_1-error:0.330892\tvalidation_1-logloss:0.605376\n",
      "[451]\tvalidation_0-error:0.30237\tvalidation_0-logloss:0.578787\tvalidation_1-error:0.330892\tvalidation_1-logloss:0.605323\n",
      "[452]\tvalidation_0-error:0.302316\tvalidation_0-logloss:0.578289\tvalidation_1-error:0.33088\tvalidation_1-logloss:0.604809\n",
      "[453]\tvalidation_0-error:0.302244\tvalidation_0-logloss:0.578208\tvalidation_1-error:0.330752\tvalidation_1-logloss:0.604768\n",
      "[454]\tvalidation_0-error:0.302175\tvalidation_0-logloss:0.578159\tvalidation_1-error:0.3307\tvalidation_1-logloss:0.604752\n",
      "[455]\tvalidation_0-error:0.302148\tvalidation_0-logloss:0.578101\tvalidation_1-error:0.330672\tvalidation_1-logloss:0.604717\n",
      "[456]\tvalidation_0-error:0.302019\tvalidation_0-logloss:0.578039\tvalidation_1-error:0.33069\tvalidation_1-logloss:0.604695\n",
      "[457]\tvalidation_0-error:0.302022\tvalidation_0-logloss:0.577977\tvalidation_1-error:0.330642\tvalidation_1-logloss:0.604684\n",
      "[458]\tvalidation_0-error:0.301953\tvalidation_0-logloss:0.577913\tvalidation_1-error:0.330647\tvalidation_1-logloss:0.604666\n",
      "[459]\tvalidation_0-error:0.301911\tvalidation_0-logloss:0.577856\tvalidation_1-error:0.330607\tvalidation_1-logloss:0.604631\n",
      "[460]\tvalidation_0-error:0.301875\tvalidation_0-logloss:0.577802\tvalidation_1-error:0.330536\tvalidation_1-logloss:0.604615\n",
      "[461]\tvalidation_0-error:0.30183\tvalidation_0-logloss:0.577741\tvalidation_1-error:0.330514\tvalidation_1-logloss:0.604582\n",
      "[462]\tvalidation_0-error:0.301779\tvalidation_0-logloss:0.57768\tvalidation_1-error:0.330526\tvalidation_1-logloss:0.604547\n",
      "[463]\tvalidation_0-error:0.301782\tvalidation_0-logloss:0.577621\tvalidation_1-error:0.330588\tvalidation_1-logloss:0.605063\n",
      "[464]\tvalidation_0-error:0.301692\tvalidation_0-logloss:0.577562\tvalidation_1-error:0.330613\tvalidation_1-logloss:0.605081\n",
      "[465]\tvalidation_0-error:0.301692\tvalidation_0-logloss:0.577462\tvalidation_1-error:0.330642\tvalidation_1-logloss:0.605083\n",
      "[466]\tvalidation_0-error:0.301617\tvalidation_0-logloss:0.57741\tvalidation_1-error:0.330616\tvalidation_1-logloss:0.60507\n",
      "[467]\tvalidation_0-error:0.301554\tvalidation_0-logloss:0.577353\tvalidation_1-error:0.330585\tvalidation_1-logloss:0.605033\n",
      "[468]\tvalidation_0-error:0.301477\tvalidation_0-logloss:0.577304\tvalidation_1-error:0.330533\tvalidation_1-logloss:0.605019\n",
      "[469]\tvalidation_0-error:0.30142\tvalidation_0-logloss:0.577223\tvalidation_1-error:0.33052\tvalidation_1-logloss:0.604985\n",
      "[470]\tvalidation_0-error:0.30133\tvalidation_0-logloss:0.577168\tvalidation_1-error:0.330527\tvalidation_1-logloss:0.605005\n",
      "[471]\tvalidation_0-error:0.301243\tvalidation_0-logloss:0.577119\tvalidation_1-error:0.330505\tvalidation_1-logloss:0.604973\n",
      "[472]\tvalidation_0-error:0.301141\tvalidation_0-logloss:0.577077\tvalidation_1-error:0.330535\tvalidation_1-logloss:0.604952\n",
      "[473]\tvalidation_0-error:0.30112\tvalidation_0-logloss:0.577012\tvalidation_1-error:0.330546\tvalidation_1-logloss:0.604913\n",
      "[474]\tvalidation_0-error:0.301084\tvalidation_0-logloss:0.576924\tvalidation_1-error:0.330548\tvalidation_1-logloss:0.60487\n",
      "[475]\tvalidation_0-error:0.300985\tvalidation_0-logloss:0.576845\tvalidation_1-error:0.330526\tvalidation_1-logloss:0.604872\n",
      "[476]\tvalidation_0-error:0.300979\tvalidation_0-logloss:0.576804\tvalidation_1-error:0.33056\tvalidation_1-logloss:0.604861\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[477]\tvalidation_0-error:0.300895\tvalidation_0-logloss:0.57675\tvalidation_1-error:0.330501\tvalidation_1-logloss:0.604836\n",
      "[478]\tvalidation_0-error:0.300844\tvalidation_0-logloss:0.576701\tvalidation_1-error:0.330527\tvalidation_1-logloss:0.604829\n",
      "[479]\tvalidation_0-error:0.300847\tvalidation_0-logloss:0.576637\tvalidation_1-error:0.330515\tvalidation_1-logloss:0.604816\n",
      "[480]\tvalidation_0-error:0.300799\tvalidation_0-logloss:0.576558\tvalidation_1-error:0.330446\tvalidation_1-logloss:0.604787\n",
      "[481]\tvalidation_0-error:0.300814\tvalidation_0-logloss:0.576508\tvalidation_1-error:0.330411\tvalidation_1-logloss:0.604755\n",
      "[482]\tvalidation_0-error:0.300718\tvalidation_0-logloss:0.576558\tvalidation_1-error:0.330428\tvalidation_1-logloss:0.604791\n",
      "[483]\tvalidation_0-error:0.300598\tvalidation_0-logloss:0.576511\tvalidation_1-error:0.330409\tvalidation_1-logloss:0.604781\n",
      "[484]\tvalidation_0-error:0.300556\tvalidation_0-logloss:0.576453\tvalidation_1-error:0.330347\tvalidation_1-logloss:0.604738\n",
      "[485]\tvalidation_0-error:0.300511\tvalidation_0-logloss:0.576412\tvalidation_1-error:0.330321\tvalidation_1-logloss:0.604711\n",
      "[486]\tvalidation_0-error:0.300487\tvalidation_0-logloss:0.576136\tvalidation_1-error:0.330315\tvalidation_1-logloss:0.604592\n",
      "[487]\tvalidation_0-error:0.300457\tvalidation_0-logloss:0.576073\tvalidation_1-error:0.330334\tvalidation_1-logloss:0.604592\n",
      "[488]\tvalidation_0-error:0.300343\tvalidation_0-logloss:0.576\tvalidation_1-error:0.330344\tvalidation_1-logloss:0.604598\n",
      "[489]\tvalidation_0-error:0.300268\tvalidation_0-logloss:0.575963\tvalidation_1-error:0.330307\tvalidation_1-logloss:0.604578\n",
      "[490]\tvalidation_0-error:0.300241\tvalidation_0-logloss:0.575902\tvalidation_1-error:0.330281\tvalidation_1-logloss:0.604554\n",
      "[491]\tvalidation_0-error:0.300202\tvalidation_0-logloss:0.575843\tvalidation_1-error:0.33025\tvalidation_1-logloss:0.60452\n",
      "[492]\tvalidation_0-error:0.300121\tvalidation_0-logloss:0.575788\tvalidation_1-error:0.330233\tvalidation_1-logloss:0.604492\n",
      "[493]\tvalidation_0-error:0.300112\tvalidation_0-logloss:0.575737\tvalidation_1-error:0.330226\tvalidation_1-logloss:0.604478\n",
      "[494]\tvalidation_0-error:0.30004\tvalidation_0-logloss:0.575666\tvalidation_1-error:0.330168\tvalidation_1-logloss:0.604479\n",
      "[495]\tvalidation_0-error:0.299975\tvalidation_0-logloss:0.575606\tvalidation_1-error:0.330164\tvalidation_1-logloss:0.604462\n",
      "[496]\tvalidation_0-error:0.299882\tvalidation_0-logloss:0.575545\tvalidation_1-error:0.330145\tvalidation_1-logloss:0.604456\n",
      "[497]\tvalidation_0-error:0.299822\tvalidation_0-logloss:0.575498\tvalidation_1-error:0.330095\tvalidation_1-logloss:0.604422\n",
      "[498]\tvalidation_0-error:0.299729\tvalidation_0-logloss:0.575444\tvalidation_1-error:0.330108\tvalidation_1-logloss:0.604403\n",
      "[499]\tvalidation_0-error:0.299726\tvalidation_0-logloss:0.575419\tvalidation_1-error:0.330096\tvalidation_1-logloss:0.604389\n",
      "[500]\tvalidation_0-error:0.299654\tvalidation_0-logloss:0.575366\tvalidation_1-error:0.330112\tvalidation_1-logloss:0.60436\n",
      "[501]\tvalidation_0-error:0.299618\tvalidation_0-logloss:0.5753\tvalidation_1-error:0.330106\tvalidation_1-logloss:0.604321\n",
      "[502]\tvalidation_0-error:0.299543\tvalidation_0-logloss:0.575254\tvalidation_1-error:0.330043\tvalidation_1-logloss:0.604297\n",
      "[503]\tvalidation_0-error:0.299561\tvalidation_0-logloss:0.575202\tvalidation_1-error:0.330077\tvalidation_1-logloss:0.604308\n",
      "[504]\tvalidation_0-error:0.299504\tvalidation_0-logloss:0.575147\tvalidation_1-error:0.330075\tvalidation_1-logloss:0.604333\n",
      "[505]\tvalidation_0-error:0.299444\tvalidation_0-logloss:0.575059\tvalidation_1-error:0.330013\tvalidation_1-logloss:0.604325\n",
      "[506]\tvalidation_0-error:0.299411\tvalidation_0-logloss:0.575007\tvalidation_1-error:0.330003\tvalidation_1-logloss:0.604323\n",
      "[507]\tvalidation_0-error:0.299372\tvalidation_0-logloss:0.574951\tvalidation_1-error:0.329993\tvalidation_1-logloss:0.604314\n",
      "[508]\tvalidation_0-error:0.299309\tvalidation_0-logloss:0.574904\tvalidation_1-error:0.329959\tvalidation_1-logloss:0.604293\n",
      "[509]\tvalidation_0-error:0.299237\tvalidation_0-logloss:0.574856\tvalidation_1-error:0.329988\tvalidation_1-logloss:0.604298\n",
      "[510]\tvalidation_0-error:0.299189\tvalidation_0-logloss:0.574794\tvalidation_1-error:0.329948\tvalidation_1-logloss:0.604265\n",
      "[511]\tvalidation_0-error:0.299084\tvalidation_0-logloss:0.574737\tvalidation_1-error:0.329891\tvalidation_1-logloss:0.604208\n",
      "[512]\tvalidation_0-error:0.298985\tvalidation_0-logloss:0.574667\tvalidation_1-error:0.329824\tvalidation_1-logloss:0.604172\n",
      "[513]\tvalidation_0-error:0.299024\tvalidation_0-logloss:0.574608\tvalidation_1-error:0.329802\tvalidation_1-logloss:0.604139\n",
      "[514]\tvalidation_0-error:0.298919\tvalidation_0-logloss:0.57454\tvalidation_1-error:0.329764\tvalidation_1-logloss:0.604113\n",
      "[515]\tvalidation_0-error:0.298853\tvalidation_0-logloss:0.574491\tvalidation_1-error:0.329749\tvalidation_1-logloss:0.604086\n",
      "[516]\tvalidation_0-error:0.298823\tvalidation_0-logloss:0.574441\tvalidation_1-error:0.329733\tvalidation_1-logloss:0.604066\n",
      "[517]\tvalidation_0-error:0.298787\tvalidation_0-logloss:0.574389\tvalidation_1-error:0.329764\tvalidation_1-logloss:0.604065\n",
      "[518]\tvalidation_0-error:0.298721\tvalidation_0-logloss:0.574549\tvalidation_1-error:0.329739\tvalidation_1-logloss:0.604255\n",
      "[519]\tvalidation_0-error:0.298712\tvalidation_0-logloss:0.574281\tvalidation_1-error:0.32972\tvalidation_1-logloss:0.604011\n",
      "[520]\tvalidation_0-error:0.298688\tvalidation_0-logloss:0.57438\tvalidation_1-error:0.329774\tvalidation_1-logloss:0.604215\n",
      "[521]\tvalidation_0-error:0.298646\tvalidation_0-logloss:0.574326\tvalidation_1-error:0.329758\tvalidation_1-logloss:0.604199\n",
      "[522]\tvalidation_0-error:0.298652\tvalidation_0-logloss:0.574256\tvalidation_1-error:0.329759\tvalidation_1-logloss:0.604185\n",
      "[523]\tvalidation_0-error:0.298616\tvalidation_0-logloss:0.574203\tvalidation_1-error:0.329771\tvalidation_1-logloss:0.604178\n",
      "[524]\tvalidation_0-error:0.298583\tvalidation_0-logloss:0.574141\tvalidation_1-error:0.329768\tvalidation_1-logloss:0.604183\n",
      "[525]\tvalidation_0-error:0.298505\tvalidation_0-logloss:0.574087\tvalidation_1-error:0.329748\tvalidation_1-logloss:0.604151\n",
      "[526]\tvalidation_0-error:0.29838\tvalidation_0-logloss:0.574029\tvalidation_1-error:0.329687\tvalidation_1-logloss:0.604126\n",
      "[527]\tvalidation_0-error:0.298314\tvalidation_0-logloss:0.573975\tvalidation_1-error:0.329606\tvalidation_1-logloss:0.604099\n",
      "[528]\tvalidation_0-error:0.298203\tvalidation_0-logloss:0.573922\tvalidation_1-error:0.329569\tvalidation_1-logloss:0.604079\n",
      "[529]\tvalidation_0-error:0.298188\tvalidation_0-logloss:0.573867\tvalidation_1-error:0.329572\tvalidation_1-logloss:0.60407\n",
      "[530]\tvalidation_0-error:0.298089\tvalidation_0-logloss:0.573817\tvalidation_1-error:0.329516\tvalidation_1-logloss:0.604033\n",
      "[531]\tvalidation_0-error:0.297963\tvalidation_0-logloss:0.573776\tvalidation_1-error:0.329421\tvalidation_1-logloss:0.604031\n",
      "[532]\tvalidation_0-error:0.297951\tvalidation_0-logloss:0.573736\tvalidation_1-error:0.329393\tvalidation_1-logloss:0.604013\n",
      "[533]\tvalidation_0-error:0.297933\tvalidation_0-logloss:0.573685\tvalidation_1-error:0.329359\tvalidation_1-logloss:0.60398\n",
      "[534]\tvalidation_0-error:0.297873\tvalidation_0-logloss:0.573635\tvalidation_1-error:0.329263\tvalidation_1-logloss:0.603945\n",
      "[535]\tvalidation_0-error:0.297768\tvalidation_0-logloss:0.573602\tvalidation_1-error:0.329243\tvalidation_1-logloss:0.60395\n",
      "[536]\tvalidation_0-error:0.29772\tvalidation_0-logloss:0.573545\tvalidation_1-error:0.329194\tvalidation_1-logloss:0.603944\n",
      "[537]\tvalidation_0-error:0.29769\tvalidation_0-logloss:0.573468\tvalidation_1-error:0.329204\tvalidation_1-logloss:0.60391\n",
      "[538]\tvalidation_0-error:0.297666\tvalidation_0-logloss:0.573435\tvalidation_1-error:0.32911\tvalidation_1-logloss:0.6039\n",
      "[539]\tvalidation_0-error:0.297603\tvalidation_0-logloss:0.573384\tvalidation_1-error:0.329068\tvalidation_1-logloss:0.603872\n",
      "[540]\tvalidation_0-error:0.297546\tvalidation_0-logloss:0.573339\tvalidation_1-error:0.329062\tvalidation_1-logloss:0.60384\n",
      "[541]\tvalidation_0-error:0.297543\tvalidation_0-logloss:0.573291\tvalidation_1-error:0.329101\tvalidation_1-logloss:0.603847\n",
      "[542]\tvalidation_0-error:0.297516\tvalidation_0-logloss:0.573249\tvalidation_1-error:0.329092\tvalidation_1-logloss:0.603821\n",
      "[543]\tvalidation_0-error:0.297441\tvalidation_0-logloss:0.573199\tvalidation_1-error:0.329092\tvalidation_1-logloss:0.603811\n",
      "[544]\tvalidation_0-error:0.297429\tvalidation_0-logloss:0.573141\tvalidation_1-error:0.329079\tvalidation_1-logloss:0.603785\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[545]\tvalidation_0-error:0.297402\tvalidation_0-logloss:0.573087\tvalidation_1-error:0.329065\tvalidation_1-logloss:0.603765\n",
      "[546]\tvalidation_0-error:0.297309\tvalidation_0-logloss:0.57303\tvalidation_1-error:0.329018\tvalidation_1-logloss:0.603743\n",
      "[547]\tvalidation_0-error:0.297279\tvalidation_0-logloss:0.572981\tvalidation_1-error:0.328995\tvalidation_1-logloss:0.603709\n",
      "[548]\tvalidation_0-error:0.297219\tvalidation_0-logloss:0.572926\tvalidation_1-error:0.329008\tvalidation_1-logloss:0.603688\n",
      "[549]\tvalidation_0-error:0.297156\tvalidation_0-logloss:0.572875\tvalidation_1-error:0.329002\tvalidation_1-logloss:0.603685\n",
      "[550]\tvalidation_0-error:0.297054\tvalidation_0-logloss:0.572824\tvalidation_1-error:0.328941\tvalidation_1-logloss:0.603675\n",
      "[551]\tvalidation_0-error:0.296974\tvalidation_0-logloss:0.572748\tvalidation_1-error:0.328899\tvalidation_1-logloss:0.603671\n",
      "[552]\tvalidation_0-error:0.296935\tvalidation_0-logloss:0.572702\tvalidation_1-error:0.328925\tvalidation_1-logloss:0.603645\n",
      "[553]\tvalidation_0-error:0.296887\tvalidation_0-logloss:0.572658\tvalidation_1-error:0.328949\tvalidation_1-logloss:0.603633\n",
      "[554]\tvalidation_0-error:0.296881\tvalidation_0-logloss:0.572603\tvalidation_1-error:0.328881\tvalidation_1-logloss:0.603616\n",
      "[555]\tvalidation_0-error:0.296842\tvalidation_0-logloss:0.572559\tvalidation_1-error:0.328882\tvalidation_1-logloss:0.603597\n",
      "[556]\tvalidation_0-error:0.296863\tvalidation_0-logloss:0.572513\tvalidation_1-error:0.328896\tvalidation_1-logloss:0.603569\n",
      "[557]\tvalidation_0-error:0.296737\tvalidation_0-logloss:0.572564\tvalidation_1-error:0.328829\tvalidation_1-logloss:0.603587\n",
      "[558]\tvalidation_0-error:0.296632\tvalidation_0-logloss:0.572498\tvalidation_1-error:0.328841\tvalidation_1-logloss:0.603567\n",
      "[559]\tvalidation_0-error:0.296599\tvalidation_0-logloss:0.572448\tvalidation_1-error:0.328817\tvalidation_1-logloss:0.603518\n",
      "[560]\tvalidation_0-error:0.296569\tvalidation_0-logloss:0.572286\tvalidation_1-error:0.328845\tvalidation_1-logloss:0.603459\n",
      "[561]\tvalidation_0-error:0.296551\tvalidation_0-logloss:0.572224\tvalidation_1-error:0.328828\tvalidation_1-logloss:0.603439\n",
      "[562]\tvalidation_0-error:0.296479\tvalidation_0-logloss:0.572274\tvalidation_1-error:0.328785\tvalidation_1-logloss:0.603465\n",
      "[563]\tvalidation_0-error:0.29644\tvalidation_0-logloss:0.57222\tvalidation_1-error:0.328751\tvalidation_1-logloss:0.603438\n",
      "[564]\tvalidation_0-error:0.296338\tvalidation_0-logloss:0.572173\tvalidation_1-error:0.328671\tvalidation_1-logloss:0.603405\n",
      "[565]\tvalidation_0-error:0.296293\tvalidation_0-logloss:0.572132\tvalidation_1-error:0.328692\tvalidation_1-logloss:0.60341\n",
      "[566]\tvalidation_0-error:0.296254\tvalidation_0-logloss:0.572066\tvalidation_1-error:0.328679\tvalidation_1-logloss:0.603387\n",
      "[567]\tvalidation_0-error:0.296308\tvalidation_0-logloss:0.572023\tvalidation_1-error:0.328705\tvalidation_1-logloss:0.60339\n",
      "[568]\tvalidation_0-error:0.29626\tvalidation_0-logloss:0.571967\tvalidation_1-error:0.328718\tvalidation_1-logloss:0.603367\n",
      "[569]\tvalidation_0-error:0.296236\tvalidation_0-logloss:0.571926\tvalidation_1-error:0.328671\tvalidation_1-logloss:0.603345\n",
      "[570]\tvalidation_0-error:0.296143\tvalidation_0-logloss:0.571869\tvalidation_1-error:0.328646\tvalidation_1-logloss:0.603308\n",
      "[571]\tvalidation_0-error:0.296095\tvalidation_0-logloss:0.571834\tvalidation_1-error:0.328714\tvalidation_1-logloss:0.60334\n",
      "[572]\tvalidation_0-error:0.296065\tvalidation_0-logloss:0.571781\tvalidation_1-error:0.328714\tvalidation_1-logloss:0.603326\n",
      "[573]\tvalidation_0-error:0.296047\tvalidation_0-logloss:0.571689\tvalidation_1-error:0.328695\tvalidation_1-logloss:0.603265\n",
      "[574]\tvalidation_0-error:0.295975\tvalidation_0-logloss:0.571664\tvalidation_1-error:0.32867\tvalidation_1-logloss:0.603269\n",
      "[575]\tvalidation_0-error:0.295999\tvalidation_0-logloss:0.571552\tvalidation_1-error:0.328622\tvalidation_1-logloss:0.603215\n",
      "[576]\tvalidation_0-error:0.295954\tvalidation_0-logloss:0.571513\tvalidation_1-error:0.328653\tvalidation_1-logloss:0.603206\n",
      "[577]\tvalidation_0-error:0.295873\tvalidation_0-logloss:0.571449\tvalidation_1-error:0.328645\tvalidation_1-logloss:0.603191\n",
      "[578]\tvalidation_0-error:0.295825\tvalidation_0-logloss:0.571406\tvalidation_1-error:0.328652\tvalidation_1-logloss:0.603184\n",
      "[579]\tvalidation_0-error:0.295762\tvalidation_0-logloss:0.571358\tvalidation_1-error:0.328608\tvalidation_1-logloss:0.603171\n",
      "[580]\tvalidation_0-error:0.295699\tvalidation_0-logloss:0.571302\tvalidation_1-error:0.328634\tvalidation_1-logloss:0.603173\n",
      "[581]\tvalidation_0-error:0.295645\tvalidation_0-logloss:0.57125\tvalidation_1-error:0.328622\tvalidation_1-logloss:0.603161\n",
      "[582]\tvalidation_0-error:0.295582\tvalidation_0-logloss:0.571201\tvalidation_1-error:0.328535\tvalidation_1-logloss:0.603153\n",
      "[583]\tvalidation_0-error:0.295469\tvalidation_0-logloss:0.571144\tvalidation_1-error:0.328546\tvalidation_1-logloss:0.603144\n",
      "[584]\tvalidation_0-error:0.295412\tvalidation_0-logloss:0.571098\tvalidation_1-error:0.328543\tvalidation_1-logloss:0.60312\n",
      "[585]\tvalidation_0-error:0.295358\tvalidation_0-logloss:0.57105\tvalidation_1-error:0.328513\tvalidation_1-logloss:0.603106\n",
      "[586]\tvalidation_0-error:0.295388\tvalidation_0-logloss:0.571011\tvalidation_1-error:0.328485\tvalidation_1-logloss:0.603084\n",
      "[587]\tvalidation_0-error:0.295277\tvalidation_0-logloss:0.570954\tvalidation_1-error:0.328433\tvalidation_1-logloss:0.603053\n",
      "[588]\tvalidation_0-error:0.29519\tvalidation_0-logloss:0.570893\tvalidation_1-error:0.328435\tvalidation_1-logloss:0.603035\n",
      "[589]\tvalidation_0-error:0.29519\tvalidation_0-logloss:0.570829\tvalidation_1-error:0.328473\tvalidation_1-logloss:0.603016\n",
      "[590]\tvalidation_0-error:0.295142\tvalidation_0-logloss:0.570781\tvalidation_1-error:0.328466\tvalidation_1-logloss:0.603006\n",
      "[591]\tvalidation_0-error:0.295055\tvalidation_0-logloss:0.57073\tvalidation_1-error:0.328478\tvalidation_1-logloss:0.603036\n",
      "[592]\tvalidation_0-error:0.294959\tvalidation_0-logloss:0.570664\tvalidation_1-error:0.328515\tvalidation_1-logloss:0.603038\n",
      "[593]\tvalidation_0-error:0.294887\tvalidation_0-logloss:0.570604\tvalidation_1-error:0.328473\tvalidation_1-logloss:0.602978\n",
      "[594]\tvalidation_0-error:0.294905\tvalidation_0-logloss:0.570555\tvalidation_1-error:0.328451\tvalidation_1-logloss:0.602948\n",
      "[595]\tvalidation_0-error:0.294869\tvalidation_0-logloss:0.570517\tvalidation_1-error:0.328432\tvalidation_1-logloss:0.602938\n",
      "[596]\tvalidation_0-error:0.294851\tvalidation_0-logloss:0.57046\tvalidation_1-error:0.328385\tvalidation_1-logloss:0.602924\n",
      "[597]\tvalidation_0-error:0.294821\tvalidation_0-logloss:0.570417\tvalidation_1-error:0.328349\tvalidation_1-logloss:0.602908\n",
      "[598]\tvalidation_0-error:0.294821\tvalidation_0-logloss:0.570381\tvalidation_1-error:0.328351\tvalidation_1-logloss:0.602903\n",
      "[599]\tvalidation_0-error:0.294803\tvalidation_0-logloss:0.570338\tvalidation_1-error:0.328368\tvalidation_1-logloss:0.602894\n",
      "[600]\tvalidation_0-error:0.294815\tvalidation_0-logloss:0.570085\tvalidation_1-error:0.328362\tvalidation_1-logloss:0.602773\n",
      "[601]\tvalidation_0-error:0.294803\tvalidation_0-logloss:0.570028\tvalidation_1-error:0.328343\tvalidation_1-logloss:0.60275\n",
      "[602]\tvalidation_0-error:0.294698\tvalidation_0-logloss:0.569958\tvalidation_1-error:0.328324\tvalidation_1-logloss:0.602735\n",
      "[603]\tvalidation_0-error:0.294644\tvalidation_0-logloss:0.569915\tvalidation_1-error:0.32828\tvalidation_1-logloss:0.602713\n",
      "[604]\tvalidation_0-error:0.294659\tvalidation_0-logloss:0.569861\tvalidation_1-error:0.32828\tvalidation_1-logloss:0.602715\n",
      "[605]\tvalidation_0-error:0.294557\tvalidation_0-logloss:0.569808\tvalidation_1-error:0.328302\tvalidation_1-logloss:0.602696\n",
      "[606]\tvalidation_0-error:0.294473\tvalidation_0-logloss:0.56976\tvalidation_1-error:0.328342\tvalidation_1-logloss:0.602718\n",
      "[607]\tvalidation_0-error:0.29441\tvalidation_0-logloss:0.569723\tvalidation_1-error:0.32834\tvalidation_1-logloss:0.60272\n",
      "[608]\tvalidation_0-error:0.294443\tvalidation_0-logloss:0.56966\tvalidation_1-error:0.328309\tvalidation_1-logloss:0.602673\n",
      "[609]\tvalidation_0-error:0.294398\tvalidation_0-logloss:0.569613\tvalidation_1-error:0.328292\tvalidation_1-logloss:0.602644\n",
      "[610]\tvalidation_0-error:0.294305\tvalidation_0-logloss:0.569564\tvalidation_1-error:0.328237\tvalidation_1-logloss:0.602629\n",
      "[611]\tvalidation_0-error:0.294287\tvalidation_0-logloss:0.569512\tvalidation_1-error:0.328253\tvalidation_1-logloss:0.602611\n",
      "[612]\tvalidation_0-error:0.294191\tvalidation_0-logloss:0.56947\tvalidation_1-error:0.328269\tvalidation_1-logloss:0.602602\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[613]\tvalidation_0-error:0.294191\tvalidation_0-logloss:0.569424\tvalidation_1-error:0.328236\tvalidation_1-logloss:0.602562\n",
      "[614]\tvalidation_0-error:0.294155\tvalidation_0-logloss:0.569382\tvalidation_1-error:0.328206\tvalidation_1-logloss:0.602538\n",
      "[615]\tvalidation_0-error:0.294185\tvalidation_0-logloss:0.569339\tvalidation_1-error:0.328171\tvalidation_1-logloss:0.602508\n",
      "[616]\tvalidation_0-error:0.29411\tvalidation_0-logloss:0.569291\tvalidation_1-error:0.328145\tvalidation_1-logloss:0.602484\n",
      "[617]\tvalidation_0-error:0.294041\tvalidation_0-logloss:0.569234\tvalidation_1-error:0.328116\tvalidation_1-logloss:0.602462\n",
      "[618]\tvalidation_0-error:0.293985\tvalidation_0-logloss:0.569188\tvalidation_1-error:0.328076\tvalidation_1-logloss:0.60244\n",
      "[619]\tvalidation_0-error:0.293967\tvalidation_0-logloss:0.569139\tvalidation_1-error:0.328055\tvalidation_1-logloss:0.602416\n",
      "[620]\tvalidation_0-error:0.293913\tvalidation_0-logloss:0.569091\tvalidation_1-error:0.328039\tvalidation_1-logloss:0.602402\n",
      "[621]\tvalidation_0-error:0.293919\tvalidation_0-logloss:0.569051\tvalidation_1-error:0.328042\tvalidation_1-logloss:0.60239\n",
      "[622]\tvalidation_0-error:0.293874\tvalidation_0-logloss:0.569007\tvalidation_1-error:0.328042\tvalidation_1-logloss:0.602374\n",
      "[623]\tvalidation_0-error:0.293808\tvalidation_0-logloss:0.568969\tvalidation_1-error:0.328004\tvalidation_1-logloss:0.602363\n",
      "[624]\tvalidation_0-error:0.293751\tvalidation_0-logloss:0.568923\tvalidation_1-error:0.327964\tvalidation_1-logloss:0.602333\n",
      "[625]\tvalidation_0-error:0.293727\tvalidation_0-logloss:0.568875\tvalidation_1-error:0.327952\tvalidation_1-logloss:0.602312\n",
      "[626]\tvalidation_0-error:0.293682\tvalidation_0-logloss:0.568828\tvalidation_1-error:0.327979\tvalidation_1-logloss:0.602291\n",
      "[627]\tvalidation_0-error:0.293634\tvalidation_0-logloss:0.568788\tvalidation_1-error:0.327967\tvalidation_1-logloss:0.602284\n",
      "[628]\tvalidation_0-error:0.293646\tvalidation_0-logloss:0.568751\tvalidation_1-error:0.327962\tvalidation_1-logloss:0.602261\n",
      "[629]\tvalidation_0-error:0.293502\tvalidation_0-logloss:0.568692\tvalidation_1-error:0.327878\tvalidation_1-logloss:0.602242\n",
      "[630]\tvalidation_0-error:0.293511\tvalidation_0-logloss:0.568753\tvalidation_1-error:0.327868\tvalidation_1-logloss:0.602288\n",
      "[631]\tvalidation_0-error:0.293424\tvalidation_0-logloss:0.568701\tvalidation_1-error:0.327857\tvalidation_1-logloss:0.602284\n",
      "[632]\tvalidation_0-error:0.293394\tvalidation_0-logloss:0.568671\tvalidation_1-error:0.327835\tvalidation_1-logloss:0.602243\n",
      "[633]\tvalidation_0-error:0.293301\tvalidation_0-logloss:0.568617\tvalidation_1-error:0.327807\tvalidation_1-logloss:0.60223\n",
      "[634]\tvalidation_0-error:0.29325\tvalidation_0-logloss:0.56857\tvalidation_1-error:0.327818\tvalidation_1-logloss:0.602213\n",
      "[635]\tvalidation_0-error:0.293187\tvalidation_0-logloss:0.568459\tvalidation_1-error:0.327813\tvalidation_1-logloss:0.602175\n",
      "[636]\tvalidation_0-error:0.293181\tvalidation_0-logloss:0.568407\tvalidation_1-error:0.32779\tvalidation_1-logloss:0.602168\n",
      "[637]\tvalidation_0-error:0.293133\tvalidation_0-logloss:0.568363\tvalidation_1-error:0.327829\tvalidation_1-logloss:0.60217\n",
      "[638]\tvalidation_0-error:0.293055\tvalidation_0-logloss:0.568326\tvalidation_1-error:0.3278\tvalidation_1-logloss:0.602149\n",
      "[639]\tvalidation_0-error:0.293046\tvalidation_0-logloss:0.568289\tvalidation_1-error:0.327803\tvalidation_1-logloss:0.602143\n",
      "[640]\tvalidation_0-error:0.293079\tvalidation_0-logloss:0.568249\tvalidation_1-error:0.32777\tvalidation_1-logloss:0.602145\n",
      "[641]\tvalidation_0-error:0.293028\tvalidation_0-logloss:0.568204\tvalidation_1-error:0.327781\tvalidation_1-logloss:0.602124\n",
      "[642]\tvalidation_0-error:0.292992\tvalidation_0-logloss:0.568169\tvalidation_1-error:0.327714\tvalidation_1-logloss:0.602098\n",
      "[643]\tvalidation_0-error:0.292902\tvalidation_0-logloss:0.568121\tvalidation_1-error:0.327666\tvalidation_1-logloss:0.602063\n",
      "[644]\tvalidation_0-error:0.292824\tvalidation_0-logloss:0.568076\tvalidation_1-error:0.32771\tvalidation_1-logloss:0.602049\n",
      "[645]\tvalidation_0-error:0.292836\tvalidation_0-logloss:0.568036\tvalidation_1-error:0.327713\tvalidation_1-logloss:0.602035\n",
      "[646]\tvalidation_0-error:0.292764\tvalidation_0-logloss:0.567997\tvalidation_1-error:0.327697\tvalidation_1-logloss:0.60201\n",
      "[647]\tvalidation_0-error:0.292731\tvalidation_0-logloss:0.567962\tvalidation_1-error:0.327692\tvalidation_1-logloss:0.602018\n",
      "[648]\tvalidation_0-error:0.292623\tvalidation_0-logloss:0.567905\tvalidation_1-error:0.327702\tvalidation_1-logloss:0.601987\n",
      "[649]\tvalidation_0-error:0.292656\tvalidation_0-logloss:0.56787\tvalidation_1-error:0.327664\tvalidation_1-logloss:0.601987\n",
      "[650]\tvalidation_0-error:0.292587\tvalidation_0-logloss:0.567825\tvalidation_1-error:0.327658\tvalidation_1-logloss:0.601968\n",
      "[651]\tvalidation_0-error:0.29256\tvalidation_0-logloss:0.567778\tvalidation_1-error:0.327652\tvalidation_1-logloss:0.601962\n",
      "[652]\tvalidation_0-error:0.292551\tvalidation_0-logloss:0.567741\tvalidation_1-error:0.327682\tvalidation_1-logloss:0.601947\n",
      "[653]\tvalidation_0-error:0.292509\tvalidation_0-logloss:0.567693\tvalidation_1-error:0.327671\tvalidation_1-logloss:0.601963\n",
      "[654]\tvalidation_0-error:0.2925\tvalidation_0-logloss:0.567656\tvalidation_1-error:0.327657\tvalidation_1-logloss:0.601947\n",
      "[655]\tvalidation_0-error:0.292512\tvalidation_0-logloss:0.56761\tvalidation_1-error:0.327682\tvalidation_1-logloss:0.601923\n",
      "[656]\tvalidation_0-error:0.292465\tvalidation_0-logloss:0.567568\tvalidation_1-error:0.327602\tvalidation_1-logloss:0.601889\n",
      "[657]\tvalidation_0-error:0.292417\tvalidation_0-logloss:0.56751\tvalidation_1-error:0.327621\tvalidation_1-logloss:0.60187\n",
      "[658]\tvalidation_0-error:0.292315\tvalidation_0-logloss:0.567464\tvalidation_1-error:0.32754\tvalidation_1-logloss:0.601847\n",
      "[659]\tvalidation_0-error:0.292273\tvalidation_0-logloss:0.567426\tvalidation_1-error:0.327543\tvalidation_1-logloss:0.601857\n",
      "[660]\tvalidation_0-error:0.292324\tvalidation_0-logloss:0.567391\tvalidation_1-error:0.327533\tvalidation_1-logloss:0.601845\n",
      "[661]\tvalidation_0-error:0.292285\tvalidation_0-logloss:0.567003\tvalidation_1-error:0.32759\tvalidation_1-logloss:0.601463\n",
      "[662]\tvalidation_0-error:0.292273\tvalidation_0-logloss:0.566949\tvalidation_1-error:0.327556\tvalidation_1-logloss:0.601423\n",
      "[663]\tvalidation_0-error:0.292267\tvalidation_0-logloss:0.567082\tvalidation_1-error:0.327553\tvalidation_1-logloss:0.601595\n",
      "[664]\tvalidation_0-error:0.29221\tvalidation_0-logloss:0.567035\tvalidation_1-error:0.327561\tvalidation_1-logloss:0.601585\n",
      "[665]\tvalidation_0-error:0.292162\tvalidation_0-logloss:0.566992\tvalidation_1-error:0.327515\tvalidation_1-logloss:0.601575\n",
      "[666]\tvalidation_0-error:0.292108\tvalidation_0-logloss:0.566948\tvalidation_1-error:0.327496\tvalidation_1-logloss:0.601573\n",
      "[667]\tvalidation_0-error:0.292084\tvalidation_0-logloss:0.566884\tvalidation_1-error:0.327477\tvalidation_1-logloss:0.601539\n",
      "[668]\tvalidation_0-error:0.29206\tvalidation_0-logloss:0.56685\tvalidation_1-error:0.327468\tvalidation_1-logloss:0.601526\n",
      "[669]\tvalidation_0-error:0.291973\tvalidation_0-logloss:0.566822\tvalidation_1-error:0.327443\tvalidation_1-logloss:0.601534\n",
      "[670]\tvalidation_0-error:0.29194\tvalidation_0-logloss:0.566773\tvalidation_1-error:0.327386\tvalidation_1-logloss:0.601497\n",
      "[671]\tvalidation_0-error:0.291904\tvalidation_0-logloss:0.566734\tvalidation_1-error:0.327389\tvalidation_1-logloss:0.601488\n",
      "[672]\tvalidation_0-error:0.29188\tvalidation_0-logloss:0.566696\tvalidation_1-error:0.327358\tvalidation_1-logloss:0.601474\n",
      "[673]\tvalidation_0-error:0.291847\tvalidation_0-logloss:0.566657\tvalidation_1-error:0.327347\tvalidation_1-logloss:0.601482\n",
      "[674]\tvalidation_0-error:0.291796\tvalidation_0-logloss:0.566622\tvalidation_1-error:0.327381\tvalidation_1-logloss:0.60148\n",
      "[675]\tvalidation_0-error:0.291757\tvalidation_0-logloss:0.56657\tvalidation_1-error:0.327389\tvalidation_1-logloss:0.601457\n",
      "[676]\tvalidation_0-error:0.291718\tvalidation_0-logloss:0.566548\tvalidation_1-error:0.327385\tvalidation_1-logloss:0.601443\n",
      "[677]\tvalidation_0-error:0.291724\tvalidation_0-logloss:0.566497\tvalidation_1-error:0.32736\tvalidation_1-logloss:0.601412\n",
      "[678]\tvalidation_0-error:0.291709\tvalidation_0-logloss:0.566447\tvalidation_1-error:0.327326\tvalidation_1-logloss:0.601382\n",
      "[679]\tvalidation_0-error:0.291613\tvalidation_0-logloss:0.566399\tvalidation_1-error:0.327314\tvalidation_1-logloss:0.601365\n",
      "[680]\tvalidation_0-error:0.291583\tvalidation_0-logloss:0.566363\tvalidation_1-error:0.327307\tvalidation_1-logloss:0.60135\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[681]\tvalidation_0-error:0.291535\tvalidation_0-logloss:0.56633\tvalidation_1-error:0.327286\tvalidation_1-logloss:0.601341\n",
      "[682]\tvalidation_0-error:0.291499\tvalidation_0-logloss:0.56628\tvalidation_1-error:0.327286\tvalidation_1-logloss:0.60135\n",
      "[683]\tvalidation_0-error:0.29143\tvalidation_0-logloss:0.566238\tvalidation_1-error:0.327293\tvalidation_1-logloss:0.601331\n",
      "[684]\tvalidation_0-error:0.291439\tvalidation_0-logloss:0.5662\tvalidation_1-error:0.327254\tvalidation_1-logloss:0.60132\n",
      "[685]\tvalidation_0-error:0.291424\tvalidation_0-logloss:0.566156\tvalidation_1-error:0.327276\tvalidation_1-logloss:0.601317\n",
      "[686]\tvalidation_0-error:0.291364\tvalidation_0-logloss:0.566115\tvalidation_1-error:0.327208\tvalidation_1-logloss:0.601307\n",
      "[687]\tvalidation_0-error:0.291325\tvalidation_0-logloss:0.566085\tvalidation_1-error:0.327199\tvalidation_1-logloss:0.601286\n",
      "[688]\tvalidation_0-error:0.291286\tvalidation_0-logloss:0.56603\tvalidation_1-error:0.327166\tvalidation_1-logloss:0.601268\n",
      "[689]\tvalidation_0-error:0.291274\tvalidation_0-logloss:0.565994\tvalidation_1-error:0.327178\tvalidation_1-logloss:0.601267\n",
      "[690]\tvalidation_0-error:0.291208\tvalidation_0-logloss:0.565945\tvalidation_1-error:0.327153\tvalidation_1-logloss:0.601261\n",
      "[691]\tvalidation_0-error:0.291187\tvalidation_0-logloss:0.565894\tvalidation_1-error:0.327166\tvalidation_1-logloss:0.601237\n",
      "[692]\tvalidation_0-error:0.291178\tvalidation_0-logloss:0.565877\tvalidation_1-error:0.32718\tvalidation_1-logloss:0.601236\n",
      "[693]\tvalidation_0-error:0.291142\tvalidation_0-logloss:0.565839\tvalidation_1-error:0.327113\tvalidation_1-logloss:0.601231\n",
      "[694]\tvalidation_0-error:0.291124\tvalidation_0-logloss:0.565801\tvalidation_1-error:0.327128\tvalidation_1-logloss:0.60122\n",
      "[695]\tvalidation_0-error:0.291088\tvalidation_0-logloss:0.565709\tvalidation_1-error:0.3271\tvalidation_1-logloss:0.601182\n",
      "[696]\tvalidation_0-error:0.291052\tvalidation_0-logloss:0.565713\tvalidation_1-error:0.327088\tvalidation_1-logloss:0.601179\n",
      "[697]\tvalidation_0-error:0.291004\tvalidation_0-logloss:0.565682\tvalidation_1-error:0.327081\tvalidation_1-logloss:0.601163\n",
      "[698]\tvalidation_0-error:0.290972\tvalidation_0-logloss:0.565602\tvalidation_1-error:0.327067\tvalidation_1-logloss:0.601139\n",
      "[699]\tvalidation_0-error:0.290942\tvalidation_0-logloss:0.565559\tvalidation_1-error:0.327073\tvalidation_1-logloss:0.601133\n",
      "[700]\tvalidation_0-error:0.290912\tvalidation_0-logloss:0.565565\tvalidation_1-error:0.327066\tvalidation_1-logloss:0.601149\n",
      "[701]\tvalidation_0-error:0.290864\tvalidation_0-logloss:0.565494\tvalidation_1-error:0.327039\tvalidation_1-logloss:0.601147\n",
      "[702]\tvalidation_0-error:0.29081\tvalidation_0-logloss:0.56545\tvalidation_1-error:0.327022\tvalidation_1-logloss:0.601134\n",
      "[703]\tvalidation_0-error:0.290837\tvalidation_0-logloss:0.565409\tvalidation_1-error:0.326995\tvalidation_1-logloss:0.601133\n",
      "[704]\tvalidation_0-error:0.290828\tvalidation_0-logloss:0.565372\tvalidation_1-error:0.32705\tvalidation_1-logloss:0.601119\n",
      "[705]\tvalidation_0-error:0.290756\tvalidation_0-logloss:0.565332\tvalidation_1-error:0.327081\tvalidation_1-logloss:0.601133\n",
      "[706]\tvalidation_0-error:0.290687\tvalidation_0-logloss:0.565366\tvalidation_1-error:0.327072\tvalidation_1-logloss:0.601178\n",
      "[707]\tvalidation_0-error:0.290663\tvalidation_0-logloss:0.565332\tvalidation_1-error:0.327042\tvalidation_1-logloss:0.601168\n",
      "[708]\tvalidation_0-error:0.290711\tvalidation_0-logloss:0.565277\tvalidation_1-error:0.326992\tvalidation_1-logloss:0.601145\n",
      "[709]\tvalidation_0-error:0.290732\tvalidation_0-logloss:0.565245\tvalidation_1-error:0.326977\tvalidation_1-logloss:0.601145\n",
      "[710]\tvalidation_0-error:0.290657\tvalidation_0-logloss:0.565206\tvalidation_1-error:0.326972\tvalidation_1-logloss:0.601132\n",
      "[711]\tvalidation_0-error:0.290558\tvalidation_0-logloss:0.56517\tvalidation_1-error:0.326988\tvalidation_1-logloss:0.601142\n",
      "[712]\tvalidation_0-error:0.290522\tvalidation_0-logloss:0.565126\tvalidation_1-error:0.326969\tvalidation_1-logloss:0.601103\n",
      "[713]\tvalidation_0-error:0.290405\tvalidation_0-logloss:0.565086\tvalidation_1-error:0.326949\tvalidation_1-logloss:0.601104\n",
      "[714]\tvalidation_0-error:0.290351\tvalidation_0-logloss:0.565053\tvalidation_1-error:0.326924\tvalidation_1-logloss:0.601109\n",
      "[715]\tvalidation_0-error:0.290285\tvalidation_0-logloss:0.565007\tvalidation_1-error:0.326963\tvalidation_1-logloss:0.601111\n",
      "[716]\tvalidation_0-error:0.29024\tvalidation_0-logloss:0.564964\tvalidation_1-error:0.326954\tvalidation_1-logloss:0.601102\n",
      "[717]\tvalidation_0-error:0.290222\tvalidation_0-logloss:0.564899\tvalidation_1-error:0.326921\tvalidation_1-logloss:0.601076\n",
      "[718]\tvalidation_0-error:0.290165\tvalidation_0-logloss:0.564851\tvalidation_1-error:0.326946\tvalidation_1-logloss:0.601071\n",
      "[719]\tvalidation_0-error:0.290123\tvalidation_0-logloss:0.564813\tvalidation_1-error:0.326958\tvalidation_1-logloss:0.601093\n",
      "[720]\tvalidation_0-error:0.29006\tvalidation_0-logloss:0.564775\tvalidation_1-error:0.326943\tvalidation_1-logloss:0.601098\n",
      "[721]\tvalidation_0-error:0.289934\tvalidation_0-logloss:0.564732\tvalidation_1-error:0.326927\tvalidation_1-logloss:0.601092\n",
      "[722]\tvalidation_0-error:0.289889\tvalidation_0-logloss:0.564527\tvalidation_1-error:0.326907\tvalidation_1-logloss:0.601004\n",
      "[723]\tvalidation_0-error:0.289838\tvalidation_0-logloss:0.564492\tvalidation_1-error:0.32691\tvalidation_1-logloss:0.601004\n",
      "[724]\tvalidation_0-error:0.289841\tvalidation_0-logloss:0.56444\tvalidation_1-error:0.326911\tvalidation_1-logloss:0.600965\n",
      "[725]\tvalidation_0-error:0.289808\tvalidation_0-logloss:0.564396\tvalidation_1-error:0.326896\tvalidation_1-logloss:0.600935\n",
      "[726]\tvalidation_0-error:0.289706\tvalidation_0-logloss:0.564338\tvalidation_1-error:0.326877\tvalidation_1-logloss:0.600929\n",
      "[727]\tvalidation_0-error:0.289676\tvalidation_0-logloss:0.564311\tvalidation_1-error:0.326865\tvalidation_1-logloss:0.600922\n",
      "[728]\tvalidation_0-error:0.289598\tvalidation_0-logloss:0.564244\tvalidation_1-error:0.326769\tvalidation_1-logloss:0.600884\n",
      "[729]\tvalidation_0-error:0.289565\tvalidation_0-logloss:0.564215\tvalidation_1-error:0.326762\tvalidation_1-logloss:0.600872\n",
      "[730]\tvalidation_0-error:0.289523\tvalidation_0-logloss:0.564175\tvalidation_1-error:0.326743\tvalidation_1-logloss:0.600855\n",
      "[731]\tvalidation_0-error:0.289481\tvalidation_0-logloss:0.564147\tvalidation_1-error:0.326725\tvalidation_1-logloss:0.600832\n",
      "[732]\tvalidation_0-error:0.289449\tvalidation_0-logloss:0.564096\tvalidation_1-error:0.326701\tvalidation_1-logloss:0.600833\n",
      "[733]\tvalidation_0-error:0.289452\tvalidation_0-logloss:0.564052\tvalidation_1-error:0.326687\tvalidation_1-logloss:0.600831\n",
      "[734]\tvalidation_0-error:0.289389\tvalidation_0-logloss:0.563994\tvalidation_1-error:0.326636\tvalidation_1-logloss:0.600807\n",
      "[735]\tvalidation_0-error:0.289314\tvalidation_0-logloss:0.563947\tvalidation_1-error:0.32665\tvalidation_1-logloss:0.600794\n",
      "[736]\tvalidation_0-error:0.289284\tvalidation_0-logloss:0.563904\tvalidation_1-error:0.326656\tvalidation_1-logloss:0.600773\n",
      "[737]\tvalidation_0-error:0.289278\tvalidation_0-logloss:0.563888\tvalidation_1-error:0.326675\tvalidation_1-logloss:0.600781\n",
      "[738]\tvalidation_0-error:0.289284\tvalidation_0-logloss:0.563854\tvalidation_1-error:0.32662\tvalidation_1-logloss:0.600767\n",
      "[739]\tvalidation_0-error:0.289251\tvalidation_0-logloss:0.563796\tvalidation_1-error:0.326673\tvalidation_1-logloss:0.600746\n",
      "[740]\tvalidation_0-error:0.289266\tvalidation_0-logloss:0.563753\tvalidation_1-error:0.326689\tvalidation_1-logloss:0.600729\n",
      "[741]\tvalidation_0-error:0.289236\tvalidation_0-logloss:0.563715\tvalidation_1-error:0.326692\tvalidation_1-logloss:0.600726\n",
      "[742]\tvalidation_0-error:0.289143\tvalidation_0-logloss:0.563678\tvalidation_1-error:0.326684\tvalidation_1-logloss:0.600717\n",
      "[743]\tvalidation_0-error:0.289116\tvalidation_0-logloss:0.563639\tvalidation_1-error:0.326722\tvalidation_1-logloss:0.600719\n",
      "[744]\tvalidation_0-error:0.289086\tvalidation_0-logloss:0.563603\tvalidation_1-error:0.326746\tvalidation_1-logloss:0.601074\n",
      "[745]\tvalidation_0-error:0.289053\tvalidation_0-logloss:0.56357\tvalidation_1-error:0.326732\tvalidation_1-logloss:0.601056\n",
      "[746]\tvalidation_0-error:0.289047\tvalidation_0-logloss:0.563533\tvalidation_1-error:0.326715\tvalidation_1-logloss:0.601046\n",
      "[747]\tvalidation_0-error:0.288996\tvalidation_0-logloss:0.563487\tvalidation_1-error:0.326735\tvalidation_1-logloss:0.601064\n",
      "[748]\tvalidation_0-error:0.288954\tvalidation_0-logloss:0.563464\tvalidation_1-error:0.326746\tvalidation_1-logloss:0.601038\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[749]\tvalidation_0-error:0.288924\tvalidation_0-logloss:0.563428\tvalidation_1-error:0.326722\tvalidation_1-logloss:0.601008\n",
      "[750]\tvalidation_0-error:0.288924\tvalidation_0-logloss:0.563394\tvalidation_1-error:0.326697\tvalidation_1-logloss:0.600992\n",
      "[751]\tvalidation_0-error:0.288885\tvalidation_0-logloss:0.563361\tvalidation_1-error:0.326678\tvalidation_1-logloss:0.600973\n",
      "[752]\tvalidation_0-error:0.288801\tvalidation_0-logloss:0.563281\tvalidation_1-error:0.326675\tvalidation_1-logloss:0.600959\n",
      "[753]\tvalidation_0-error:0.288816\tvalidation_0-logloss:0.563239\tvalidation_1-error:0.326638\tvalidation_1-logloss:0.600935\n",
      "[754]\tvalidation_0-error:0.288774\tvalidation_0-logloss:0.563222\tvalidation_1-error:0.326589\tvalidation_1-logloss:0.600899\n",
      "[755]\tvalidation_0-error:0.288783\tvalidation_0-logloss:0.563189\tvalidation_1-error:0.326574\tvalidation_1-logloss:0.600898\n",
      "[756]\tvalidation_0-error:0.288726\tvalidation_0-logloss:0.563103\tvalidation_1-error:0.326571\tvalidation_1-logloss:0.600859\n",
      "[757]\tvalidation_0-error:0.288684\tvalidation_0-logloss:0.563079\tvalidation_1-error:0.326565\tvalidation_1-logloss:0.60085\n",
      "[758]\tvalidation_0-error:0.288681\tvalidation_0-logloss:0.563044\tvalidation_1-error:0.32656\tvalidation_1-logloss:0.600841\n",
      "[759]\tvalidation_0-error:0.288642\tvalidation_0-logloss:0.563\tvalidation_1-error:0.326537\tvalidation_1-logloss:0.600802\n",
      "[760]\tvalidation_0-error:0.288621\tvalidation_0-logloss:0.563007\tvalidation_1-error:0.32649\tvalidation_1-logloss:0.600813\n",
      "[761]\tvalidation_0-error:0.288549\tvalidation_0-logloss:0.562927\tvalidation_1-error:0.326505\tvalidation_1-logloss:0.600793\n",
      "[762]\tvalidation_0-error:0.288525\tvalidation_0-logloss:0.562891\tvalidation_1-error:0.326484\tvalidation_1-logloss:0.600799\n",
      "[763]\tvalidation_0-error:0.288447\tvalidation_0-logloss:0.562847\tvalidation_1-error:0.326452\tvalidation_1-logloss:0.600793\n",
      "[764]\tvalidation_0-error:0.288438\tvalidation_0-logloss:0.562809\tvalidation_1-error:0.326462\tvalidation_1-logloss:0.600798\n",
      "[765]\tvalidation_0-error:0.288402\tvalidation_0-logloss:0.562777\tvalidation_1-error:0.326441\tvalidation_1-logloss:0.600771\n",
      "[766]\tvalidation_0-error:0.288375\tvalidation_0-logloss:0.562737\tvalidation_1-error:0.326471\tvalidation_1-logloss:0.600775\n",
      "[767]\tvalidation_0-error:0.288333\tvalidation_0-logloss:0.562708\tvalidation_1-error:0.326487\tvalidation_1-logloss:0.600765\n",
      "[768]\tvalidation_0-error:0.288303\tvalidation_0-logloss:0.562669\tvalidation_1-error:0.326468\tvalidation_1-logloss:0.600749\n",
      "[769]\tvalidation_0-error:0.288258\tvalidation_0-logloss:0.562645\tvalidation_1-error:0.326471\tvalidation_1-logloss:0.600746\n",
      "[770]\tvalidation_0-error:0.288195\tvalidation_0-logloss:0.562607\tvalidation_1-error:0.326425\tvalidation_1-logloss:0.600722\n",
      "[771]\tvalidation_0-error:0.288129\tvalidation_0-logloss:0.562596\tvalidation_1-error:0.326441\tvalidation_1-logloss:0.60073\n",
      "[772]\tvalidation_0-error:0.288108\tvalidation_0-logloss:0.562537\tvalidation_1-error:0.326416\tvalidation_1-logloss:0.600704\n",
      "[773]\tvalidation_0-error:0.288099\tvalidation_0-logloss:0.56252\tvalidation_1-error:0.326421\tvalidation_1-logloss:0.600704\n",
      "[774]\tvalidation_0-error:0.288093\tvalidation_0-logloss:0.562472\tvalidation_1-error:0.326413\tvalidation_1-logloss:0.600692\n",
      "[775]\tvalidation_0-error:0.288078\tvalidation_0-logloss:0.562454\tvalidation_1-error:0.326402\tvalidation_1-logloss:0.600682\n",
      "[776]\tvalidation_0-error:0.288048\tvalidation_0-logloss:0.562417\tvalidation_1-error:0.326421\tvalidation_1-logloss:0.600682\n",
      "[777]\tvalidation_0-error:0.288042\tvalidation_0-logloss:0.562378\tvalidation_1-error:0.326373\tvalidation_1-logloss:0.600669\n",
      "[778]\tvalidation_0-error:0.288045\tvalidation_0-logloss:0.562328\tvalidation_1-error:0.326348\tvalidation_1-logloss:0.600641\n",
      "[779]\tvalidation_0-error:0.288024\tvalidation_0-logloss:0.562285\tvalidation_1-error:0.326357\tvalidation_1-logloss:0.600636\n",
      "[780]\tvalidation_0-error:0.288\tvalidation_0-logloss:0.562253\tvalidation_1-error:0.326334\tvalidation_1-logloss:0.600614\n",
      "[781]\tvalidation_0-error:0.287994\tvalidation_0-logloss:0.562217\tvalidation_1-error:0.326344\tvalidation_1-logloss:0.600608\n",
      "[782]\tvalidation_0-error:0.287988\tvalidation_0-logloss:0.562174\tvalidation_1-error:0.326314\tvalidation_1-logloss:0.600588\n",
      "[783]\tvalidation_0-error:0.288009\tvalidation_0-logloss:0.562145\tvalidation_1-error:0.326295\tvalidation_1-logloss:0.600571\n",
      "[784]\tvalidation_0-error:0.287977\tvalidation_0-logloss:0.562115\tvalidation_1-error:0.326254\tvalidation_1-logloss:0.600555\n",
      "[785]\tvalidation_0-error:0.287896\tvalidation_0-logloss:0.562067\tvalidation_1-error:0.326258\tvalidation_1-logloss:0.60054\n",
      "[786]\tvalidation_0-error:0.28786\tvalidation_0-logloss:0.562028\tvalidation_1-error:0.326232\tvalidation_1-logloss:0.600512\n",
      "[787]\tvalidation_0-error:0.287812\tvalidation_0-logloss:0.561987\tvalidation_1-error:0.326211\tvalidation_1-logloss:0.600492\n",
      "[788]\tvalidation_0-error:0.287791\tvalidation_0-logloss:0.561946\tvalidation_1-error:0.326186\tvalidation_1-logloss:0.600492\n",
      "[789]\tvalidation_0-error:0.287797\tvalidation_0-logloss:0.561906\tvalidation_1-error:0.326176\tvalidation_1-logloss:0.600481\n",
      "[790]\tvalidation_0-error:0.287746\tvalidation_0-logloss:0.561875\tvalidation_1-error:0.326173\tvalidation_1-logloss:0.60047\n",
      "[791]\tvalidation_0-error:0.287692\tvalidation_0-logloss:0.561835\tvalidation_1-error:0.326161\tvalidation_1-logloss:0.600457\n",
      "[792]\tvalidation_0-error:0.287677\tvalidation_0-logloss:0.561804\tvalidation_1-error:0.326165\tvalidation_1-logloss:0.600448\n",
      "[793]\tvalidation_0-error:0.287635\tvalidation_0-logloss:0.56177\tvalidation_1-error:0.326161\tvalidation_1-logloss:0.600448\n",
      "[794]\tvalidation_0-error:0.287611\tvalidation_0-logloss:0.561735\tvalidation_1-error:0.326176\tvalidation_1-logloss:0.600432\n",
      "[795]\tvalidation_0-error:0.287554\tvalidation_0-logloss:0.561704\tvalidation_1-error:0.32618\tvalidation_1-logloss:0.600426\n",
      "[796]\tvalidation_0-error:0.287575\tvalidation_0-logloss:0.561673\tvalidation_1-error:0.326118\tvalidation_1-logloss:0.600434\n",
      "[797]\tvalidation_0-error:0.287503\tvalidation_0-logloss:0.561708\tvalidation_1-error:0.326114\tvalidation_1-logloss:0.600454\n",
      "[798]\tvalidation_0-error:0.287509\tvalidation_0-logloss:0.561675\tvalidation_1-error:0.326131\tvalidation_1-logloss:0.60044\n",
      "[799]\tvalidation_0-error:0.287473\tvalidation_0-logloss:0.561639\tvalidation_1-error:0.326131\tvalidation_1-logloss:0.600431\n",
      "[800]\tvalidation_0-error:0.287473\tvalidation_0-logloss:0.561603\tvalidation_1-error:0.326091\tvalidation_1-logloss:0.600405\n",
      "[801]\tvalidation_0-error:0.287488\tvalidation_0-logloss:0.56157\tvalidation_1-error:0.326127\tvalidation_1-logloss:0.599692\n",
      "[802]\tvalidation_0-error:0.287473\tvalidation_0-logloss:0.561539\tvalidation_1-error:0.326115\tvalidation_1-logloss:0.599682\n",
      "[803]\tvalidation_0-error:0.287473\tvalidation_0-logloss:0.561498\tvalidation_1-error:0.326111\tvalidation_1-logloss:0.599673\n",
      "[804]\tvalidation_0-error:0.287443\tvalidation_0-logloss:0.561461\tvalidation_1-error:0.326088\tvalidation_1-logloss:0.599686\n",
      "[805]\tvalidation_0-error:0.287416\tvalidation_0-logloss:0.561431\tvalidation_1-error:0.326072\tvalidation_1-logloss:0.599676\n",
      "[806]\tvalidation_0-error:0.287329\tvalidation_0-logloss:0.561393\tvalidation_1-error:0.326063\tvalidation_1-logloss:0.599675\n",
      "[807]\tvalidation_0-error:0.287356\tvalidation_0-logloss:0.561363\tvalidation_1-error:0.326109\tvalidation_1-logloss:0.599667\n",
      "[808]\tvalidation_0-error:0.28732\tvalidation_0-logloss:0.561333\tvalidation_1-error:0.326105\tvalidation_1-logloss:0.599676\n",
      "[809]\tvalidation_0-error:0.287311\tvalidation_0-logloss:0.561294\tvalidation_1-error:0.326087\tvalidation_1-logloss:0.599681\n",
      "[810]\tvalidation_0-error:0.28732\tvalidation_0-logloss:0.561255\tvalidation_1-error:0.32609\tvalidation_1-logloss:0.599689\n",
      "[811]\tvalidation_0-error:0.287284\tvalidation_0-logloss:0.56122\tvalidation_1-error:0.32608\tvalidation_1-logloss:0.59969\n",
      "[812]\tvalidation_0-error:0.287185\tvalidation_0-logloss:0.561184\tvalidation_1-error:0.326053\tvalidation_1-logloss:0.599672\n",
      "[813]\tvalidation_0-error:0.287116\tvalidation_0-logloss:0.561159\tvalidation_1-error:0.326009\tvalidation_1-logloss:0.599643\n",
      "[814]\tvalidation_0-error:0.287068\tvalidation_0-logloss:0.561123\tvalidation_1-error:0.326032\tvalidation_1-logloss:0.599638\n",
      "[815]\tvalidation_0-error:0.287056\tvalidation_0-logloss:0.561089\tvalidation_1-error:0.326044\tvalidation_1-logloss:0.599635\n",
      "[816]\tvalidation_0-error:0.287086\tvalidation_0-logloss:0.561046\tvalidation_1-error:0.326037\tvalidation_1-logloss:0.599627\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[817]\tvalidation_0-error:0.28705\tvalidation_0-logloss:0.561016\tvalidation_1-error:0.326059\tvalidation_1-logloss:0.599624\n",
      "[818]\tvalidation_0-error:0.286936\tvalidation_0-logloss:0.560967\tvalidation_1-error:0.326021\tvalidation_1-logloss:0.599617\n",
      "[819]\tvalidation_0-error:0.286927\tvalidation_0-logloss:0.560965\tvalidation_1-error:0.32599\tvalidation_1-logloss:0.599621\n",
      "[820]\tvalidation_0-error:0.286858\tvalidation_0-logloss:0.560924\tvalidation_1-error:0.326024\tvalidation_1-logloss:0.59965\n",
      "[821]\tvalidation_0-error:0.28678\tvalidation_0-logloss:0.560891\tvalidation_1-error:0.325978\tvalidation_1-logloss:0.599653\n",
      "[822]\tvalidation_0-error:0.286786\tvalidation_0-logloss:0.560863\tvalidation_1-error:0.325963\tvalidation_1-logloss:0.599638\n",
      "[823]\tvalidation_0-error:0.286717\tvalidation_0-logloss:0.56083\tvalidation_1-error:0.325926\tvalidation_1-logloss:0.599631\n",
      "[824]\tvalidation_0-error:0.286657\tvalidation_0-logloss:0.5608\tvalidation_1-error:0.325863\tvalidation_1-logloss:0.599608\n",
      "[825]\tvalidation_0-error:0.286645\tvalidation_0-logloss:0.560767\tvalidation_1-error:0.325888\tvalidation_1-logloss:0.599608\n",
      "[826]\tvalidation_0-error:0.286567\tvalidation_0-logloss:0.560738\tvalidation_1-error:0.325895\tvalidation_1-logloss:0.599612\n",
      "[827]\tvalidation_0-error:0.286519\tvalidation_0-logloss:0.560696\tvalidation_1-error:0.325917\tvalidation_1-logloss:0.599605\n",
      "[828]\tvalidation_0-error:0.286492\tvalidation_0-logloss:0.560662\tvalidation_1-error:0.32595\tvalidation_1-logloss:0.599631\n",
      "[829]\tvalidation_0-error:0.286456\tvalidation_0-logloss:0.560628\tvalidation_1-error:0.325907\tvalidation_1-logloss:0.599608\n",
      "[830]\tvalidation_0-error:0.286442\tvalidation_0-logloss:0.560599\tvalidation_1-error:0.32591\tvalidation_1-logloss:0.599601\n",
      "[831]\tvalidation_0-error:0.286403\tvalidation_0-logloss:0.560573\tvalidation_1-error:0.325897\tvalidation_1-logloss:0.599589\n",
      "[832]\tvalidation_0-error:0.286382\tvalidation_0-logloss:0.560551\tvalidation_1-error:0.325888\tvalidation_1-logloss:0.599577\n",
      "[833]\tvalidation_0-error:0.286358\tvalidation_0-logloss:0.560519\tvalidation_1-error:0.325873\tvalidation_1-logloss:0.599568\n",
      "[834]\tvalidation_0-error:0.286337\tvalidation_0-logloss:0.560482\tvalidation_1-error:0.32586\tvalidation_1-logloss:0.599558\n",
      "[835]\tvalidation_0-error:0.286316\tvalidation_0-logloss:0.560439\tvalidation_1-error:0.325866\tvalidation_1-logloss:0.599566\n",
      "[836]\tvalidation_0-error:0.286325\tvalidation_0-logloss:0.560409\tvalidation_1-error:0.325845\tvalidation_1-logloss:0.599548\n",
      "[837]\tvalidation_0-error:0.286337\tvalidation_0-logloss:0.56033\tvalidation_1-error:0.325843\tvalidation_1-logloss:0.599505\n",
      "[838]\tvalidation_0-error:0.286292\tvalidation_0-logloss:0.560332\tvalidation_1-error:0.325883\tvalidation_1-logloss:0.599524\n",
      "[839]\tvalidation_0-error:0.286253\tvalidation_0-logloss:0.560256\tvalidation_1-error:0.326001\tvalidation_1-logloss:0.59952\n",
      "[840]\tvalidation_0-error:0.286229\tvalidation_0-logloss:0.560223\tvalidation_1-error:0.326001\tvalidation_1-logloss:0.599508\n",
      "[841]\tvalidation_0-error:0.286193\tvalidation_0-logloss:0.560194\tvalidation_1-error:0.325987\tvalidation_1-logloss:0.599503\n",
      "[842]\tvalidation_0-error:0.28616\tvalidation_0-logloss:0.560156\tvalidation_1-error:0.325964\tvalidation_1-logloss:0.599496\n",
      "[843]\tvalidation_0-error:0.286136\tvalidation_0-logloss:0.560124\tvalidation_1-error:0.325951\tvalidation_1-logloss:0.599487\n",
      "[844]\tvalidation_0-error:0.286094\tvalidation_0-logloss:0.560092\tvalidation_1-error:0.32592\tvalidation_1-logloss:0.599485\n",
      "[845]\tvalidation_0-error:0.286088\tvalidation_0-logloss:0.560052\tvalidation_1-error:0.32591\tvalidation_1-logloss:0.599493\n",
      "[846]\tvalidation_0-error:0.286106\tvalidation_0-logloss:0.560146\tvalidation_1-error:0.325947\tvalidation_1-logloss:0.599669\n",
      "[847]\tvalidation_0-error:0.286061\tvalidation_0-logloss:0.560118\tvalidation_1-error:0.325962\tvalidation_1-logloss:0.599663\n",
      "[848]\tvalidation_0-error:0.286052\tvalidation_0-logloss:0.559939\tvalidation_1-error:0.325979\tvalidation_1-logloss:0.59952\n",
      "[849]\tvalidation_0-error:0.285986\tvalidation_0-logloss:0.559891\tvalidation_1-error:0.325938\tvalidation_1-logloss:0.599509\n",
      "[850]\tvalidation_0-error:0.285938\tvalidation_0-logloss:0.559858\tvalidation_1-error:0.325919\tvalidation_1-logloss:0.599505\n",
      "[851]\tvalidation_0-error:0.285914\tvalidation_0-logloss:0.559867\tvalidation_1-error:0.325929\tvalidation_1-logloss:0.599532\n",
      "[852]\tvalidation_0-error:0.285884\tvalidation_0-logloss:0.559785\tvalidation_1-error:0.325899\tvalidation_1-logloss:0.599494\n",
      "[853]\tvalidation_0-error:0.285842\tvalidation_0-logloss:0.559746\tvalidation_1-error:0.325882\tvalidation_1-logloss:0.59949\n",
      "[854]\tvalidation_0-error:0.285797\tvalidation_0-logloss:0.559717\tvalidation_1-error:0.325888\tvalidation_1-logloss:0.599473\n",
      "[855]\tvalidation_0-error:0.285779\tvalidation_0-logloss:0.559685\tvalidation_1-error:0.325873\tvalidation_1-logloss:0.599467\n",
      "[856]\tvalidation_0-error:0.285782\tvalidation_0-logloss:0.559653\tvalidation_1-error:0.325868\tvalidation_1-logloss:0.599466\n",
      "[857]\tvalidation_0-error:0.28574\tvalidation_0-logloss:0.559621\tvalidation_1-error:0.325852\tvalidation_1-logloss:0.59946\n",
      "[858]\tvalidation_0-error:0.285725\tvalidation_0-logloss:0.559592\tvalidation_1-error:0.325827\tvalidation_1-logloss:0.599461\n",
      "[859]\tvalidation_0-error:0.285698\tvalidation_0-logloss:0.55956\tvalidation_1-error:0.325804\tvalidation_1-logloss:0.59945\n",
      "[860]\tvalidation_0-error:0.285653\tvalidation_0-logloss:0.559527\tvalidation_1-error:0.325821\tvalidation_1-logloss:0.599471\n",
      "[861]\tvalidation_0-error:0.285626\tvalidation_0-logloss:0.559485\tvalidation_1-error:0.325833\tvalidation_1-logloss:0.599473\n",
      "[862]\tvalidation_0-error:0.285611\tvalidation_0-logloss:0.559453\tvalidation_1-error:0.325823\tvalidation_1-logloss:0.599479\n",
      "[863]\tvalidation_0-error:0.285599\tvalidation_0-logloss:0.559413\tvalidation_1-error:0.32584\tvalidation_1-logloss:0.599483\n",
      "[864]\tvalidation_0-error:0.285596\tvalidation_0-logloss:0.559511\tvalidation_1-error:0.325837\tvalidation_1-logloss:0.599625\n",
      "[865]\tvalidation_0-error:0.285584\tvalidation_0-logloss:0.55948\tvalidation_1-error:0.325833\tvalidation_1-logloss:0.599604\n",
      "[866]\tvalidation_0-error:0.285539\tvalidation_0-logloss:0.559448\tvalidation_1-error:0.325811\tvalidation_1-logloss:0.599587\n",
      "[867]\tvalidation_0-error:0.285494\tvalidation_0-logloss:0.559408\tvalidation_1-error:0.325829\tvalidation_1-logloss:0.599601\n",
      "[868]\tvalidation_0-error:0.285395\tvalidation_0-logloss:0.559356\tvalidation_1-error:0.325835\tvalidation_1-logloss:0.599614\n",
      "[869]\tvalidation_0-error:0.28535\tvalidation_0-logloss:0.559326\tvalidation_1-error:0.325821\tvalidation_1-logloss:0.599606\n",
      "[870]\tvalidation_0-error:0.285296\tvalidation_0-logloss:0.559284\tvalidation_1-error:0.32582\tvalidation_1-logloss:0.599589\n",
      "[871]\tvalidation_0-error:0.285278\tvalidation_0-logloss:0.559235\tvalidation_1-error:0.325823\tvalidation_1-logloss:0.599572\n",
      "[872]\tvalidation_0-error:0.285275\tvalidation_0-logloss:0.55921\tvalidation_1-error:0.325808\tvalidation_1-logloss:0.599557\n",
      "[873]\tvalidation_0-error:0.285218\tvalidation_0-logloss:0.559179\tvalidation_1-error:0.325775\tvalidation_1-logloss:0.599543\n",
      "[874]\tvalidation_0-error:0.285188\tvalidation_0-logloss:0.559134\tvalidation_1-error:0.325784\tvalidation_1-logloss:0.599532\n",
      "[875]\tvalidation_0-error:0.285203\tvalidation_0-logloss:0.559107\tvalidation_1-error:0.325789\tvalidation_1-logloss:0.59952\n",
      "[876]\tvalidation_0-error:0.285155\tvalidation_0-logloss:0.559065\tvalidation_1-error:0.325827\tvalidation_1-logloss:0.599511\n",
      "[877]\tvalidation_0-error:0.285113\tvalidation_0-logloss:0.559025\tvalidation_1-error:0.325789\tvalidation_1-logloss:0.599511\n",
      "[878]\tvalidation_0-error:0.285101\tvalidation_0-logloss:0.558996\tvalidation_1-error:0.325823\tvalidation_1-logloss:0.599503\n",
      "[879]\tvalidation_0-error:0.285083\tvalidation_0-logloss:0.558966\tvalidation_1-error:0.325802\tvalidation_1-logloss:0.599483\n",
      "[880]\tvalidation_0-error:0.285071\tvalidation_0-logloss:0.558932\tvalidation_1-error:0.325789\tvalidation_1-logloss:0.599483\n",
      "[881]\tvalidation_0-error:0.285035\tvalidation_0-logloss:0.558914\tvalidation_1-error:0.325808\tvalidation_1-logloss:0.599504\n",
      "[882]\tvalidation_0-error:0.285017\tvalidation_0-logloss:0.558881\tvalidation_1-error:0.325824\tvalidation_1-logloss:0.599501\n",
      "[883]\tvalidation_0-error:0.28488\tvalidation_0-logloss:0.558842\tvalidation_1-error:0.325765\tvalidation_1-logloss:0.599496\n",
      "[884]\tvalidation_0-error:0.284865\tvalidation_0-logloss:0.558795\tvalidation_1-error:0.325756\tvalidation_1-logloss:0.599477\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[885]\tvalidation_0-error:0.284865\tvalidation_0-logloss:0.558756\tvalidation_1-error:0.325772\tvalidation_1-logloss:0.599463\n",
      "[886]\tvalidation_0-error:0.284871\tvalidation_0-logloss:0.558737\tvalidation_1-error:0.325722\tvalidation_1-logloss:0.599451\n",
      "[887]\tvalidation_0-error:0.284814\tvalidation_0-logloss:0.558696\tvalidation_1-error:0.325708\tvalidation_1-logloss:0.599432\n",
      "[888]\tvalidation_0-error:0.284769\tvalidation_0-logloss:0.558655\tvalidation_1-error:0.32573\tvalidation_1-logloss:0.599414\n",
      "[889]\tvalidation_0-error:0.284718\tvalidation_0-logloss:0.558626\tvalidation_1-error:0.325721\tvalidation_1-logloss:0.599415\n",
      "[890]\tvalidation_0-error:0.28473\tvalidation_0-logloss:0.558617\tvalidation_1-error:0.325728\tvalidation_1-logloss:0.59942\n",
      "[891]\tvalidation_0-error:0.284709\tvalidation_0-logloss:0.558568\tvalidation_1-error:0.325719\tvalidation_1-logloss:0.599392\n",
      "[892]\tvalidation_0-error:0.284706\tvalidation_0-logloss:0.558554\tvalidation_1-error:0.325697\tvalidation_1-logloss:0.599402\n",
      "[893]\tvalidation_0-error:0.284673\tvalidation_0-logloss:0.558523\tvalidation_1-error:0.325671\tvalidation_1-logloss:0.599389\n",
      "[894]\tvalidation_0-error:0.284664\tvalidation_0-logloss:0.558496\tvalidation_1-error:0.325705\tvalidation_1-logloss:0.599382\n",
      "[895]\tvalidation_0-error:0.284643\tvalidation_0-logloss:0.558455\tvalidation_1-error:0.325682\tvalidation_1-logloss:0.599381\n",
      "[896]\tvalidation_0-error:0.284631\tvalidation_0-logloss:0.558426\tvalidation_1-error:0.325659\tvalidation_1-logloss:0.599361\n",
      "[897]\tvalidation_0-error:0.284655\tvalidation_0-logloss:0.558387\tvalidation_1-error:0.325653\tvalidation_1-logloss:0.599356\n",
      "[898]\tvalidation_0-error:0.284655\tvalidation_0-logloss:0.558355\tvalidation_1-error:0.325632\tvalidation_1-logloss:0.599348\n",
      "[899]\tvalidation_0-error:0.284634\tvalidation_0-logloss:0.558327\tvalidation_1-error:0.325635\tvalidation_1-logloss:0.599335\n",
      "[900]\tvalidation_0-error:0.284613\tvalidation_0-logloss:0.558136\tvalidation_1-error:0.325613\tvalidation_1-logloss:0.599238\n",
      "[901]\tvalidation_0-error:0.284547\tvalidation_0-logloss:0.558115\tvalidation_1-error:0.325595\tvalidation_1-logloss:0.599246\n",
      "[902]\tvalidation_0-error:0.28446\tvalidation_0-logloss:0.558082\tvalidation_1-error:0.325594\tvalidation_1-logloss:0.599236\n",
      "[903]\tvalidation_0-error:0.284475\tvalidation_0-logloss:0.558045\tvalidation_1-error:0.325619\tvalidation_1-logloss:0.599226\n",
      "[904]\tvalidation_0-error:0.284448\tvalidation_0-logloss:0.558093\tvalidation_1-error:0.325613\tvalidation_1-logloss:0.599258\n",
      "[905]\tvalidation_0-error:0.284391\tvalidation_0-logloss:0.557976\tvalidation_1-error:0.325617\tvalidation_1-logloss:0.599228\n",
      "[906]\tvalidation_0-error:0.284352\tvalidation_0-logloss:0.557948\tvalidation_1-error:0.325601\tvalidation_1-logloss:0.599231\n",
      "[907]\tvalidation_0-error:0.284364\tvalidation_0-logloss:0.557994\tvalidation_1-error:0.325632\tvalidation_1-logloss:0.599268\n",
      "[908]\tvalidation_0-error:0.284328\tvalidation_0-logloss:0.557941\tvalidation_1-error:0.32565\tvalidation_1-logloss:0.599262\n",
      "[909]\tvalidation_0-error:0.284238\tvalidation_0-logloss:0.557906\tvalidation_1-error:0.325638\tvalidation_1-logloss:0.599252\n",
      "[910]\tvalidation_0-error:0.284193\tvalidation_0-logloss:0.557873\tvalidation_1-error:0.325626\tvalidation_1-logloss:0.599247\n",
      "[911]\tvalidation_0-error:0.284154\tvalidation_0-logloss:0.557842\tvalidation_1-error:0.325628\tvalidation_1-logloss:0.599235\n",
      "[912]\tvalidation_0-error:0.284109\tvalidation_0-logloss:0.557814\tvalidation_1-error:0.32561\tvalidation_1-logloss:0.599241\n",
      "[913]\tvalidation_0-error:0.284076\tvalidation_0-logloss:0.557777\tvalidation_1-error:0.325625\tvalidation_1-logloss:0.599232\n",
      "[914]\tvalidation_0-error:0.284025\tvalidation_0-logloss:0.557739\tvalidation_1-error:0.325586\tvalidation_1-logloss:0.599231\n",
      "[915]\tvalidation_0-error:0.283977\tvalidation_0-logloss:0.557704\tvalidation_1-error:0.325563\tvalidation_1-logloss:0.599224\n",
      "[916]\tvalidation_0-error:0.283962\tvalidation_0-logloss:0.557672\tvalidation_1-error:0.325511\tvalidation_1-logloss:0.599215\n",
      "[917]\tvalidation_0-error:0.283947\tvalidation_0-logloss:0.557645\tvalidation_1-error:0.325476\tvalidation_1-logloss:0.599204\n",
      "[918]\tvalidation_0-error:0.283959\tvalidation_0-logloss:0.557616\tvalidation_1-error:0.325486\tvalidation_1-logloss:0.59919\n",
      "[919]\tvalidation_0-error:0.283938\tvalidation_0-logloss:0.557578\tvalidation_1-error:0.325467\tvalidation_1-logloss:0.599178\n",
      "[920]\tvalidation_0-error:0.283884\tvalidation_0-logloss:0.557546\tvalidation_1-error:0.325473\tvalidation_1-logloss:0.599167\n",
      "[921]\tvalidation_0-error:0.283881\tvalidation_0-logloss:0.557522\tvalidation_1-error:0.325452\tvalidation_1-logloss:0.599166\n",
      "[922]\tvalidation_0-error:0.283845\tvalidation_0-logloss:0.557483\tvalidation_1-error:0.325477\tvalidation_1-logloss:0.599164\n",
      "[923]\tvalidation_0-error:0.283806\tvalidation_0-logloss:0.55745\tvalidation_1-error:0.325467\tvalidation_1-logloss:0.599142\n",
      "[924]\tvalidation_0-error:0.28377\tvalidation_0-logloss:0.557409\tvalidation_1-error:0.325445\tvalidation_1-logloss:0.599135\n",
      "[925]\tvalidation_0-error:0.28377\tvalidation_0-logloss:0.557377\tvalidation_1-error:0.325455\tvalidation_1-logloss:0.59914\n",
      "[926]\tvalidation_0-error:0.283758\tvalidation_0-logloss:0.557345\tvalidation_1-error:0.325467\tvalidation_1-logloss:0.599145\n",
      "[927]\tvalidation_0-error:0.283692\tvalidation_0-logloss:0.557316\tvalidation_1-error:0.325483\tvalidation_1-logloss:0.599156\n",
      "[928]\tvalidation_0-error:0.283674\tvalidation_0-logloss:0.557284\tvalidation_1-error:0.325471\tvalidation_1-logloss:0.599153\n",
      "[929]\tvalidation_0-error:0.283656\tvalidation_0-logloss:0.557239\tvalidation_1-error:0.325479\tvalidation_1-logloss:0.599147\n",
      "[930]\tvalidation_0-error:0.283656\tvalidation_0-logloss:0.55721\tvalidation_1-error:0.325519\tvalidation_1-logloss:0.599154\n",
      "[931]\tvalidation_0-error:0.283608\tvalidation_0-logloss:0.557179\tvalidation_1-error:0.32553\tvalidation_1-logloss:0.59914\n",
      "[932]\tvalidation_0-error:0.283584\tvalidation_0-logloss:0.557148\tvalidation_1-error:0.325544\tvalidation_1-logloss:0.599133\n",
      "[933]\tvalidation_0-error:0.283551\tvalidation_0-logloss:0.557103\tvalidation_1-error:0.325554\tvalidation_1-logloss:0.59912\n",
      "[934]\tvalidation_0-error:0.283572\tvalidation_0-logloss:0.557075\tvalidation_1-error:0.325541\tvalidation_1-logloss:0.599098\n",
      "[935]\tvalidation_0-error:0.283539\tvalidation_0-logloss:0.557038\tvalidation_1-error:0.32551\tvalidation_1-logloss:0.599081\n",
      "[936]\tvalidation_0-error:0.283506\tvalidation_0-logloss:0.556997\tvalidation_1-error:0.325514\tvalidation_1-logloss:0.599109\n",
      "[937]\tvalidation_0-error:0.2835\tvalidation_0-logloss:0.556962\tvalidation_1-error:0.325459\tvalidation_1-logloss:0.599125\n",
      "[938]\tvalidation_0-error:0.283464\tvalidation_0-logloss:0.556919\tvalidation_1-error:0.325505\tvalidation_1-logloss:0.599118\n",
      "[939]\tvalidation_0-error:0.283455\tvalidation_0-logloss:0.556884\tvalidation_1-error:0.32548\tvalidation_1-logloss:0.599125\n",
      "[940]\tvalidation_0-error:0.283441\tvalidation_0-logloss:0.556856\tvalidation_1-error:0.325485\tvalidation_1-logloss:0.599116\n",
      "[941]\tvalidation_0-error:0.283408\tvalidation_0-logloss:0.556821\tvalidation_1-error:0.325392\tvalidation_1-logloss:0.599113\n",
      "[942]\tvalidation_0-error:0.28339\tvalidation_0-logloss:0.556782\tvalidation_1-error:0.325384\tvalidation_1-logloss:0.599113\n",
      "[943]\tvalidation_0-error:0.283375\tvalidation_0-logloss:0.556754\tvalidation_1-error:0.325377\tvalidation_1-logloss:0.599108\n",
      "[944]\tvalidation_0-error:0.283321\tvalidation_0-logloss:0.556726\tvalidation_1-error:0.325374\tvalidation_1-logloss:0.599094\n",
      "[945]\tvalidation_0-error:0.283267\tvalidation_0-logloss:0.556667\tvalidation_1-error:0.325369\tvalidation_1-logloss:0.599084\n",
      "[946]\tvalidation_0-error:0.283261\tvalidation_0-logloss:0.556628\tvalidation_1-error:0.325361\tvalidation_1-logloss:0.599103\n",
      "[947]\tvalidation_0-error:0.283255\tvalidation_0-logloss:0.556601\tvalidation_1-error:0.325312\tvalidation_1-logloss:0.599088\n",
      "[948]\tvalidation_0-error:0.283216\tvalidation_0-logloss:0.556575\tvalidation_1-error:0.325294\tvalidation_1-logloss:0.599087\n",
      "[949]\tvalidation_0-error:0.28324\tvalidation_0-logloss:0.55655\tvalidation_1-error:0.325279\tvalidation_1-logloss:0.599078\n",
      "[950]\tvalidation_0-error:0.283219\tvalidation_0-logloss:0.556515\tvalidation_1-error:0.325257\tvalidation_1-logloss:0.599065\n",
      "[951]\tvalidation_0-error:0.283207\tvalidation_0-logloss:0.55648\tvalidation_1-error:0.325248\tvalidation_1-logloss:0.599068\n",
      "[952]\tvalidation_0-error:0.283174\tvalidation_0-logloss:0.556445\tvalidation_1-error:0.325241\tvalidation_1-logloss:0.59906\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[953]\tvalidation_0-error:0.283186\tvalidation_0-logloss:0.556419\tvalidation_1-error:0.325257\tvalidation_1-logloss:0.599052\n",
      "[954]\tvalidation_0-error:0.283204\tvalidation_0-logloss:0.556383\tvalidation_1-error:0.325231\tvalidation_1-logloss:0.599027\n",
      "[955]\tvalidation_0-error:0.28315\tvalidation_0-logloss:0.556349\tvalidation_1-error:0.325185\tvalidation_1-logloss:0.599036\n",
      "[956]\tvalidation_0-error:0.283108\tvalidation_0-logloss:0.556298\tvalidation_1-error:0.325208\tvalidation_1-logloss:0.599028\n",
      "[957]\tvalidation_0-error:0.283075\tvalidation_0-logloss:0.556261\tvalidation_1-error:0.32521\tvalidation_1-logloss:0.599025\n",
      "[958]\tvalidation_0-error:0.282997\tvalidation_0-logloss:0.55623\tvalidation_1-error:0.325235\tvalidation_1-logloss:0.599044\n",
      "[959]\tvalidation_0-error:0.28294\tvalidation_0-logloss:0.556208\tvalidation_1-error:0.325244\tvalidation_1-logloss:0.599048\n",
      "[960]\tvalidation_0-error:0.282943\tvalidation_0-logloss:0.556167\tvalidation_1-error:0.325241\tvalidation_1-logloss:0.599055\n",
      "[961]\tvalidation_0-error:0.282907\tvalidation_0-logloss:0.556122\tvalidation_1-error:0.325294\tvalidation_1-logloss:0.599045\n",
      "[962]\tvalidation_0-error:0.282904\tvalidation_0-logloss:0.556098\tvalidation_1-error:0.325288\tvalidation_1-logloss:0.599051\n",
      "[963]\tvalidation_0-error:0.282895\tvalidation_0-logloss:0.556069\tvalidation_1-error:0.325315\tvalidation_1-logloss:0.599059\n",
      "[964]\tvalidation_0-error:0.282874\tvalidation_0-logloss:0.556039\tvalidation_1-error:0.325319\tvalidation_1-logloss:0.599069\n",
      "[965]\tvalidation_0-error:0.282859\tvalidation_0-logloss:0.555995\tvalidation_1-error:0.325307\tvalidation_1-logloss:0.59906\n",
      "[966]\tvalidation_0-error:0.282829\tvalidation_0-logloss:0.555963\tvalidation_1-error:0.325331\tvalidation_1-logloss:0.59906\n",
      "[967]\tvalidation_0-error:0.282829\tvalidation_0-logloss:0.555932\tvalidation_1-error:0.32529\tvalidation_1-logloss:0.599075\n",
      "[968]\tvalidation_0-error:0.282793\tvalidation_0-logloss:0.555907\tvalidation_1-error:0.325294\tvalidation_1-logloss:0.599058\n",
      "[969]\tvalidation_0-error:0.282709\tvalidation_0-logloss:0.555864\tvalidation_1-error:0.325294\tvalidation_1-logloss:0.59905\n",
      "[970]\tvalidation_0-error:0.282664\tvalidation_0-logloss:0.555837\tvalidation_1-error:0.32531\tvalidation_1-logloss:0.599038\n",
      "[971]\tvalidation_0-error:0.282619\tvalidation_0-logloss:0.555813\tvalidation_1-error:0.325265\tvalidation_1-logloss:0.599035\n",
      "[972]\tvalidation_0-error:0.282571\tvalidation_0-logloss:0.555775\tvalidation_1-error:0.325276\tvalidation_1-logloss:0.599035\n",
      "[973]\tvalidation_0-error:0.282559\tvalidation_0-logloss:0.555756\tvalidation_1-error:0.325309\tvalidation_1-logloss:0.599056\n",
      "[974]\tvalidation_0-error:0.28252\tvalidation_0-logloss:0.555717\tvalidation_1-error:0.325306\tvalidation_1-logloss:0.599054\n",
      "[975]\tvalidation_0-error:0.282499\tvalidation_0-logloss:0.555686\tvalidation_1-error:0.325306\tvalidation_1-logloss:0.599058\n",
      "[976]\tvalidation_0-error:0.282439\tvalidation_0-logloss:0.555666\tvalidation_1-error:0.325279\tvalidation_1-logloss:0.599055\n",
      "[977]\tvalidation_0-error:0.282439\tvalidation_0-logloss:0.555636\tvalidation_1-error:0.325262\tvalidation_1-logloss:0.599032\n",
      "[978]\tvalidation_0-error:0.282385\tvalidation_0-logloss:0.555612\tvalidation_1-error:0.325262\tvalidation_1-logloss:0.599029\n",
      "[979]\tvalidation_0-error:0.282355\tvalidation_0-logloss:0.555321\tvalidation_1-error:0.325234\tvalidation_1-logloss:0.598728\n",
      "[980]\tvalidation_0-error:0.282364\tvalidation_0-logloss:0.555298\tvalidation_1-error:0.325222\tvalidation_1-logloss:0.598713\n",
      "[981]\tvalidation_0-error:0.282325\tvalidation_0-logloss:0.55526\tvalidation_1-error:0.325198\tvalidation_1-logloss:0.59871\n",
      "[982]\tvalidation_0-error:0.28228\tvalidation_0-logloss:0.555223\tvalidation_1-error:0.325216\tvalidation_1-logloss:0.598718\n",
      "[983]\tvalidation_0-error:0.282298\tvalidation_0-logloss:0.555198\tvalidation_1-error:0.3252\tvalidation_1-logloss:0.598726\n",
      "[984]\tvalidation_0-error:0.282253\tvalidation_0-logloss:0.555169\tvalidation_1-error:0.325182\tvalidation_1-logloss:0.598716\n",
      "[985]\tvalidation_0-error:0.282232\tvalidation_0-logloss:0.555141\tvalidation_1-error:0.32517\tvalidation_1-logloss:0.598709\n",
      "[986]\tvalidation_0-error:0.282232\tvalidation_0-logloss:0.555122\tvalidation_1-error:0.3252\tvalidation_1-logloss:0.598721\n",
      "[987]\tvalidation_0-error:0.282265\tvalidation_0-logloss:0.555096\tvalidation_1-error:0.325216\tvalidation_1-logloss:0.598711\n",
      "[988]\tvalidation_0-error:0.282271\tvalidation_0-logloss:0.555053\tvalidation_1-error:0.325228\tvalidation_1-logloss:0.598696\n",
      "[989]\tvalidation_0-error:0.282229\tvalidation_0-logloss:0.555011\tvalidation_1-error:0.325239\tvalidation_1-logloss:0.598697\n",
      "[990]\tvalidation_0-error:0.282208\tvalidation_0-logloss:0.554985\tvalidation_1-error:0.32521\tvalidation_1-logloss:0.598671\n",
      "[991]\tvalidation_0-error:0.28216\tvalidation_0-logloss:0.554959\tvalidation_1-error:0.325208\tvalidation_1-logloss:0.598667\n",
      "[992]\tvalidation_0-error:0.282118\tvalidation_0-logloss:0.554932\tvalidation_1-error:0.325188\tvalidation_1-logloss:0.598647\n",
      "[993]\tvalidation_0-error:0.282127\tvalidation_0-logloss:0.554906\tvalidation_1-error:0.32516\tvalidation_1-logloss:0.598641\n",
      "[994]\tvalidation_0-error:0.282088\tvalidation_0-logloss:0.554868\tvalidation_1-error:0.325176\tvalidation_1-logloss:0.598638\n",
      "[995]\tvalidation_0-error:0.282052\tvalidation_0-logloss:0.554842\tvalidation_1-error:0.325183\tvalidation_1-logloss:0.598646\n",
      "[996]\tvalidation_0-error:0.282028\tvalidation_0-logloss:0.55482\tvalidation_1-error:0.325164\tvalidation_1-logloss:0.598621\n",
      "[997]\tvalidation_0-error:0.282004\tvalidation_0-logloss:0.554787\tvalidation_1-error:0.325201\tvalidation_1-logloss:0.59861\n",
      "[998]\tvalidation_0-error:0.281942\tvalidation_0-logloss:0.554761\tvalidation_1-error:0.3252\tvalidation_1-logloss:0.5986\n",
      "[999]\tvalidation_0-error:0.281918\tvalidation_0-logloss:0.55474\tvalidation_1-error:0.325203\tvalidation_1-logloss:0.598598\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.7, eta=0.01, gamma=0, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=8, min_child_weight=6, missing=None,\n",
       "       n_estimators=1000, n_jobs=1, nthread=None,\n",
       "       objective='binary:logistic', random_state=0, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=None, silent=True,\n",
       "       subsample=0.7)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "eval_set = [(X_train, y_train), (X_test, y_test)]\n",
    "params = {\n",
    "    'n_estimators': 1000,\n",
    "    'booster': 'gbtree',\n",
    "    'eta': .01,\n",
    "    'min_child_weight': 6,\n",
    "    'max_depth': 8,\n",
    "    'subsample': 0.7,\n",
    "    'colsample_bytree': 0.7\n",
    "}\n",
    "\n",
    "xgb = XGBClassifier(**params)\n",
    "xgb.fit(X_train, y_train, eval_metric=[\"error\", \"logloss\"], eval_set=eval_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6747974802796145"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_pred = xgb.predict(X_test)\n",
    "accuracy_score(y_test, xgb_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1a38933668>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4HNW5+PHvu+q9W7JVbLkBLmAbYXo3YAjB3ITqFALhOuRCSEIa/FIgJPxCSJ4AufAL4eaalEtwCCVxKPGFBJLQbMsFd2O5y7as3stK2vf3x4zktSzZa2ullXbfz/Pso50zZ2bOeP28Z+bMmXNEVTHGGBMZPKEugDHGmOFjQd8YYyKIBX1jjIkgFvSNMSaCWNA3xpgIYkHfGGMiiAV9Y4yJIBb0jTEmgljQN8aYCBId6gL0lZ2drRMmTAh1MYwxZlRZtWpVtarmHCvfiAv6EyZMoLS0NNTFMMaYUUVEdgeSz5p3jDEmgljQN8aYCGJB3xhjIsiIa9M3xpjj0dnZSXl5Oe3t7aEuyrCIj4+noKCAmJiYE9regr4xZlQrLy8nJSWFCRMmICKhLs6QUlVqamooLy+nuLj4hPZhzTvGmFGtvb2drKyssA/4ACJCVlbWoO5qLOgbY0a9SAj4PQZ7ruET9Bv2wd8fguqyUJfEGGNGrPAJ+s0V8M9HoHZ7qEtijIkgNTU1zJo1i1mzZpGXl0d+fn7vstfrDWgft956K1u3bh3ikjoCepArIvOBx4Eo4Feq+vAA+a4D/gicoaqlfulFwCbgAVX96aBL3e/B3fpLfUOye2OM6U9WVhZr164F4IEHHiA5OZmvf/3rh+VRVVQVj6f/6+xnnnlmyMvZ45hX+iISBTwJXAlMA24WkWn95EsB7gaW97ObR4HXB1fUozvQ6NSoe2qah/IwxhgTkLKyMmbMmMEdd9zBnDlzOHDgAIsWLaKkpITp06fz4IMP9uY977zzWLt2LV1dXaSnp3Pvvfdy2mmncfbZZ1NZWRnUcgVypT8XKFPVHQAisgRYgHPl7u8HwCPAYVWciFwL7ABaBl3ao2hs72IsUN/STtFQHsgYM2J9/y8b2bS/Maj7nDYulfs/Pv2Ett20aRPPPPMMTz31FAAPP/wwmZmZdHV1cfHFF3Pdddcxbdrh19ANDQ1ceOGFPPzww9xzzz0sXryYe++9d9Dn0SOQNv18YK/fcrmb1ktEZgOFqvpKn/Qk4FvA9wdZzmPySBTg3EYZY8xIMGnSJM4444ze5eeee445c+YwZ84cNm/ezKZNfa+dISEhgSuvvBKA008/nV27dgW1TIFc6ffXP6g3soqIB6f55nP95Ps+8KiqNh+tm5GILAIWARQVneB1usfa9I2JdCd6RT5UkpKSer9v27aNxx9/nBUrVpCens6nP/3pfvvbx8bG9n6Pioqiq6srqGUK5Eq/HCj0Wy4A9vstpwAzgLdFZBdwFrBUREqAM4FH3PSvAP9HRO7qewBVfVpVS1S1JCfnmMNB90s8TqWiPgv6xpiRp7GxkZSUFFJTUzlw4ADLli0LSTkCudJfCUwRkWJgH3ATsLBnpao2ANk9yyLyNvB1t/fO+X7pDwDNqvpEUEreh7jNO2BB3xgz8syZM4dp06YxY8YMJk6cyLnnnhuSchwz6Ktql3t1vgyny+ZiVd0oIg8Cpaq6dKgLGQjpad6xK31jTIg88MADvd8nT57c25UTnDdpf/e73/W73TvvvNP7vb6+vvf7TTfdxE033RTUMgbUT19VXwNe65P2vQHyXjRA+gPHWbbj0vPMwB7kGmPMwMLmjVyxB7nGGHNMYRT0e7psWtA3xpiBhE/Q7+lZam36xhgzoPAJ+r1jWlibvjHGDCRsgr6np3nHrvSNMWZAYRP0e0bZtDZ9Y8xwCsbQygCLFy+moqJiCEvqCJs5cnveyBV7OcsYM4wCGVo5EIsXL2bOnDnk5eUFu4iHCZugbwOuGWNGmt/85jc8+eSTeL1ezjnnHJ544gl8Ph+33nora9euRVVZtGgRubm5rF27lhtvvJGEhARWrFhx2Bg8wRQ2Qb/nSt967xgTwV6/FyrWB3efeTPhyn7njTqqDRs28PLLL/Pee+8RHR3NokWLWLJkCZMmTaK6upr1651y1tfXk56ezn/+53/yxBNPMGvWrOCWv48wCvru2DvWpm+MGQHefPNNVq5cSUlJCQBtbW0UFhZyxRVXsHXrVr785S9z1VVXcfnllw9rucIn6PcOw2BB35iIdQJX5ENFVbntttv4wQ9+cMS6devW8frrr/Pzn/+cF198kaeffnrYyhU2vXd6RtkUC/rGmBFg3rx5PP/881RXVwNOL589e/ZQVVWFqnL99dfz/e9/n9WrVwOQkpJCU1PTkJcrbK70eyYcVns5yxgzAsycOZP777+fefPm4fP5iImJ4amnniIqKorPf/7zqCoiwo9//GMAbr31Vm6//XZ7kBsoG1rZGBNq/kMrAyxcuJCFCxcekW/NmjVHpN1www3ccMMNQ1W0XuHTvGOjbBpjzDGFT9AXG3vHGGOOJaCgLyLzRWSriJSJyL1HyXediKg7Py4icpmIrBKR9e7fS4JV8L48dqVvTMSKpJcyB3uuxwz64nSLeRK4EpgG3Cwi0/rJlwLcDSz3S64GPq6qM4FbgP7nCgsC66dvTGSKj4+npqYmIgK/qlJTU0N8fPwJ7yOQB7lzgTJV3QEgIkuABcCmPvl+ADwC9A46oar+Tys2AvEiEqeqHSdc4gEcutIP/x/eGHNIQUEB5eXlVFVVhboowyI+Pp6CgoIT3j6QoJ8P7PVbLgfO9M8gIrOBQlV9RUQGGmnok8Ca/gK+iCwCFgEUFRUFUu4jWPOOMZEpJiaG4uLiUBdj1AikTV/6Seu9nBbnCeqjwNcG3IHIdODHwBf6W6+qT6tqiaqW5OTkBFCkgYsZCbd4xhhzogIJ+uVAod9yAbDfbzkFmAG8LSK7gLOApX4PcwuAl4HPqur2YBS6Px6PB5+KvZFrjDFHEUjQXwlMEZFiEYkFbgKW9qxU1QZVzVbVCao6AfgAuEZVS0UkHXgVuE9V3x2C8vfyCPgQa94xxpijOGbQV9Uu4C5gGbAZeF5VN4rIgyJyzTE2vwuYDHxXRNa6nzGDLnU/RARF7EGuMcYcRUDDMKjqa8BrfdK+N0Dei/y+/xD44SDKFzCPQBcCdA/H4YwxZlQKozdye670Q10SY4wZucIm6AP48FibvjHGHEWYBX3rvWOMMUcTVkEf671jjDFHFVZB34fYJCrGGHMUYRf0rXnHGGMGFlZBX/FYP31jjDmKMAv6WJu+McYcRZgFfQ/WUd8YYwYWVkG/Szx4fF2hLoYxxoxYYRX0O4gl2hf0+VmMMSZshFXQ9xJLtK891MUwxpgRK6yCvnOl7w11MYwxZsQKw6BvzTvGGDOQ8Ar6Ekt08OdcN8aYsBFQ0BeR+SKyVUTKROTeo+S7TkS0Z6pEN+0+d7utInJFMAo9kE6JI7rbgr4xxgzkmJOoiEgU8CRwGc58uStFZKmqbuqTLwW4G1julzYNZ3rF6cA44E0RmaqqQzLTSZcnjihr3jHGmAEFcqU/FyhT1R2q6gWWAAv6yfcD4BHAv/vMAmCJqnao6k6gzN3fkPBFxRFjQd8YYwYUSNDPB/b6LZe7ab1EZDZQqKqvHO+2wdQRlUKCr3modm+MMaNeIEFf+knrHetARDzAo8DXjndbv30sEpFSESmtqqoKoEj9a41JJ0HbYNe7J7wPY4wJZ4EE/XKg0G+5ANjvt5wCzADeFpFdwFnAUvdh7rG2BUBVn1bVElUtycnJOb4z8BPrceuTX191wvswxphwFkjQXwlMEZFiEYnFeTC7tGelqjaoaraqTlDVCcAHwDWqWurmu0lE4kSkGJgCrAj6Wbha49wKIz59qA5hjDGj2jGDvqp2AXcBy4DNwPOqulFEHhSRa46x7UbgeWAT8FfgzqHquQOwPudqAA5mnTFUhzDGmFHtmF02AVT1NeC1PmnfGyDvRX2WHwIeOsHyHZeY6GhW+E6ie88+cofjgMYYM8qE1Ru5PoUGTSZNWkJdFGOMGZHCKugrSgNJpFrQN8aYfoVV0L9w6hgaNIkMsb76xhjTn7AK+pdNyyUpLYsk2qG7M9TFMcaYESesgj5AVFKm86W9IbQFMcaYESjsgr4vwQ36LdWhLYgxxoxAYRf0u5LyANCmihCXxBhjRp6wC/qS7AT9rg0vh7gkxhgz8oRf0M9whvqJWfNr6GgKbWGMMWaECbugn5SYyDc6FzkLLSc+YqcxxoSjsAv6qfHRVKk74FpLTWgLY4wxI0z4Bf2EGGo01VmwK31jjDlM+AX9+BjKNdtZePkL4BuyQT2NMWbUCb+gnxBNHe6Vfkcj7PxnaAtkjDEjSNgF/eykOAC+4P2qk1D2Jvh8ISyRMcaMHAEFfRGZLyJbRaRMRO7tZ/0dIrJeRNaKyDsiMs1NjxGR37jrNovIfcE+gb48HmFqbjJv+05zEt5/AlY8PdSHNcaYUeGYQV9EooAngSuBacDNPUHdz+9VdaaqzgIeAX7mpl8PxKnqTOB04AsiMiFIZR/Qtz82jQ5iaUuf6iTsXzPUhzTGmFEhkCv9uUCZqu5QVS+wBFjgn0FVG/0WkwDtWQUkiUg0kAB4Af+8QyI9IQaA9y7/C0y6FDYvBW/rUB/WGGNGvECCfj6w12+53E07jIjcKSLbca7073aTXwBagAPAHuCnqlo7qBIHIDc1HoD9De1wytXQ2QoN5UN9WGOMGfECCfrST5oekaD6pKpOAr4FfMdNngt0A+OAYuBrIjLxiAOILBKRUhEpraoafN/63FTnYe53/7yRzjT3cM0HB71fY4wZ7QIJ+uVAod9yAbD/KPmXANe63xcCf1XVTlWtBN4FSvpuoKpPq2qJqpbk5OQEVvKjEBE+Mdu5GdnQEOsk1u4Y9H6NMWa0CyTorwSmiEixiMQCNwFL/TOIyBS/xY8B29zve4BLxJEEnAVsGXyxj+1LlzpFuvGFStpSJ8KrX4PXvzUchzbGmBHrmEFfVbuAu4BlwGbgeVXdKCIPisg1bra7RGSjiKwF7gFucdOfBJKBDTiVxzOqui7YJ9Gf7GTnCt9LDD/LfxRSx8Lyp6Bu93Ac3hhjRqToQDKp6mvAa33Svuf3/csDbNeM021z2CXHHTq1GtLh/K/BX74Mvzwf7t0TiiIZY0zIhd0buT1EhN/eNheA9fsa0KnznRXtDdZ90xgTscI26ANcMDWHRRdMZFtlMwd96fDZPzsrNrwQ2oIZY0yIhHXQB5hTlAFATUsHFF8I2SfB0ruhYkOIS2aMMcMv7IN+lvtAt7bFCyJw1U8Ahf/9ztE3NMaYMBT2QT8j0Qn61c0dTsLEC+HS78GOt+Afj4SwZMYYM/zCPugXZiaQlhDDT/66lXXl9U7i3C/A5MvgrYfgtW/aRCvGmIgR9kE/LjqKJxbOpqq5g9t+XeomJsP1z8DMG2DFL2HN70JbSGOMGSZhH/QBzp+Sw50XT6a6uYM2r3tVH5cCC56AMdOc/vsHhuWdMWOMCamICPoARZmJAPzsja2HEqPj4HOvQlya87auMcaEuYgJ+mdPygLgxdX7UPUbJDQx03m4u/V1qByWYYGMMSZkIiboj01L4MEF06lt8bJmb/3hKy/8Jnii4Q+fAj1i1GhjjAkbERP0ARbMyic9MYbF7+w8fEXeTLjs+1BTBn/4NHR1hKaAxhgzxCIq6KclxHDtrHxeWXeAFTv7TOB16o1w9l2w5RV7ccsYE7YiKugDfGXeFLKTY/nenzfQ3unXP98TBVc8BDM+CetfgNYhn9XRGGOGXcQF/fTEWG47r5gtFU1c/Z/v0O3r04Z/zpegoxF+cQ7UbA9NIY0xZohEXNAHuOXsCZw9MYuyymZ+/d6uw1eOmw1XPwZNB+CPt4C3JSRlNMaYoRBQ0BeR+SKyVUTKROTeftbfISLrRWStiLwjItP81p0qIu+7M2utF5H4YJ7AiUiKi+bZ28/k4pNy+OGrm6hoaD88w5zPwMd/DhXr4dkbrKnHGBM2jhn0RSQKZ9rDK4FpwM3+Qd31e1WdqaqzgEeAn7nbRgP/A9yhqtOBi4DO4BX/xHk8wj2XnYQqvLb+wJEZTr8Frn0K9i6HZ6+zK35jTFgI5Ep/LlCmqjtU1QssARb4Z1DVRr/FJKCnofxyYJ2qfujmq1HVETO62fRxqcwuSudHr2+mqqmfbpqzboYFT8K+VbDkU8NfQGOMCbJAgn4+sNdvudxNO4yI3Cki23Gu9O92k6cCKiLLRGS1iHxzsAUOJo9HeOSTp9LZrfx57b7+M51246GhmD9aNrwFNMaYIAsk6Es/aUe8tqqqT6rqJOBbQE9H92jgPOBT7t9/E5FLjziAyCIRKRWR0qqqqoALHwxTclM4ZWwqfywtP7InT48z74CMCfCnL0JTxbCWzxhjgimQoF8OFPotFwD7j5J/CXCt37b/UNVqVW0FXgPm9N1AVZ9W1RJVLcnJyQms5EH0idn5bD3YxHf+NMAUirFJcOOz0NEEv74aWqqHt4DGGBMkgQT9lcAUESkWkVjgJmCpfwYRmeK3+DFgm/t9GXCqiCS6D3UvBDYNvtjB9e8XTOTmuYW8uKqchrYBnjPnzYBP/gpqtsEvL4BtbwxvIY0xJgiOGfRVtQu4CyeAbwaeV9WNIvKgiFzjZrvL7ZK5FrgHuMXdtg6nJ89KYC2wWlVfHYLzGLRPnzUeb7ePRb8tpdXb1X+maQvgtv91Bmd79jpngnXrzmmMGUVER9iokiUlJVpaWhqSY7+0upyv//FDbjyjiB99YubAGfethte/CfvXwuRLYeEfhq+QxhjTDxFZpaolx8oXkW/kDuQTcwq48YwiXl5TTm2Ld+CM+XPg9jfhonvho7/CnuXDV0hjjBkEC/p9fOas8XR2K9/98wAPdf2deQck58FzN8HelTYWvzFmxLOg38e0cancdfFkXl13gDc2HTx65rhkuPU1iIqB/54HT86FDS+Bb8S8f2aMMYexoN+PL140iQlZifz4r1vwDdR3v0fWJLhzOcx7wFl+4VZ44Ta76jfGjEgW9PsRHxPFp84cT1llM/cv3XjsDRIy4Lyvwhffd/5u+hP85uPQMMBbvsYYEyIW9AfwuXMn8Ik5+fzP8t283t+AbP2JioZL74cr/i/s+QAenQbv/z9oDHB7Y4wZYhb0BxAT5eGha2dyakE63/3zBmqaA5w3VwTOvhMWLnGWl90HPzsZlj8NXUfpEWSMMcPAgv5RJMRG8dC1M2hq7+L/vLz++DaePA++uRMu+S6k5sPr34BfXWpDOBhjQsqC/jHMyE/jU2eO562tVYfPqRuIxEy44Ovw1Y3w8cehchMsng9bXoPuAd76NcaYIWRBPwCXT8/F2+XjF2+f4Jy5InD65+C6xdBWC0tuhoeL4I37oXl4RxU1xkQ2C/oBOGtiFtfOGseTb5Wx7WDTie9o2gL4wr/gYz+DiRfCu4/BTyfDU+fDhheDV2BjjBmAjb0ToMrGdi579J8kxESx9EvnMiZlkFP9qsL+Nc7kLKt/C3W7IHMSnPF5mLvIeeHLGGMCFOjYOxb0j8M/PqrilsUrKMhI4MefPJVzJmUh0t8cM8epywtrfgfr/wh73nce/J7ycZh4ERRf4Iznb4wxR2FBf4j8bfNBvvHCOmpbvJw/JZv/+mwJ8TFRwdm5zwebXoaVi53gr90QHQ/FF0L2FEjJgzHTnMrAE6RjGmPCggX9IbRpfyPfeOFDNu5v5KYzCnn4k6cG/yCd7bD3A9j6V/jodeftXp87wUtGMZx0Jcy8HsbNdh4UG2MiWlCDvojMBx4HooBfqerDfdbfAdwJdAPNwCJV3eS3vghnxqwHVPWnRzvWaAj6PX7wyiaeeXcnb339IsZnDUMTTP0e2P532Pgn2P0edHdAQibM+ASccg2MP8eeBRgToYIW9EUkCvgIuAxnztuVwM19gnqqqja6368B/kNV5/utfxHwAcvDKegfaGjjnIf/zkVTc3j6syXERA1jZ6jWWtj8F9j5D9jyKnS1Q3waTP+E82JY8fnOsjEmIgQa9KMD2NdcoExVd7g7XgIswG+u256A70oCemsSEbkW2AG0BFb00WNsWgJfv/wkfrJsK99+eT0PXDOdxNhA/kmDIDETTr/F+XhbYPtbsPEl+PA5WPUMxCRCyW1O88+kS5z8xpiIF0iEygf2+i2XA2f2zSQid+LMjxsLXOKmJQHfwrlL+PpgCzsS/cdFk6hv9fJf/9rJBztqeWLhbE4tSB/eQsQmwSlXOx9vK2xbBmt/D+8/cShPxgTngfCUy513BOJShreMxpgRIZDmneuBK1T1dnf5M8BcVf3SAPkXuvlvEZGfAitU9XkReQBo7q95R0QWAYsAioqKTt+9e/dgzikklu+o4Z7nP6SmpYMnbp7DvGm5oS6S8zC4Yj3sfgfKS2HHP8Db5EzsfvLVMOE8yJrsVAhpBfY8wJhRLJht+mfjPIC9wl2+D0BVfzRAfg9Qp6ppIvIvoNBdlY7Trv89VX2iv21hdLXp97W7poVbn1nJ/oY2nv5MCRdMzQl1kQ7X5YW9y523f7e+Ds0Vh9YlZjvPAk652qkAxs6yXkHGjCLBDPrROA9yLwX24TzIXaiqG/3yTFHVbe73jwP39z340a70/Y3moA9Q2dTOTU9/wK7qFpbedR4z8kfow1RVpzdQ/W6o3ALb/+ZUCG11zvrUAph6BeROg9yZkDwGUsdBdFxoy22M6Vewu2xeBTyG02Vzsao+JCIPAqWqulREHgfmAZ1AHXCXf6Xg7uMBIiDoAzS0dXLuw38nJkq4/+PTmVmQxqSc5FAX69g626B8JexfC2VvOt1Ce94NAOdFscnznK6hqeOg8CxIHRu68hpjetnLWSG2ZMUe7n3p0Bj873zrYgoyEkNYohPQ1QHNB6FiAzTug+qPYPMr0LT/UJ60Iig8AwrPhIIzYOxp9rawMSFgQX8E2Hawie/8aQPLd9aSEBPFf322hPOmZIe6WIOj6jQB1e1ymoP2Loc9yw9VBKn5znhBKXmQXgR5pzpDR8SOsgrPmFHGgv4I8vbWSh56dTM7qlt46YvncFrhMHfpHA4N5c7bwuueh9odzh2Cz50oRjyQPRVyZzjNQkk5kDIWknMgrdCpKGIGOWqpMRHOgv4I09DWycU/fZuU+GgWf+6M0dHGPxg+HzTsdbqMVqyDA+ugciM0HXSGj+grKccJ/mkFTkWQlu88M8ieDAkZw19+Y0YZC/oj0FtbKrl7yRoSYqL4zW1zOWVsaqiLNPxUob3BuRNoPujcITTscyqIxn3ucjl4mw9tk5gNY05x7hISs53nBpnFkDkRkkZ5c5kxQWJBf4T6x0dV3P3cGtq83Sw8s4ivzptKWqK9FHUYVWjc70wyU7fT6U3UsNdJa648/E4hd4bzdnFyrvNmcny6M+REQrrz4llitlMxJGRYd1MT1izoj2CVTe08+sY2ni/dy0m5KTz16dMpyrIHnQHp7nSeGdTtgoMbYOc/nTePmw8669rrD79L8BeX6twlZE1yxiRKyoGoWOfBdFyqU2mIx3m+kJwLyXkQNUxjKRkzSBb0R4E3Nx3ky0vW0OlT7r5kMp85a4Jd9QdDZxu0VDt3Ca21zmT0rbXOncKBtVD1kTMcxbGIB6LinNFKEzL8PulOhZFWcGj4CsW548ia7Nxp2NvMZphZ0B8l9ta28qPXN/Pa+gpiooSHrp3JDWcUHntDc+K6O507g5Yq512ExCzoaHI+qFNpNFU4lURXm/MMoq0OWuucv211zrb+L675SxrjBP/Ucc4neYxTWcSnH6o04tOddHHfaRCxisIMigX9UURVWVfewEOvbmbFrlq+Om8qd10ymSiPBYERy+eDlkp3RrMuJ2C3N0DVFji40RnionGfU3F0e4++L4/bhBSX4jQzxaVCfCrEJjtNTj2fuFRQn9P8FJ/u5ktzP37fY5OtAolAFvRHoZaOLr76h7X876aDnFaQxqILJjF/Rp4F/9FMFToa3TuEeueZQ8/dQsM+5+3lbq/TlNTe6OTt+ettduZK8LY63zsanXzqO/oxxeNXIaQ6FYQIRCc4XWHj3AolrqdSSe5/uaeyEXHOwyqSEc2C/iilqvyxtJwn3y5jd00r47MS+cWnTmfauAjs3mkO5/OBx+OMltrR6NxZ+H8OS+uzXn2Husp6m49999FLnF5PXe0Qk+Q0T8UkAAqeGL+7kJRD36MTnLsf7XbSE7OcaT0Ts5y7FF+3M8lPT/74dKeCsofmg2JBf5Tr9ilvbKrggaWbqG7uYN4pufz7BRM5fby9qGSCoMvrdyfh/u1ocpdbnAfd3hboaHYCfnQ8dLY6dyjeFueq39d1KH9H86H9dHU4lZN4nPSBnn30FZfqHCc6zulV1fNXu3Eqn3i3aSvNqUCiE5xjiDhlbKpw9uHrdLaNS3Uqneh4txLyOU1pUbHuJ+bw79Fxzv56K65O5w4nuueYmYfydHudZ0O+Lmd9bHLIKy0L+mFib20rv/rXDv5n+R66fUpBRgI/vHYGF0zJwWPNPmakU3Uqk9Ya59PV4TRpeVucSsTb4tfsVe8E764O512Mrg636SsKUGddp/tgvWdfqm4wj3J6VHV7ne9dHW5FFEAvrWCIinWfs/Q0laU4lYLXrQwl6tBAhD1Nb+JxKpaeiqe7E4rOhAu+cUJFsKAfZmpbvLy0upyn/rHDvfIfw0+vP430xNhQF82YkcvncyuC6EN3J92dh67Uu73Op6eC0W63Qmp3rv5FnHVt9U7X364Op5LpuQsBJ62l0snjbT501xMVc+j5SM+dBjiVYHu908wVFesc19fpVAxTLoNLvnNCp2pBP0w1d3Txm/d28dibH5GfnsAPr505+kfuNMYMWqBB3zMchTHBkxwXzZ0XT+bZ288C4NP/vZxbn1lBWeUw3cYaY0a1gIK+iMwXka0iUiYi9/az/g4RWS8ia0XkHRGZ5qZfJiKr3HWrROSSYJ9ApJpbnMlfv3IBX5k3hdJddVzx2L/40eubqWhoD3XRjDEjWCBz5EbhzJF7GVCOM0fuzaq6yS9Pqqo2ut8qlL8gAAATW0lEQVSvAf5DVeeLyGzgoKruF5EZwDJVzT/a8ax55/jtr2/joVc38/qGA0R7PFw1M4+zJ2Vx7ex84qJtFitjIkGgzTuB9DGaC5Sp6g53x0uABUBv0O8J+K4knJFIUNU1fukbgXgRiVPVfgZUNydqXHoCT35qDntrW/l/b2/n5TXl/Gntfn7+tzK+Mm8KV80cS1Kc9YE2xgR2pX8dMF9Vb3eXPwOcqap39cl3J3APEAtcoqrb+tnPHao6r59jLAIWARQVFZ2+e/fuEz8jQ1e3j39tq+bRNz9iXXkDHoHTCtM5vSiDK2bkUTI+A7G3K40JK0HrvSMi1wNX9An6c1X1SwPkX+jmv8UvbTqwFLhcVbcf7XjWvBM8qsq/tlWzclct/9pWzeYDjXR0+SjKTOS+K09m/ow8C/7GhIlgNu+UA/7DPhYA+4+SfwnwC7+CFAAvA589VsA3wSUiXDA1hwum5vC1y0+ipaOLZRsreOzNbXzx2dXMLkpnwWnjuPSUXAozbTx/YyJBIFf60TgPci8F9uE8yF2oqhv98kzpac4RkY8D96tqiYikA/8AHlTVFwMpkF3pDz1vl48/rNzDb9/fzbbKZqI8wsUnjeG60wu4YnquXf0bMwoF9eUsEbkKeAyIAhar6kMi8iBQqqpLReRxYB7QCdQBd6nqRhH5DnAf4N++f7mqVg50LAv6w0dV2V3TynMr9vDi6n1UN3dw+bRcbjuvmDOLMy34GzOK2Bu55rh0dfv473d28pNlW+nyKXOK0vnSJVO4+OQxoS6aMSYAFvTNCWlo7eTPH+7jF29v50BDOwtmjePcydlcOSOPlHibytGYkcqCvhmUxvZO/u+rm3lt/QEa27tIio3i9AmZXD1zLFfOtArAmJHGgr4JClVl9Z46Xli1j/e2V7O7ppXYKA9FWYksumCi3QEYM0JY0DdBp6qs3VvPa+sP8LctleyoaiEu2sPM/DROK0zn9vOLGZuWEOpiGhORLOibIeXzKWvL6/nzmn1s3N/I2r31AFx96liumjmWucWZNta/McMomC9nGXMEj0eYU5TBnCJn+sbdNS389v3dLFmxhz+t3Y8InD8lh4Vzizh9fAY5KXEhLrExBuxK3wRZe2c368obeGdbFb9fsZfqZmdsvXMnZ/HJOQWcWpDGxOxkm+rRmCCz5h0Tch1dTgXwXlkNz5fuZV99GwCp8dHMLc5kbnEmV0zPY3xWUohLaszoZ0HfjCg+n7LpQCObDzSyancdy3fWsrO6BYDi7CSKs5M4ZWwKswozOK0wjTEp8SEusTGji7XpmxHF4xFm5KcxIz+N60uc8fv21bfx2roDlO6uZVd1K//4qIpun3MRkpcaz2XTcjl3chYXTh1DQqxNBmNMMNiVvhkx2rzdbNzfwNq99bxbVs2722vwdvkYmxbPv58/kcum2WigxgzEmnfMqNfZ7eO97TX87I2P+NDtEjozP42rZo7lqpn2LMAYfxb0TVjZWd3CG5sqeHV9RW8FMGVMMteXFDC7KIMZ49KsCchENAv6JmztrW3lzc0HeWFVORv3O9MzewQKMhI5tSCNOUUZzC3OZPq4VBse2kQMC/omIlQ1dfDh3nrW7Wtge2Uzq/fUcaChHYAZ+alcMCWH/IwEMhNjmT4ujcLMBKsITFgKau8dEZkPPI4zicqvVPXhPuvvAO4EuoFmYJGqbnLX3Qd83l13t6ouO54TMeZoclLimDctl3nTcnvTDjS0sWxDBX9Zd4Cn/7mDLt+hC5vU+GhmFWVwelEGc8anM6sw3QaMMxElkOkSo3CmS7wMZ77clcDNPUHdzZOqqo3u92uA/1DV+SIyDXgOmAuMA94Epqpq90DHsyt9E0wNrZ00tndS2+Jl4/5G1u+rZ82eerYebEIVROCk3BTmjM/gjAkZlIzPpCDD7gbM6BPMK/25QJmq7nB3vARYAPQG/Z6A70oCemqSBcASVe0AdopImbu/9wM6C2MGKS0xhrTEGAozEzmtMB0oApz5Aj7cW8+q3XWs2l3HX9bu5/fL9wCQGBtFQUYC507O5owJmZxWmM64tHirCExYCCTo5wN7/ZbLgTP7ZhKRO4F7gFjgEr9tP+izbf4JldSYIEqNj+H8KTmcPyUHgG6fsqWikdV76tlR1cxHB5t4dvkennl3FwBjUuI4a2IWc4szmZSTzKkFaSTF2buNZvQJ5H9tf5c3R7QJqeqTwJMishD4DnBLoNuKyCJgEUBRUVEARTImuKI8wvRxaUwfl9ab1u1TVu2uY2tFIyt31fH+jhqWfri/N//JeSnMKcrg1II08tLimZCVZC+PmREvkKBfDhT6LRcA+4+Sfwnwi+PZVlWfBp4Gp00/gDIZM+SiPNI7MNxnzp6AqrKvvo1tlc2s3l3H6j11vLS6nN99sLt3m+LsJC6cmsPFJ49hVmE6aQn2kNiMLIEE/ZXAFBEpBvYBNwEL/TOIyBRV3eYufgzo+b4U+L2I/AznQe4UYEUwCm7McBMRCjISKchI5OKTxgDO3cCe2laqmjrYtL+Btz+q4rkVe/j1e7sAyEyKZUJWIrMKnd5CM/PTyE9PIDrKE8IzMZEsoH76InIV8BhOl83FqvqQiDwIlKrqUhF5HJgHdAJ1wF2qutHd9tvAbUAX8BVVff1ox7LeO2a0a+/s5oMdNWytaGJXTSvbK5v5sLyeji4f4HQznVOUzvRxaYxLT+CUsSlMzU0hxioCMwj2cpYxI4i3y8eWikY+LG/g/e3VbKloYkdVS+/6uGgP08elcmpBOqcVpjEzP53i7CSibLIZEyAL+saMcPWtXqdZ6EAj68obWFdez4Z9jbR1Oq+xZCTGcGpBOlnJsaTGxzAlN5kzi7MoyEggPsbGGTKHs/H0jRnh0hNjSU+MZUpuCgtmOT2Zu7p9lFU1s768gfe317CtspmyymbqWr20ep3KIDbaw0m5KeSlxXNKXgpFWUmcnJfClNxk4qKtMjBHZ0HfmBEkOsrDyXmpnJyX2jvZDDgzj+2obmH17jqnaai6mR1Vzby5+SA9N+siuMNLZFCYmUhRZiIz89PISIyxF8tMLwv6xowCHo8weUwyk8ckH5bu7fKxp7aV9fvq2bivkZW76/j1u7vwdvt686TERzMhK4nxWYlMyEpiQnYSE7ISGZ+VRHZyrFUIEcaCvjGjWGy0p7cy+LfZTlpnt4+6Fi+bK5rYdrCJPbWt7KppZV15A6+tP4Df+HMkxUYxPiuJCdmJFGU6lUFRViKFGYmMTYu3rqVhyIK+MWEmJsrDmNR4xqTGc+HUnMPWebt87KtvY1dNC7urW9hV08rumha2HGjijU0H6ew+VCNEeYSxafEUZiRSmJlAUWYiE7KTOHtiFlnJccN9WiZILOgbE0Fioz0UZydRnJ0EJx2+rtun7K9vY09tK+V1reytbWNvXSt7a1t5a2sVVU0dvXnHpsVz9qQsZhemc1JeKiflpdjbx6OEBX1jDOBc2RdmJg44flCbt5stFY2U7qrjw/J6/r6lkpdW7+tdn50cS05KPJlJMZyUm9p7d1Dk7tO6mY4MFvSNMQFJiI1idlEGs4syAFBVDjS0s7WiiY8ONrldSzs52NjOkpV7eruY9hiTEtdbARRnJzEuPYHE2CgSY6NIS4jh5LxUm+d4GFjQN8acEBFhXHoC49ITuPjkMYetU1VqWrzsqXWah/bWtrLH/SzfUcPLa/b1u8/MpFhyU+MZlxZPcnw0GYmxZCbFkp0cx+wi5y1lu2MYHAv6xpigExGyk+PITo5jjntn4K/V20VVUwet3m5avd3UNHewpaKJisZ2Dja0s7+hnZbKLupavTS1dx22bX56ArMK0zk5L4WirETy3IfWY1LibI6DANi/kDFm2CXGRjM+6/Dwc/n0vH7zert8HGxsp3R3LeW1bWw92MSH5fW8uv7AEXmT46IZkxJHTkoceWnxjE1LYExKHNkpcYxJiWNidhI5KXER/W6CBX1jzIgWG+3p9wFze2c35XWtVDR0UNnUzsFG52+l+3fV7joONh44rBsqQEpcNBNzkshNjacwM5GpucmkJ8aSkRhLRmIM6W6TUrgOdmdB3xgzKsXHRDF5TAqTx6QMmMfnU+rbOqlu7qCioZ2d1S1sr2pmR1ULu2ta+fuWSrp8Rw46GRftITs5jkljkpk+LpUpY5JJS4hhbFoC49LjSUsYvUNbWNA3xoQtj0fITHKu3KfmpnBBn5fV2ju7qWnxUtfipb61k7pWL3WtXnbXtFLT3MHWg8381z93HFExxEZ7yEmOIyvZ2XdOchy5qfGkJ8aQHBdNcnw049ITKMxIHHFDXVjQN8ZErPiYKPLTE8hPTxgwT3tnN/vr22ho62R/fTsVje1UNrZT1dRBbauXmmYvmw80UtXUQT83DSTERPW+szDeHQMpKymOjMQYPB4hJyWO/PThGy47oKAvIvOBx3FmzvqVqj7cZ/09wO04s2NVAbep6m533SM4Uyh6gDeAL+tIG8TfGGMGEB8TxcQcZ6C72UUD5+v2Kc0dXbR0dNHY3sn++jbnrebaVna73VbfLavpnS+hrzEpcVxz2ji+c/W0oTiNXscM+iISBTwJXIYz0flKEVmqqpv8sq0BSlS1VUS+CDwC3Cgi5wDnAqe6+d4BLgTeDt4pGGNM6EV5hLSEGNISYhhHAifnpR6RR1WpbvZS2+J8un1KVXN7b+Uw9ih3HMESyJX+XKBMVXcAiMgSYAHQG/RV9S2//B8An+5ZBcQDsYAAMcDBwRfbGGNGHxGnOScnJXQD1gUybmo+sNdvudxNG8jngdcBVPV94C3ggPtZpqqb+24gIotEpFRESquqqgItuzHGmOMUSNDv77Fzv23yIvJpoAT4ibs8GTgFKMCpKC4RkQuO2Jnq06paoqolOTk5fVcbY4wJkkCCfjlQ6LdcAOzvm0lE5gHfBq5R1Z4xWP8N+EBVm1W1GecO4KzBFdkYY8yJCiTorwSmiEixiMQCNwFL/TOIyGzglzgBv9Jv1R7gQhGJFpEYnIe4RzTvGGOMGR7HDPqq2gXcBSzDCdjPq+pGEXlQRK5xs/0ESAb+KCJrRaSnUngB2A6sBz4EPlTVvwT7JIwxxgRGRlqX+ZKSEi0tLQ11MYwxZlQRkVWqWnKsfDbrsTHGRBAL+sYYE0FGXPOOiFQBuwexi2ygOkjFGQ0i7XzBzjlS2Dkfn/Gqesw+7yMu6A+WiJQG0q4VLiLtfMHOOVLYOQ8Na94xxpgIYkHfGGMiSDgG/adDXYBhFmnnC3bOkcLOeQiEXZu+McaYgYXjlb4xxpgBhE3QF5H5IrJVRMpE5N5QlydYRKRQRN4Skc0islFEvuymZ4rIGyKyzf2b4aaLiPzc/XdYJyJzQnsGJ0ZEokRkjYi84i4Xi8hy93z/4I4DhYjEuctl7voJoSz3YIhIuoi8ICJb3N/77Aj4nb/q/r/eICLPiUh8uP3WIrJYRCpFZINf2nH/riJyi5t/m4jccqLlCYug7ze715XANOBmERnaOceGTxfwNVU9BWeE0jvdc7sX+JuqTgH+5i6D828wxf0sAn4x/EUOii9z+OB8PwYedc+3DmfeBty/dao6GXjUzTdaPQ78VVVPBk7DOf+w/Z1FJB+4G2fWvRk407HeRPj91r8G5vdJO67fVUQygfuBM3Emtrq/p6I4bqo66j/A2TgTtPQs3wfcF+pyDdG5/hln6sqtwFg3bSyw1f3+S+Bmv/y9+UbLB2f47r8BlwCv4MzpUA1E9/29cQYCPNv9Hu3mk1Cfwwmccyqws2/Zw/x37pmgKdP97V4BrgjH3xqYAGw40d8VuBn4pV/6YfmO5xMWV/oc/+xeo5J7OzsbWA7kquoBAPfvGDdbOPxbPAZ8E/C5y1lAvTojvsLh59R7vu76Bjf/aDMRqAKecZu1fiUiSYTx76yq+4Cf4gzBfgDnt1tF+P/WcPy/a9B+73AJ+gHP7jVaiUgy8CLwFVVtPFrWftJGzb+FiFwNVKrqKv/kfrJqAOtGk2hgDvALVZ0NtHDolr8/o/683eaJBUAxMA5Iwmne6CvcfuujGegcg3bu4RL0A5rda7RyJ6B5EXhWVV9ykw+KyFh3/VigZ/Ka0f5vcS5wjYjsApbgNPE8BqSLSLSbx/+ces/XXZ8G1A5ngYOkHChX1eXu8gs4lUC4/s4A84Cdqlqlqp3AS8A5hP9vDcf/uwbt9w6XoH/M2b1GKxER4L+Bzar6M79VS4GeJ/i34LT196R/1u0FcBbQ0HMbORqo6n2qWqCqE3B+x7+r6qeAt4Dr3Gx9z7fn3+E6N/+ou/pT1Qpgr4ic5CZdCmwiTH9n1x7gLBFJdP+f95xzWP/WruP9XZcBl4tIhnuHdLmbdvxC/YAjiA9KrgI+wpmp69uhLk8Qz+s8nNu4dcBa93MVTlvm34Bt7t9MN7/g9GTqmbGsJNTnMIhzvwh4xf0+EVgBlAF/BOLc9Hh3ucxdPzHU5R7E+c4CSt3f+k9ARrj/zsD3gS3ABuB3QFy4/dbAczjPLDpxrtg/fyK/K3Cbe+5lwK0nWh57I9cYYyJIuDTvGGOMCYAFfWOMiSAW9I0xJoJY0DfGmAhiQd8YYyKIBX1jjIkgFvSNMSaCWNA3xpgI8v8BQhrd17LITMEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "results = xgb.evals_result()\n",
    "epochs = len(results['validation_0']['error'])\n",
    "x_axis = range(0, epochs)\n",
    "# plot log loss\n",
    "fig, ax = plt.subplots()\n",
    "ax.plot(x_axis, results['validation_0']['error'], label='Train')\n",
    "ax.plot(x_axis, results['validation_1']['error'], label='Test')\n",
    "ax.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, '2-class Precision-Recall curve: AP=0.62')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAG2RJREFUeJzt3XuYJHV97/H3x10uRtZFXTC6LKxyiRJE0Q1ITBQjcoAoeIxHWW9gCGgiUU/UxHOS6EriMeoxHhPxQoJREUXgMWajKDGIgkbMLoeLgkJWBFnBCAiLXOT6zR9Vw7azMzU9w9b07PB+Pc8801316+pv//ryqfpVV3WqCkmSJvOQURcgSZrbDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng2ILluSoJF8fdR2bW5JLkxwwRZudk9yaZMEsldW7JFclObC9vCrJJ0ddkwQGxaxLsk2Sk5JcneRnSS5Mcsio6xpG+0F2R/sB/Z9J/iHJdpv7fqrqV6vqq1O0+WFVbVdV927u+28/pO9uH+fNSf4tyf6b+34eLJJ8LMk9SR47bvpm6eckL23fT7cl+VySR3a0XZDkL5NcO/D+276dd2SSC5LckmR9kncnWTj9Rzz/GBSzbyFwDfAsYDHw58BpSZaPsKbpeH5VbQc8Ffg14M/GN0hjS39tfaZ9nEuAc4DTR1zPZjcbH4JJHgb8DrABeNkETcb6eQfg68Bnk2Qay/9V4CPAK4BHA7cDH+y4yduBXwf2Bx7e3u7n7bxfAt5A85zvBzwHeNOwtcxnW/qbeYtTVbdV1aqquqqq7quqzwM/AJ422W2SLEvy2STXJ7kxyQcmaff+JNe0a0QXJPnNgXn7JlnbzvvPJH/dTt82ySfb5d6cZE2SRw/xOH4EfBHYq13OV5O8I8k3aN6sj0+yuN16ui7Jj9o1ufuHipIck+S77ZrdZUme2k4fHIKZrO7lSWrswy7JY5OsTvLTJOuSHDNwP6uSnJbkE+19XZpkxVSPsX2c9wCnAEuT7DCwzOcluWhgTXjvgXkTPl9Jdk3ylXbaDUlOGVubna4kh7f3f0uS7yc5eHzfDTz2T47rs6OT/BD4SpIvJTlu3LIvTvLC9vITkny57dfLk7x4mqX+DnAzcDxw5GSNqupu4OPALwOPmsbyXwb8c1WdW1W30qx4vTDJovENkzyCJgiOqaqrq/Gdqvp5W8OHquq8qrqrfX2fAjxjGrXMWwbFiLUfynsAl04yfwHweeBqYDmwFDh1ksWtAZ4CPBL4FHB6km3bee8H3l9VDwd2BU5rpx9Js2WzjOYN+hrgjiHqXgYcClw4MPkVwLHAorbejwP3ALsB+wAHAb/X3v5/AKuAV9Ks2R0G3DjBXU1W93ifBtYDjwVeBPyfJM8ZmH8YTb9tD6wGJgzbCR7n1m2NNwI3tdOeCnwUeDVNn30EWJ1mWLHr+QrwzrbGJ9L0+aph6hhX077AJ4A3t4/nmcBV01jEs9r7/280r5OVA8veE9gF+EK7NfDlts2ObbsPtmvxY0M+l0xxX0fSPDenAk8YWxmY4DFtAxwFrK+qG5L8RhvCk/39RnvTXwUuHltOVX0fuIvmPTXek2hejy9K8uMkVyR5bUftz2SS9+WDTlX5N6I/YCvgX4GPdLTZH7geWDjBvKOAr3fc9ibgye3lc2k2u5eMa/O7wL8Bew9R71XArTRriFfTbOI/tJ33VeD4gbaPBu4cm99OWwmc014+C3h9x/0cOEXdy4GiGcpbBtwLLBqY/07gY+3lVcC/DszbE7ij43Guovmwubld7o3AAQPzPwT8xbjbXE7zATzp8zXB/bwAuHCSx70K+OQkt/sI8L6p+m78cgb67PED8xcBtwG7tNffAXy0vfwS4LwJ7vttQ76+dwbuA54y8Jy/f5J+/gnwFeBp03wPnQ28Zty0Hw0+XwPTX9o+/pOAhwJ7t8/Vcydo+yqaFY8l06lnvv65RTEiacbwT6Z5oxw3MP2LaXbu3ZrkZTQfgldXMwQy1TLf2A7lbEhyM82WwpJ29tE0a1nfa4eXntdOP5nmDXxqmh18706yVcfdvKCqtq+qXarqD6pqcOvjmoHLu9AE4XVja4E0HzI7tvOXAd+f6jF11D3oscBPq+pnA9OuplmbH/Pjgcu3A9smWZjkZQP9/cWBNqdV1fY0gfcdfnFocBfgjYNruO3jeSwdz1eSHZOc2g7D3QJ8ko3Pz3QM23eTuf95avvsC8AR7aQjaIZcoHmc+417nC+jGR4axiuA71bVRe31U4CXjnt9nda+nnasqt+qqgum+VhupdkiHfRw4GcTtB17rR5fVXdU1SU0WzqHDjZK8gLgr4BDquqGadYzL7lHfwSShGat5tHAodWMzwJQVYeMa7s/sHOShV1hkWZ/xJ/Q7IC7tKruS3ITzXAHVfUfwMo2oF4InJHkUVV1G80a+9vT7FA/k2bt+KQZPLTBUxFfQ7NFsWSSuq+hGUrqXuAkdY9rdi3wyCSLBsJiZ5o1y6mWfwobPxgnmn9DklcDa5J8qqqua2t/R1W9Y3z7KZ6vd9L00d5VdWP7gTTUENg4XX13G81O2TETfaiPP2X0p4G3JTmXZk37nIH7+VpVPXcGNUIzZLdzkrGQXkgzVHcIzfDfpNrX8xc7mhxSVefRDA09eeB2jwe2Aa6Y4DZjw2STnjK73dfzd8BvV9W3u2p8MHGLYjQ+RDNG/Pxxa+QT+XfgOuCvkjwszc7niXawLaIZf70eWJjkrQysaSV5eZIdquo+mk19gHuTPDvJk9qx9VuAu2mGWx6Q9gP1X4D3Jnl4koek2Zn7rLbJ3wNvSvK0NHZLssv45UxW97j7uoZm+Oydbf/sTbMlMmkATPOxfI9mq+uP20l/B7wmyX5t7Q9L8tvtDtSu52sR7dBdkqU0+xhm4iTgVUme0/br0iRPaOddBByRZKs0O+xfNMTyzqTZejie5ltI97XTPw/skeQV7fK2SvJrSZ441QLbwNwV2Jdmv9lTaL748Ck6dmqPqWan8nYdf+e1TU8Bnp/kN9t9KscDnx23dTm2zO8D5wF/mmZ/0hNphtc+39b8W+3yfqeq/n2qGh9MDIpZ1n4YvprmjfPjccNMm6jmOIHn0+wQ/iHNuOlLJmh6Fs0a2BU0wy4/5xeHgg4GLk1yK80O4iOq+bbHLwNn0ITEd4Gv0QyJbA6vBLYGLqPZX3IG8Jj2cZ1OMx7+KZphgs/R7IQfb7K6x1tJMwZ/LfCPNOPoX95MjwPgPcCxSXasqrXAMTRbAzcB62j2F031fL2d5mvFG2iGez47k0LaD7FXAe9rl/U1mg96aL71s2tb19tp+neq5d3Z1nLgYPv2w/YgmuGoa2mG795Fs8ZOO2w32c7eI4F/qqpvV9WPx/5onsPnpeNYh+moqktpvoBxCs1+jkXAH4zNb4dy//fATVbS9NWNNM/Bn1fV2e28P6cZrj1zkuHIB61U+cNFkqTJuUUhSepkUEiSOhkUkqROBoUkqdMWdxzFkiVLavny5aMuQ5K2KBdccMENVbXD1C03tcUFxfLly1m7du2oy5CkLUqSq2d6W4eeJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVKn3oIiyUeT/CTJdyaZnyR/k+b3jS/JJD+RKEkarT63KD5Gc4royRwC7N7+HUvzGw2SpDmmt6CoqnOBn3Y0ORz4RDXOB7ZP8piplnvXXZurQknSMEa5j2Ipv/jDOuv5xd84vl+SY5OsTbL2uutumpXiJEmNUQZFJpg24a8oVdWJVbWiqlYsXvyInsuSJA0aZVCsB5YNXN+J5ucWJUlzyCiDYjXwyvbbT08HNlTVdSOsR5I0gd7OHpvk08ABwJIk64G3AVsBVNWHgTOBQ2l+mP52mh+LlyTNMb0FRVWtnGJ+Aa/t6/4lSZuHR2ZLkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSerUa1AkOTjJ5UnWJXnLBPN3TnJOkguTXJLk0D7rkSRNX29BkWQBcAJwCLAnsDLJnuOa/RlwWlXtAxwBfLCveiRJM9PnFsW+wLqqurKq7gJOBQ4f16aAh7eXFwPX9liPJGkG+gyKpcA1A9fXt9MGrQJenmQ9cCbwhxMtKMmxSdYmWbthw0191CpJmkSfQZEJptW46yuBj1XVTsChwMlJNqmpqk6sqhVVtWLx4kf0UKokaTJ9BsV6YNnA9Z3YdGjpaOA0gKr6JrAtsKTHmiRJ09RnUKwBdk/yuCRb0+ysXj2uzQ+B5wAkeSJNUFzfY02SpGnqLSiq6h7gOOAs4Ls03266NMnxSQ5rm70ROCbJxcCngaOqavzwlCRphLKlfS7vttuKWrdu7ajLkKQtSpILqmrFTG7rkdmSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKnTwlEXMF333QdXXDHqKqT+PfKRsMRzKWsO2OKCAuDcc0ddgdSvO+9sgmLlylFXIm2BQbFwIeyzz6irkPp19dXw05+Ougqp4T4KSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktRp6J9CTbIU2GXwNlXlr1dL0jw3VFAkeRfwEuAy4N52cgGdQZHkYOD9wALg76vqryZo82JgVbu8i6vqpcMWL0nq37BbFC8AfqWq7hx2wUkWACcAzwXWA2uSrK6qywba7A78L+AZVXVTkh2HL12SNBuG3UdxJbDVNJe9L7Cuqq6sqruAU4HDx7U5Bjihqm4CqKqfTPM+JEk9G3aL4nbgoiRnA/dvVVTV6zpusxS4ZuD6emC/cW32AEjyDZrhqVVV9aUha5IkzYJhg2J1+zcdmWBaTXD/uwMHADsB5yXZq6pu/oUFJccCxwLsuOPO0yxDkvRADBUUVfXxJFvTbgEAl1fV3VPcbD2wbOD6TsC1E7Q5v13WD5JcThMca8bd/4nAiQB77LFifNhIkno01D6KJAcA/0Gzc/qDwBVJnjnFzdYAuyd5XBsyR7DpVsnngGe397GEJoiuHLp6SVLvhh16ei9wUFVdDpBkD+DTwNMmu0FV3ZPkOOAsmv0PH62qS5McD6ytqtXtvIOSjH3t9s1VdePMH44kaXMbNii2GgsJgKq6IsmU34KqqjOBM8dNe+vA5QL+qP2TJM1BwwbF2iQnASe3118GXNBPSZKkuWTYoPh94LXA62i+zXQuzb4KSdI8N+y3nu4E/rr9kyQ9iHQGRZLTqurFSb7NpsdAUFV791aZJGlOmGqL4vXt/+f1XYgkaW7qPI6iqq5rL94AXFNVVwPbAE9m04PnJEnz0LAnBTwX2Lb9TYqzgVcBH+urKEnS3DFsUKSqbgdeCPxtVf13YM/+ypIkzRVDB0WS/WmOn/hCO23oX8eTJG25hg2KN9D8wNA/tqfheDxwTn9lSZLmimGPo/ga8LWB61fSHHwnSZrnpjqO4v9V1RuS/DMTH0dxWG+VSZLmhKm2KMbO7fR/+y5EkjQ3dQZFVY2d+G8tcEdV3QeQZAHN8RSSpHlu2G8unQ0cCNzaXn8o8C/Ar/dRlCS480644opRVyENHxTbVtVYSFBVtyb5pZ5qkh70Fi+GH/8Yzj131JVo/lj0sJnectiguC3JU6vq/wMkeRpwx0zvVFK37beH/fYbdRWaXxYsmOkthw2KNwCnJxk7v9NjgJfM9E4lSVuOYY+jWJPkCcCv0Pxw0feq6u5eK5MkzQlDHZnd7o/4E+D1VfVtYHkSTz0uSQ8Cw57C4x+Au4D92+vrgb/spSJJ0pwybFDsWlXvBu4GqKo7aIagJEnz3LBBcVeSh9KexiPJrsCdvVUlSZozhv3W09uALwHLkpwCPAM4qq+iJElzx5RBkSTA92h+tOjpNENOr6+qG3quTZI0B0wZFFVVST5XVU9j448WSZIeJIbdR3F+kl/rtRJJ0pw07D6KZwOvSXIVcBvN8FNV1d59FSZJmhuGDYpDeq1CkjRnTfULd9sCrwF2A74NnFRV98xGYZKkuWGqfRQfB1bQhMQhwHt7r0iSNKdMNfS0Z1U9CSDJScC/91+SJGkumWqL4v4zxDrkJEkPTlMFxZOT3NL+/QzYe+xyklumWniSg5NcnmRdkrd0tHtRkkqyYroPQJLUr86hp6qa8S8iJVkAnAA8l+Zss2uSrK6qy8a1WwS8DvjWTO9LktSfYQ+4m4l9gXVVdWVV3QWcChw+Qbu/AN4N/LzHWiRJM9RnUCwFrhm4vr6ddr8k+wDLqurzXQtKcmyStUnWbthw/eavVJI0qT6DYqLfq6j7ZyYPAd4HvHGqBVXViVW1oqpWLF68w2YsUZI0lT6DYj2wbOD6TsC1A9cXAXsBX21PDfJ0YLU7tCVpbukzKNYAuyd5XJKtgSOA1WMzq2pDVS2pquVVtRw4Hzisqtb2WJMkaZp6C4r2uIvjgLOA7wKnVdWlSY5Pclhf9ytJ2ryGPSngjFTVmcCZ46a9dZK2B/RZiyRpZvocepIkzQMGhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjr1GhRJDk5yeZJ1Sd4ywfw/SnJZkkuSnJ1klz7rkSRNX29BkWQBcAJwCLAnsDLJnuOaXQisqKq9gTOAd/dVjyRpZvrcotgXWFdVV1bVXcCpwOGDDarqnKq6vb16PrBTj/VIkmagz6BYClwzcH19O20yRwNfnGhGkmOTrE2ydsOG6zdjiZKkqfQZFJlgWk3YMHk5sAJ4z0Tzq+rEqlpRVSsWL95hM5YoSZrKwh6XvR5YNnB9J+Da8Y2SHAj8KfCsqrqzx3okSTPQ5xbFGmD3JI9LsjVwBLB6sEGSfYCPAIdV1U96rEWSNEO9BUVV3QMcB5wFfBc4raouTXJ8ksPaZu8BtgNOT3JRktWTLE6SNCJ9Dj1RVWcCZ46b9taBywf2ef+SpAfOI7MlSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnXoNiiQHJ7k8ybokb5lg/jZJPtPO/1aS5X3WI0mavt6CIskC4ATgEGBPYGWSPcc1Oxq4qap2A94HvKuveiRJM9PnFsW+wLqqurKq7gJOBQ4f1+Zw4OPt5TOA5yRJjzVJkqZpYY/LXgpcM3B9PbDfZG2q6p4kG4BHATcMNkpyLHBse+3uFSsecVUvFW9x7lwM22wYdRVzg32xkX2xkX2x0S07zfSWfQbFRFsGNYM2VNWJwIkASdZW3bTigZe35Wv64nb7AvtikH2xkX2xUZK1M71tn0NP64FlA9d3Aq6drE2ShcBi4Kc91iRJmqY+g2INsHuSxyXZGjgCWD2uzWrgyPbyi4CvVNUmWxSSpNHpbeip3edwHHAWsAD4aFVdmuR4YG1VrQZOAk5Oso5mS+KIIRZ9Yl81b4Hsi43si43si43si41m3BdxBV6S1MUjsyVJnQwKSVKnORsUnv5joyH64o+SXJbkkiRnJ9llFHXOhqn6YqDdi5JUknn71chh+iLJi9vXxqVJPjXbNc6WId4jOyc5J8mF7fvk0FHU2bckH03ykyTfmWR+kvxN20+XJHnqUAuuqjn3R7Pz+/vA44GtgYuBPce1+QPgw+3lI4DPjLruEfbFs4Ffai///oO5L9p2i4BzgfOBFaOue4Svi92BC4FHtNd3HHXdI+yLE4Hfby/vCVw16rp76otnAk8FvjPJ/EOBL9Icw/Z04FvDLHeublF4+o+NpuyLqjqnqm5vr55Pc8zKfDTM6wLgL4B3Az+fzeJm2TB9cQxwQlXdBFBVP5nlGmfLMH1RwMPby4vZ9JiueaGqzqX7WLTDgU9U43xg+ySPmWq5czUoJjr9x9LJ2lTVPcDY6T/mm2H6YtDRNGsM89GUfZFkH2BZVX1+NgsbgWFeF3sAeyT5RpLzkxw8a9XNrmH6YhXw8iTrgTOBP5yd0uac6X6eAP2ewuOB2Gyn/5gHhn6cSV4OrACe1WtFo9PZF0keQnMW4qNmq6ARGuZ1sZBm+OkAmq3M85LsVVU391zbbBumL1YCH6uq9ybZn+b4rb2q6r7+y5tTZvS5OVe3KDz9x0bD9AVJDgT+FDisqu6cpdpm21R9sQjYC/hqkqtoxmBXz9Md2sO+R/6pqu6uqh8Al9MEx3wzTF8cDZwGUFXfBLYFlsxKdXPLUJ8n483VoPD0HxtN2RftcMtHaEJivo5DwxR9UVUbqmpJVS2vquU0+2sOq6oZnwxtDhvmPfI5mi86kGQJzVDUlbNa5ewYpi9+CDwHIMkTaYLi+lmtcm5YDbyy/fbT04ENVXXdVDeak0NP1d/pP7Y4Q/bFe4DtgNPb/fk/rKrDRlZ0T4bsiweFIfviLOCgJJcB9wJvrqobR1d1P4bsizcCf5fkf9IMtRw1H1csk3yaZqhxSbs/5m3AVgBV9WGa/TOHAuuA24FXDbXcedhXkqTNaK4OPUmS5giDQpLUyaCQJHUyKCRJnQwKSVIng0IaJ8m9SS5K8p0k/5xk+828/KOSfKC9vCrJmzbn8qXNzaCQNnVHVT2lqvaiOUbntaMuSBolg0Lq9k0GTpqW5M1J1rTn8n/7wPRXttMuTnJyO+357W+lXJjkX5M8egT1Sw/YnDwyW5oLkiygOe3DSe31g2jOlbQvzcnVVid5JnAjzXm2nlFVNyR5ZLuIrwNPr6pK8nvAH9McISxtUQwKaVMPTXIRsBy4APhyO/2g9u/C9vp2NMHxZOCMqroBoKrGTk65E/CZ9nz/WwM/mJXqpc3MoSdpU3dU1VOAXWg+4Mf2UQR4Z7v/4ilVtVtVndROn+hcOH8LfKCqngS8muZEdNIWx6CQJlFVG4DXAW9KshXNSed+N8l2AEmWJtkROBt4cZJHtdPHhp4WAz9qLx+JtIVy6EnqUFUXJrkYOKKqTm5PUf3N9iy9twIvb89U+g7ga0nupRmaOormV9VOT/IjmlOeP24Uj0F6oDx7rCSpk0NPkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6vRftBSSzNxBHIcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(y_test, xgb_pred)\n",
    "step_kwargs = ({'step': 'post'})\n",
    "plt.step(recall, precision, color='b', alpha=0.2,\n",
    "         where='post')\n",
    "plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)\n",
    "\n",
    "from sklearn.metrics import average_precision_score\n",
    "average_precision = average_precision_score(y_test, xgb_pred)\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEGCAYAAADscbcsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFVlJREFUeJzt3X2sZdV53/HvjwEcHIwBgymBSUD21Am25DFQjOq2wnY0DKgpODIStAoTF3VSFypbSaOQtBKuHauOotgSrY2KBQLc2JjYQYxSnMmYuCWueDUmvBg7c41fGIPAMIQgHPNy79M/zpr4zPW95+w9c8/cl/P9SFv3nGevvfa6c83jtfbae+1UFZKk0Q5a7gZI0mpgspSkDkyWktSByVKSOjBZSlIHJktJ6sBkOUFJNif5VpKZJJcvd3s0XpJrkzyV5KHlbotWFpPlhCRZB3wSOAc4BbgoySnL2yp1cB2webkboZXHZDk5ZwAzVfVoVb0E3Aict8xt0hhVdTuwe7nboZXHZDk5JwCPDX3f1WKSViGT5eRkgZjPlkqrlMlycnYB64e+nwg8vkxtkbSfTJaTcw+wIcnJSQ4FLgS2LXObJO0jk+WEVNUrwGXAduAR4Kaqenh5W6VxknwOuAN4U5JdSS5Z7jZpZYhLtEnSePYsJakDk6UkdWCylKQOTJaS1IHJ8gBIsnW526B+/JtpPpPlgeF/eKuPfzPtxWQpSR2sqPssjzl6XZ20/pDlbsaS++Ezsxz7unXL3YyJ+JsHXr3cTZiIl3mRQ3jVcjdjyf2YF3ipXlxo3YLOzn7nz9Yzu2c7lf3aAy9ur6o1seTdwcvdgGEnrT+Eu7evH19QK8bZP7dxuZugHu6q2/a7jmd2z3L39p/vVHbd8TuP2e8TrhArKllKWvkKmGNuuZtxwJksJfVSFC9Xt2H4WmKylNSbPUtJGqMoZlfQxPCBYrKU1NvcFC76b7KU1EsBsyZLSRrPnqUkjVHAy16zlKTRinIYLkljFcxOX640WUrqZ/AEz/QxWUrqKcyyX2txrEomS0m9DCZ4TJaSNNLgPkuTpSSNNWfPUpJGs2cpSR0UYXYK30hjspTUm8NwSRqjCC/V2nyn1CgmS0m9DG5KdxguSWM5wSNJY1SF2bJnKUljzdmzlKTRBhM805c6pu83lrRfnOCRpI5mvc9SkkbzCR5J6mjO2XBJGm2wkIbJUpJGKsLLPu4oSaNV4U3pkjRevCldksYpprNnOX2/saT9NstBnbZRkqxP8pUkjyR5OMkHWvxDSX6Q5P62nTt0zO8mmUnyrSRnD8U3t9hMksuH4icnuSvJziSfT3Joi7+qfZ9p+08a9zubLCX1UoS56raN8QrwW1X1S8CZwKVJTmn7PlFVG9t2K0DbdyHwZmAz8Kkk65KsAz4JnAOcAlw0VM8ftLo2AM8Cl7T4JcCzVfVG4BOt3EgmS0m9DF6Fe3CnbWQ9VU9U1X3t8/PAI8AJIw45D7ixql6squ8AM8AZbZupqker6iXgRuC8JAHeBXyhHX89cP5QXde3z18A3t3KL8pkKamnMNtx61zjYBj8NuCuFrosyQNJrk1yVIudADw2dNiuFlss/jrgb6vqlXnxvepq+59r5RdlspTUSzF4gqfLBhyT5N6hbev8+pIcDnwR+GBV/R1wFfAGYCPwBPBHe4ou0py+8VF1LcrZcEm99eg1Pl1Vpy+2M8khDBLlH1fVnwJU1ZND+z8N/Fn7ugtYP3T4icDj7fNC8aeBI5Mc3HqPw+X31LUrycHAa4Hdo34Re5aSeqlKn57loto1wmuAR6rq40Px44eKvQd4qH3eBlzYZrJPBjYAdwP3ABvazPehDCaBtlVVAV8B3tuO3wLcMlTXlvb5vcBftvKLsmcpqZfBBM+SPO74DuDXgAeT3N9iv8dgNntjO9V3gd8AqKqHk9wEfIPBTPqlVTULkOQyYDuwDri2qh5u9f0OcGOS3we+ziA5035+JskMgx7lheMaa7KU1NPSvIOnqr7KwtcObx1xzEeBjy4Qv3Wh46rqUQaz5fPjPwYu6NNek6WkXgYTPD7uKEljuUSbJI2x5wmeaWOylNSbLyyTpDGq4OU5k6UkjTQYhpssJWmsPs99rxUmS0m9TOutQxPtSy+2IKek1WxpHndcbSb224xZkFPSKjbX3sMzbltLJjkM/4cFOQGS3Mhgwc1vTPCckiZsMBs+fa/CnWQ/ebEFOfeSZOuete5++MzsBJsjaSks4WslVpVJJstOi2tW1dVVdXpVnX7s66bv/62k1chh+NIatVCnpFVqWmfDJ5ks/2FBTuAHDNaL+9cTPJ+kA2StzXR3MbFkWVWvjFiQU9IqVRVeMVkurcUW5JS0ujkMl6QxvGYpSR2ZLCVpDBf/laSO1to9lF2YLCX1UgWvuPivJI3nMFySxvCapSR1VCZLSRrPCR5JGqPKa5aS1EGYdTZcksbzmqUkjeGz4ZLURQ2uW04bk6Wk3pwNl6QxygkeSerGYbgkdTCNs+HT15eWtF+qBsmyyzZKkvVJvpLkkSQPJ/lAix+dZEeSne3nUS2eJFcmmUnyQJJTh+ra0srvTLJlKH5akgfbMVcmyahzjGKylNTbXKXTNsYrwG9V1S8BZwKXJjkFuBy4rao2ALe17wDnABvathW4CgaJD7gCeDtwBnDFUPK7qpXdc9zmFl/sHIsyWUrqrarbNrqOeqKq7mufnwceAU4AzgOub8WuB85vn88DbqiBO4EjkxwPnA3sqKrdVfUssAPY3PYdUVV3VFUBN8yra6FzLMprlpJ6KcJc99nwY5LcO/T96qq6en6hJCcBbwPuAo6rqidgkFCTvL4VOwF4bOiwXS02Kr5rgTgjzrEok6Wk3npMhj9dVaePKpDkcOCLwAer6u/aZcUFiy7SlL7xfeIwXFI/SzTBA5DkEAaJ8o+r6k9b+Mk2hKb9fKrFdwHrhw4/EXh8TPzEBeKjzrEok6Wk/qrjNkKbmb4GeKSqPj60axuwZ0Z7C3DLUPziNit+JvBcG0pvBzYlOapN7GwCtrd9zyc5s53r4nl1LXSORTkMl9TbEt1n+Q7g14AHk9zfYr8HfAy4KcklwPeBC9q+W4FzgRngR8D7Bm2p3Uk+AtzTyn24qna3z+8HrgMOA77UNkacY1EmS0m9FDA3t//Jsqq+ysLXFQHevUD5Ai5dpK5rgWsXiN8LvGWB+DMLnWMUk6WkfgqYwid4TJaSevPZcEnqwmQpSeN0uy1orTFZSurPnqUkjVFQSzAbvtqYLCXtA5OlJI3nMFySOjBZStIY3pQuSd14U/oISV5VVS9OsjGSVokpnA0fu0RbkjOSPAjsbN/fmuS/T7xlklasVLdtLemynuWVwL8EngGoqr8G3jnJRklawbquZbnGkmWXYfhBVfW9eUu9z06oPZJWvDjBs4jHkpwBVJJ1wH8E/mayzZK0oq2xXmMXXZLl+xkMxX8eeBL4cotJmlZzy92AA29ssqyqp4ALD0BbJK0G3me5sCSfZoFOd1VtnUiLJK14a22mu4suw/AvD33+GeA97P1Cc0nTxmT506rq88Pfk3wG2DGxFknSCrQvjzueDPzCUjcE4OEXjuYtd/6bSVStCVn3gdcudxPUw8ufvXNJ6nEYvoAkz/KTTvdBwG7g8kk2StIKVkzl444jk2UGd6K/FfhBC821d/dKmmZTmAVGPu7YEuPNVTXbtin8J5I0n8+GL+zuJKdOvCWSVg+fDf+JJAdX1SvAPwP+XZJvAy8wePlGVZUJVJpWaywRdjHqmuXdwKnA+QeoLZJWgbU4xO5iVLIMQFV9+wC1RdJq4Wz4Xo5N8puL7ayqj0+gPZJWAXuWe1sHHM40viBY0mgmy708UVUfPmAtkbQ6eM3yp9ijlLQwk+Ve3n3AWiFpVckULv676E3pVbX7QDZEklayLk/wSNLelugJniTXJnkqyUNDsQ8l+UGS+9t27tC+300yk+RbSc4eim9usZkklw/FT05yV5KdST6f5NAWf1X7PtP2nzSurSZLSf10fC684yTQdcDmBeKfqKqNbbsVIMkpDF5x8+Z2zKeSrGsvUvwkcA5wCnBRKwvwB62uDcCzwCUtfgnwbFW9EfhEKzeSyVJSf0vUs6yq2xks+9jFecCNVfViVX0HmAHOaNtMVT1aVS8BNwLntVXT3gV8oR1/PT95IvG89p22/92Z977v+UyWkvrrniyPSXLv0Nb13V2XJXmgDdOParET2PuVNrtabLH464C/bWtcDMf3qqvtf66VX5TJUlIvYTAb3mUDnq6q04e2qzuc4irgDcBG4Angj4ZOPV/tQ3xUXYsyWUrqZ2mvWf509VVPtvVz54BPMxhmw6BnuH6o6InA4yPiTwNHJjl4Xnyvutr+1zLmcoDJUlJ/E1zPMsnxQ1/fA+yZKd8GXNhmsk8GNjBYHe0eYEOb+T6UwSTQtrZY+VeA97bjtwC3DNW1pX1+L/CX4xY335cXlkmadkv0BE+SzwFnMbi2uQu4AjgrycZ2lu8CvwFQVQ8nuQn4BvAKcGlVzbZ6LgO2M1jT4tqqerid4neAG5P8PvB14JoWvwb4TJIZBj3KC8e11WQpqbeleja8qi5aIHzNArE95T8KfHSB+K3ArQvEH+Unw/jh+I+BC/q01WQpqT+fDZekMWo6nw03WUrqz56lJI3nepaS1IXJUpLGWIPvBO/CZCmpl+AwXJI6MVlKUhcmS0nqwGQpSWP4KlxJ6shkKUnj+bijJHXgMFySxvGmdEnqyGQpSaP5BI8kdZS56cuWJktJ/XjNUpK6cRguSV2YLCVpPHuWktSFyVKSxvDtjpI0nvdZSlJXNX3Z0mQpqTd7lpI0zpTelH7QpCpOcm2Sp5I8NKlzSFoemeu2rSUTS5bAdcDmCdYvaZlMY7Kc2DC8qm5PctKk6pe0TAoneJZDkq3AVoCDj33tMrdGUhfTOMEzyWF4J1V1dVWdXlWnH3zEq5e7OZK6qI7bGrLsPUtJq8u03pS+7D1LSatMFZnrto2z0F0zSY5OsiPJzvbzqBZPkiuTzCR5IMmpQ8dsaeV3JtkyFD8tyYPtmCuTZNQ5RpnkrUOfA+4A3pRkV5JLJnUuSQfY0g3Dr+On75q5HLitqjYAt7XvAOcAG9q2FbgKBokPuAJ4O3AGcMVQ8ruqld1z3OYx51jUxJJlVV1UVcdX1SFVdWJVXTOpc0k6sFLdtnGq6nZg97zwecD17fP1wPlD8Rtq4E7gyCTHA2cDO6pqd1U9C+wANrd9R1TVHVVVwA3z6lroHIvymqWkfgqY7Dt4jquqJwCq6okkr2/xE4DHhsrtarFR8V0LxEedY1EmS0n9dc+VxyS5d+j71VV19T6eNYu0pG98n5gsJfXWYzb86ao6vWf1TyY5vvX4jgeeavFdwPqhcicCj7f4WfPi/6fFT1yg/KhzLMrZcEm9LdVs+CK2AXtmtLcAtwzFL26z4mcCz7Wh9HZgU5Kj2sTOJmB72/d8kjPbLPjF8+pa6ByLsmcpqZ8lvOG83TVzFoPh+i4Gs9ofA25qd9B8H7igFb8VOBeYAX4EvA+gqnYn+QhwTyv34araM2n0fgYz7ocBX2obI86xKJOlpF4GN6UvTbasqosW2fXuBcoWcOki9VwLXLtA/F7gLQvEn1noHKOYLCX1t8ZWFOrCZCmpt6XqWa4mJktJ/azBRTK6MFlK6mm/ZrpXLZOlpP4chkvSGLX2XhnRhclSUn/2LCWpg+nLlSZLSf1lbvrG4SZLSf0U3pQuSeOE8qZ0SerEZClJHZgsJWkMr1lKUjfOhkvSWOUwXJLGKkyWktTJ9I3CTZaS+vM+S0nqwmQpSWNUwez0jcNNlpL6s2cpSR2YLCVpjAJ8B48kjVNQXrOUpNEKJ3gkqROvWUpSByZLSRrHhTQkabwCXKJNkjqwZylJ4/i4oySNV1DeZylJHfgEjyR1MIXXLA9a7gZIWmWqBrPhXbYxknw3yYNJ7k9yb4sdnWRHkp3t51EtniRXJplJ8kCSU4fq2dLK70yyZSh+Wqt/ph2bff21TZaS+qvqtnXzzqraWFWnt++XA7dV1QbgtvYd4BxgQ9u2AlfBILkCVwBvB84ArtiTYFuZrUPHbd7XX9lkKamnomZnO2376Dzg+vb5euD8ofgNNXAncGSS44GzgR1VtbuqngV2AJvbviOq6o6qKuCGobp6M1lK6mfPEm1dNjgmyb1D29YFavuLJF8b2ndcVT0B0H6+vsVPAB4bOnZXi42K71ogvk+c4JHUX/dbh54eGl4v5B1V9XiS1wM7knxzRNmFrjfWPsT3iT1LSb0UUHPVaRtbV9Xj7edTwM0Mrjk+2YbQtJ9PteK7gPVDh58IPD4mfuIC8X1ispTUT7XFf7tsIyT52SSv2fMZ2AQ8BGwD9sxobwFuaZ+3ARe3WfEzgefaMH07sCnJUW1iZxOwve17PsmZbRb84qG6enMYLqm3/Zi8GXYccHO7m+dg4LNV9edJ7gFuSnIJ8H3gglb+VuBcYAb4EfA+gKraneQjwD2t3Ieranf7/H7gOuAw4Ett2yepFXRzaZIfAt9b7nZMwDHA08vdCPWyVv9mv1BVx+5PBUn+nMG/TxdPV9U+366zkqyoZLlWJbl3zEVurTD+zTSf1ywlqQOTpSR1YLI8MK5e7gaoN/9m2ovJ8gCoqmX9Dy/JbFuo4KEkf5Lk1ftR11lJ/qx9/ldJLh9R9sgk/2EfzvGhJP9pX9u4FJb7b6aVx2Q5Hf6+LVTwFuAl4N8P72z3rfX+30JVbauqj40ociTQO1lKK5HJcvr8FfDGJCcleSTJp4D7gPVJNiW5I8l9rQd6OECSzUm+meSrwK/uqSjJryf5H+3zcUluTvLXbfunwMeAN7Re7R+2cr+d5J62xNZ/HarrPyf5VpIvA286YP8aUkcmyymS5GAGy1w92EJvYrCKy9uAF4D/AvxyVZ0K3Av8ZpKfAT4N/Arwz4F/tEj1VwL/t6reCpwKPMxgaa1vt17tbyfZxGCZrDOAjcBpSf5FktOAC4G3MUjG/2SJf3Vpv/kEz3Q4LMn97fNfAdcAPwd8ry11BXAmcArw/9oTFYcCdwC/CHynqnYCJPlfDNYHnO9dDB4no6pmgeeG1hTcY1Pbvt6+H84geb4GuLmqftTOsW2/fltpAkyW0+Hvq2rjcKAlxBeGQwzWBLxoXrmN7MdKLfME+G9V9T/nneODS3gOaSIchmuPO4F3JHkjQJJXJ/nHwDeBk5O8oZW7aJHjb2PwHC5J1iU5AnieQa9xj+3Avx26FnpCW5rrduA9SQ5rCyv8yhL/btJ+M1kKgKr6IfDrwOeSPMAgef5iVf2YwbD7f7cJnsWe3f8A8M4kDwJfA95cVc8wGNY/lOQPq+ovgM8Cd7RyXwBeU1X3AZ8H7ge+yOBSgbSi+Gy4JHVgz1KSOjBZSlIHJktJ6sBkKUkdmCwlqQOTpSR1YLKUpA7+Py3sUAG2lwxvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test, xgb.predict(X_test))\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "cax = ax.matshow(cm)\n",
    "fig.colorbar(cax)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision xgb:  0.7065748269277093\n",
      "Recall xgb:  0.5977742072680875\n",
      "F1 score xgb:  0.6476367842794676\n"
     ]
    }
   ],
   "source": [
    "print('Precision xgb: ', precision_score(y_test, xgb_pred))\n",
    "print('Recall xgb: ', recall_score(y_test, xgb_pred))\n",
    "print('F1 score xgb: ', f1_score(y_test, xgb_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oob_score is  0.9815292830267872\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "#from sklearn import svm\n",
    "#from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "# bagging with SVM\n",
    "# oob Out-Of-Bag\n",
    "bag_log_reg_svm = BaggingClassifier(\n",
    "     LinearSVC(), n_estimators=200,\n",
    "    max_samples=50000, bootstrap=True, n_jobs=-1, oob_score=True)\n",
    "\n",
    "bag_log_reg_svm.fit(X_train,y_train)\n",
    "print('oob_score is ', bag_log_reg_svm.oob_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6793646949726676"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_reg_svm_pred = bag_log_reg_svm.predict(X_test)\n",
    "accuracy_score(y_test, bag_reg_svm_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4VGX6//H3k9AhECAJJZACoRcFYlBUiiDFAoroYls7uuoW3VWwYcFdXf2Jrt91VVTWsutaKErHSlFBwUaJgPSEToAACanz/P44QSMGMgkzc6Z8XtfFZWbmMHMfAx+enHOf+xhrLSIiEl6i3C5ARER8T+EuIhKGFO4iImFI4S4iEoYU7iIiYUjhLiIShhTuIiJhSOEuIhKGFO4iImGohlsfHBcXZ1NSUtz6eBGRkPT111/vtdbGV7ada+GekpLC8uXL3fp4EZGQZIzZ4s12OiwjIhKGFO4iImFI4S4iEoYU7iIiYUjhLiIShioNd2PMZGPMbmPMquO8bowxzxpj1htjVhhjevq+TBERqQpvVu6vAkNP8PowoF3ZrzHA8ydflohImMr6ChY/5fzXjyrtc7fWLjLGpJxgkxHA69a5X99SY0ysMaaFtXaHj2oUEQl91lL49X+oNedPGOuB6NpwzQxoneGXj/PFRUyJQFa5x9llz/0q3I0xY3BW9yQlJfngo0VEglhJIWxeDGvnUrh6NrXzd/DTXatLi5zXgjjcTQXPVXjXbWvtJGASQHp6uu7MLSLhJ38f/PgBrJ0D6z+GosMUmTosKOnKrtpncqWdSbSnBKJrQcrZfivDF+GeDbQu97gVsN0H7ysiEhpyNjhhvnYubF0C1gMNmuPpdin3Z7Zm+oE2/LZvR+4Y1J7onV87K/aUs/22agffhPsM4HZjzFtAbyBXx9tFJKx5Sp0TomvnwLp5sHed83yzbnD2XziYfC4xqb2Iioqmb+pORsfWoXurWGeb1hl+DfWjKg13Y8z/gP5AnDEmG3gQqAlgrX0BmAOcB6wH8oHr/FWsiIhrCg/Dhk+c1fmP8yE/B6JqQspZcNpN0GEotlFr3vtuGw//N5OxQ7dxeUYSQ7s2d6Vcb7plLq/kdQvc5rOKRESCxcHtTpivnQubFjonQevEQrvB0GEYpA2EOo0A2H7gCPe9uoxP1+6hR1Is6cmNXS3dtZG/IiJBx1rYubIs0OfAju+c5xunQsYYJ9Bbnw7Rv4zO97/bxn3TV1HqsYy/oDPX9EkhOqqiXpPAUbiLSGT7qV1xnhPqB7MB4xwXH/QQdDgP4tqDOX5YN6pbk1Nbx/LYyG60blIvUJWfkMJdRCJPBe2K1KwHbc+BAfdAuyHQ4Pg3Oyop9fDKZ5soLvVw+znt6N8hgX7t4zEn+Acg0BTuIhIZjtOuSLdRzuo8tS/UrFvp22RuP8jYqStYuS2X87u3wFqLMSaogh0U7iISrjylkL3s50A/pl2RDkOhRQ+I8m44bmFJKf/8ZD3PL9hAbL2a/OvKngzr2jzoQv0ohbuIhI+j7Yrr5jm/KmhXJLZ6o082783nhYUbGH5qSx44vzON69fycfG+pXAXkdD2i3bFRVBa6LQnthvyq3bFqsorLOHDzF1c1CORDs1j+PjO/iQ1DY4TppVRuItIaDlRu+JpNzqBnnQ6RNc8qY9Z/OMe7pm2km0HjtA1sSFpCTEhE+ygcBeRUFBSCJs/+3mFfmy7YvthEN/hhO2K3srNL+avczJ5Z3k2beLq8/aYM0hLiDnp9w00hbuIBKf8ffDjh+XaFQ9VqV2xOko9lkte+IJNe/O4tX9b/jCwHXVqRvv0MwJF4S4iweO47YqXVKldsar25RURW7cm0VGGu4Z0IDG2Ll0Tq3ecPlgo3EXEPcdtV+wKZ//ZOX5ehXbFqrLWMu2bbTwyK5OxQztyRe8khnRxZ9CXryncRSSwCg/Dxk+dMP+pXbGGM9/8tBuh/VBonOz3MrL353Pv9FUsWreHXsmNyUht4vfPDCSFu4j438HtTpCvnQsbF/q0XbE6pn+bzf3TV2GBh4d34erTk4lyedCXryncRcT3jtuumOLTdsXqalK/Nr1SmvC3i7vSqnHotDdWhcJdRHzjeO2KrU6DgQ86J0R91K5YVcWlHl5avJGSUssfBrajX/t4+raLC9rRAb6gcBeR6jtRu2L/cdB+CDRIcLXEVdtyGTt1Bau3H+TCU1oG7aAvX1O4i0jV5Gz4eXW+dQnY0oC0K1ZVQXEpz378Iy8u2kjjerV44aqeDO3awu2yAkbhLiIndsJ2xTv93q5YXVty8nlp8UZG9kjk/vM706ieO8f33aJwF5FfO2674lkBbVesqrzCEuav3snInq3o0DyGT/7cP2jujBRoCncRcRy3XfHozaAHBbRdsaoWrtvDvdNWsj33CN1bNSItISZigx0U7iKRy1rYterndsXt3zrPB0m7orf25xUxYXYm077ZRtv4+rx7c2gO+vI1hbtIJCkpKrsZdNnhltwsgqVdsTqODvrakpPP7QPSuP2ctJAd9OVrCneRcFdRu2KNuk67Yr+xQdGuWFU5hwtpXK8W0VGGcUM7kti4Ll1aBu8hIzco3EXCUYXtis2g60hndd6mX1C0K1aVtZZ3v87m0VmZjB3WkSt7JzM4TAZ9+ZrCXSQceEohe3m5dsW1zvNB3q5YFVn78rl3+koW/7iXjJQmnNGmqdslBTWFu0io+kW74nzI31uuXfGGoG1XrI5p32Rz/3urMMCEi7pyZUZS2A368jWFu0goObgD1s0N2XbF6oprUJuM1Cb89eJuJMaG3uEkNyjcRYLZCdsVbyhrVzwj6NsVq6q41MOLCzdQ6oE/DmpH3/bx9G3v21vqhTuFu0iwKSmCLeWmK4Z4u2JVrdqWy11TVvDDjoOMOPXnQV9SNV6FuzFmKPAPIBp42Vr7+DGvJwGvAbFl24yz1s7xca0i4etou+K6ufDjR2HRrlhVBcWlPPPRj7y0eCNN6tfixat7hc0t79xQabgbY6KB54BzgWxgmTFmhrU2s9xm9wPvWGufN8Z0BuYAKX6oVyR8hGm7YnVt3ZfPK59tZFTPVtx7XqeIG/Tla96s3DOA9dbajQDGmLeAEUD5cLdAw7KvGwHbfVmkSFg4XrtiQhc46w4n0FuGdrtiVR0qKGbeqp1cmt6a9s1i+PQv/cP2zkiB5k24JwJZ5R5nA72P2eYh4ANjzO+B+sAgn1QnEuqK8mBD+emKZe2KyWdC+vXQYahzcjQCfbpmN/dNX8nOgwX0SIolLSFGwe5D3oR7RWcy7DGPLwdetdY+ZYw5A3jDGNPVWuv5xRsZMwYYA5CUlFSdekWC30/tivNg44Jftiu2H+q0K9aNdbtK1+zLK2LCrEymf7uNdgkNmPK7Phr05QfehHs20Lrc41b8+rDLDcBQAGvtEmNMHSAO2F1+I2vtJGASQHp6+rH/QIiEpuO1K8Ymh3W7YnWUeiyjnv+Crfvy+cPAdtw2oC21a2jQlz94E+7LgHbGmFRgGzAauOKYbbYCA4FXjTGdgDrAHl8WKhJUjtuumA4Dx5e1K3YM23bFqtpzqJCm9Z1BX/ee14nExnXp1KJh5b9Rqq3ScLfWlhhjbgfm47Q5TrbWrjbGPAIst9bOAP4MvGSMuQPnkM211lqtzCW85O+D9R85q/NftCsOgH53Q7shENPM7SqDirWWd5Zn8ejsHxg7tCNXnZ7MoM76fxQIXvW5l/WszznmufHlvs4EzvRtaSJBIGfDz3cn2vLFMe2KwyC1H9TSScCKbM3JZ9y0FXyxIYfeqU04Ky3O7ZIiiq5QFSlP7Yo+MeXrbB54bxXRUYa/XtyVy0/ToK9AU7iLqF3R55o1rE2ftk159OKutGgUORdiBROFu0SmgzvK3Qx6gdOuWLsRtDv35+mKEdyuWFVFJR6eX7ABj7XccW57zm4Xz9ntNOjLTQp3iQw/tSvOK2tX/MZ5Pja5bHU+DJL7qF2xGr7POsDdU1awdtchRvZI1KCvIKFwl/CldkW/OlJUysQP1/LKZ5tIiKnDy79NVydMEFG4S3gp3664/mMoPKh2RT/J2p/Pa19sYXRGEuOGdaRhHf3UE0wU7hL69m38eXV+tF2xfgJ0uchZnatd0WcOlg36uqxs0NeCu/rTUndGCkoKdwk9nlLY9vXP7Yp71jjPJ3SBs/5U1q7YU+2KPvbJml3cO20Vuw8V0DOpMWkJDRTsQUzhLqHhRO2Kva5Tu6If5Rwu5JFZmbz/3XY6NIvhhat7kZbQwO2ypBIKdwleald0XanHcukLS8jan88dg9rzu/5tqVVDPxGFAoW7BA9rYdfqctMV1a7olt2HCoirX5voKMN953eiVeN6dGiusbyhROEu7iopgi2fl2tX3Oo8n5gO5zzgHD9P6KR2xQDxeCz/W7aVx+asYeywjlx9ejIDO6m7KBQp3CXwTtiueJfaFV2yeW8e46atYOnGffRp25R+usI0pCncJTDUrhjU3lmexQPvraJWdBSPj+zGb05rratMQ5zCXfzjuO2KndWuGIQSY+vSt308E0Z0pXmjOm6XIz6gcBffOdquuG4urJsPeXvK2hX7QK9rnfuHNkl1u0oBCktK+denG7DWcufgDpyZFseZmrceVhTuUn1ZXzkrc+uB3WvUrhgivt26n7FTV7Bu12Eu6dlKg77ClMJdqifrK/j3MPCUOI8bNFO7YpDLLyrhqQ/WMfnzTTRvWIfJ16ZzTkeduA5XCnepnmWv/BzsJgoyboa+f3a3JjmhbfuP8MbSLVzZO4mxQzsSo0FfYU3hLlV3cDusmQ0YJ9ija0Hq2W5XJRXIPVLM3JU7GJ2RRLtmMSy8q7/ujBQhFO5SNZ5SmDbGOc5+yStwYDOknA2tM9yuTI7xweqd3P/eKnLyikhPaUJaQgMFewRRuEvVLJ4ImxfDiH9Bt0vcrkYqsPdwIQ/NWM2sFTvo2DyGl69J16CvCKRwF+9tWQIL/gbdLoNTr3C7GqlAqccy6vkv2H6ggL8Mbs/N/dpSM1rXEkQihbt4J38fTL3RGeJ1wUTNegkyuw4WEN/AGfT14IVdaNW4Lu2aadBXJNM/6VI5a2HG7+HwLhg1GWorNIKFx2N5Y+kWBj61kP9+uQWAAR0TFOyilbt4YdnLsGYWDP4rJPZ0uxops3HPYcZNW8lXm/ZxVloc/TskuF2SBBGFu5zYzlUw/z5IOxdOv9XtaqTM28u2Mv791dSuEcUTo7pzaa9WuspUfkHhLsdXlAdTrnfGB1z0vIZ8BZFWjevRv4Mz6CuhoQZ9ya8p3OX45o6Fvevgt+9BA832dlNhSSn/9/F6AP4yRIO+pHIKd6nYqqnw7Rtw9p+hTX+3q4loX2/Zx91TVrBhTx6XpWvQl3hH4S6/tn8zzPwTtMqA/ve4XU3Eyiss4cn5a3ltyWZaNqrLa9dn0K+9foIS73h1ENUYM9QYs9YYs94YM+4421xmjMk0xqw2xrzp2zIlYEqLnePsGLjkZU13dNH2A0d486ut/Pb0ZObf0VfBLlVS6crdGBMNPAecC2QDy4wxM6y1meW2aQfcA5xprd1vjFFPVqj6ZIJzB6VLX4PGyW5XE3Fy84uZvXIHV/R2Bn0tvnsAzXTCVKrBm8MyGcB6a+1GAGPMW8AIILPcNjcBz1lr9wNYa3f7ulAJgPUfw+f/gF7XOfc2lYCat2onD7y/in15RfRu04S28Q0U7FJt3hyWSQSyyj3OLnuuvPZAe2PM58aYpcaYoRW9kTFmjDFmuTFm+Z49e6pXsfjHoV0w/WaI7wRDH3O7moiy+1ABt/73a275z9fEN6jN+7edSdt4DfqSk+PNyr2i0/K2gvdpB/QHWgGLjTFdrbUHfvGbrJ0ETAJIT08/9j3ELR6PE+yFh+GamVBTY2EDpdRjueyFJWzPLeCuIR0Y07eNBn2JT3gT7tlA63KPWwHbK9hmqbW2GNhkjFmLE/bLfFKl+NcXz8LGT+GCZyChk9vVRIQduUdoFlPHGfQ1vAutG9fTWF7xKW+WCMuAdsaYVGNMLWA0MOOYbd4DBgAYY+JwDtNs9GWh4ifZy52TqJ0vgl7Xul1N2PN4LK9+vomBTy3kP0cHfXVIULCLz1W6crfWlhhjbgfmA9HAZGvtamPMI8Bya+2MstcGG2MygVLgLmttjj8LFx8oyIUp10FMS7jwHxrj62frdx9m3NQVLN+yn77t4zmno5rKxH+8uojJWjsHmHPMc+PLfW2BO8t+SSiw1rlQKXcbXD/PmR8jfvPWV1sZP2M1dWtG89SlpzCyZ6KuMhW/0hWqkerbN2D1NBg4Xvc/DYCkpvUY1CmBh4d3JT6mttvlSARQuEei3Wtgzt2Q2g/OvMPtasJSQXEpz378IwB3D+1In7Zx9GmrQV8SOOq5ijTFR5zxArXqw8hJGuPrB8s37+O8ZxfzrwUb2JdXhHPUUiSwtHKPNB/cD7tXw5VTIaa529WElcOFJTw5bw2vL91CYmxdXr8+g76aByMuUbhHkh9mOrfMO+N2aDfI7WrCzs7cI7y1LItrzkjhriEdqF9bf73EPfrTFykOZMH7t0HLHjDwQberCRv784qYtXIHV5+eTFqCM+hLd0aSYKBwjwSlJTD1RmfMwKjJUKOW2xWFPGstc1ftZPz7qziQX0yftk1pG99AwS5BQ+EeCRb+HbKWwsiXoUkbt6sJebsPFvDA+6uYv3oX3RIb8fr1vTXoS4KOwj3cbVoEi56EU6+E7pe6XU3IK/VYLn1xCTtzC7hnWEduOCuVGhr0JUFI4R7O8nJg2hhomgbDnnC7mpC2/cARmjd0Bn09MqIrrRvXpY1W6xLEtOQIV9bCe7+D/BznOHttBVF1lHos/z5m0Fe/9vEKdgl6WrmHq6XPw4/znRV7i+5uVxOS1u8+xN1TVvDN1gP07xDPwE7N3C5JxGsK93C0/Tv4cDx0OA8yxrhdTUh688utPDRjNfVrR/P0b07holM16EtCi8I93BQecsYLNEiAEc9pjG81pcTVY3CXZjw0vAtxDTToS0KPwj3czP4L7N8E18yCek3criZkFBSX8vRH6zAYxg3ToC8JfTqhGk6+fwtWvAX9xkLKmW5XEzK+3JjDsH8s5sWFGzlUUKxBXxIWtHIPFzkbYNadkHwm9L3L7WpCwqGCYv4+bw3/WbqVpCb1ePPG3vRJ02pdwoPCPRyUFDq3y6tRC0a+BFHRblcUEnYdLGTK19nceFYqdw5uT71a+usg4UN/msPBRw/Bju9h9JvQKNHtaoLavrwiZq/YztVnpJCW0IDFd5+jOyNJWFK4h7p182HpvyDjZuh4vtvVBC1rLbNW7OChGas5WFDMmWlxtIlvoGCXsKVwD2UHtztXoTbrBuc+4nY1QWvXwQLum76Kj37YRfdWjfjvqN66wlTCnsI9VHlKnbkxxUfg0n9DTY2arUipx3JZ2aCv+87rxHVnpmjQl0QEhXuoWjwRNi+GEf+CuHZuVxN0svfn06JRXaKjDBNGdCWpST1S4uq7XZZIwGgJE4q2LIEFj0G3S+HUK9yuJqiUeiwvL97IoIkL+c9SZ9BX3/bxCnaJOFq5h5r8fc5dlWKT4PyJGi9Qztqdh7h76gq+zzrAwI4JDO6iQV8SuRTuocRamPF7OLwLbvgA6jR0u6Kg8Z+lW3h45mpi6tTkH6NPZfgpLTXoSyKawj2ULH8F1syCwY9CYk+3qwkK1lqMMaQlNOC8bi0Yf0FnmmrQl4jCPWTsXAXz7oW0QXD6bW5X47ojRaVM/HAtUVGGe4Z14vQ2TTm9TVO3yxIJGjqhGgqK8pwxvnVj4aIXICqyv21LNuQw9B+LeGnxJvILSzXoS6QCWrmHgnnjYO86uHo6NIh3uxrXHCwo5rE5a/jfV1tJblqPN2/qrbG8Isfh1RLQGDPUGLPWGLPeGDPuBNuNMsZYY0y670qMcKumwjevw1l3QNsBblfjqt0HC3nv222M6duGeX/sq2AXOYFKV+7GmGjgOeBcIBtYZoyZYa3NPGa7GOAPwJf+KDQi7d8MM/8ErU6DAfe6XY0rcg4XMvP77Vx7ZippCQ34bOwAnTAV8YI3K/cMYL21dqO1tgh4CxhRwXYTgCeAAh/WF7lKi2HKDYCBS16B6JpuVxRQ1lre/24bgyYu5K9zfmDjnsMACnYRL3kT7olAVrnH2WXP/cQY0wNoba2d5cPaItsnj8K25TD8WWic7HY1AbX9wBFueG05f3zrO5Kb1mf2H87WoC+RKvLmhGpFV4L81J5gjIkCngaurfSNjBkDjAFISkryrsJItP5j+PwZ6HUtdLnI7WoCqqTUw+hJS9lzqJAHLujMtX1SiI7SxUgiVeVNuGcDrcs9bgVsL/c4BugKLCi7IrA5MMMYM9xau7z8G1lrJwGTANLT09W/VpHDu2H6LRDfCYY85nY1AZO1L5+WsXWpER3F3y7uRlKTeiQ1red2WSIhy5vDMsuAdsaYVGNMLWA0MOPoi9baXGttnLU2xVqbAiwFfhXs4gWPB6bfDIUHnTG+tcI/3EpKPUxatIFBExfyxpLNAJzVLk7BLnKSKl25W2tLjDG3A/OBaGCytXa1MeYRYLm1dsaJ30G89sWzsOETuOBpSOjkdjV+98OOg4yduoIV2bmc27kZw7q1cLskkbDh1UVM1to5wJxjnht/nG37n3xZESh7OXwyAToNh17XuV2N372xZDMPz8ykUd2a/POKHpzfrYUGfYn4kK5QDQYFuc54gZiWTndMGIfc0UFf7ZvFcOEpLXnggs40qV/L7bJEwo7C3W3WOhcq5WbD9fOgbmO3K/KL/KIS/t/8ddSINtx7Xid6t2lKbw36EvGbyJ5AFQy+fQNWT3OuQG2d4XY1fvH5+r0MeWYRkz/fRFGJR4O+RAJAK3c37VkLc+6G1H7O7Jgwk3ukmL/N/oG3l2eRGlefd24+g4zUJm6XJRIRFO5uKT7iHGevVR9GToKoaLcr8rm9hwuZuWI7t/Rry58GtaNOzfDbR5FgpXB3ywf3w65VcOUUiGnudjU+s+eQM+jr+rNSaRvfgM/GnqMTpiIuULi74YeZsOxlOON2aHeu29X4hLWW977bxsMzM8kvLGVAxwRS4+or2EVconAPtANZ8P7t0LIHDHzQ7Wp8YtuBI9w3fSUL1u6hZ1IsT4zqTmpcfbfLEoloCvdAKi2BqTeCp8QZ41sj9Fe1zqCvJeQcLuKhCztz9Rka9CUSDBTugbTw75C1FEa+BE3bul3NSdmak09iY2fQ1+Mju5PUpB6tm2gejEiwUJ97oGxaBIuehFOvhO6XuV1NtZWUenh+wQYGPb2Q15dsBuDMtDgFu0iQ0co9EPJyYNoYaJoGw55wu5pqW709l7FTV7Bq20GGdGnG+Rr0JRK0FO7+Zi28fyvk58AV70Dt0Lyj0GtfbGbCrExi69Xi+St7aoKjSJBTuPvbly/AunnOir1Fd7erqbKjg746No9hxKmJPHBBJ2Lrhf6JYJFwp3D3p+3fwYfjocN5kDHG7WqqJK+whCfnr6VmtOG+8ztr0JdIiNEJVX8pPOSMF6gXByOeC6kxvovW7WHw04t4bclmikutBn2JhCCt3P1lzl2wfxNcMxPqhcawrNz8YibMzmTK19m0iXcGfZ2WEhq1i8gvKdz94fu34Pv/Qb9xkHKW29V4bW9eIXNX7uDW/m35w0AN+hIJZQp3X8vZALPuhKQ+0Pcut6up1O5DBcz4bjs3nt3mp0FfjTUPRiTkKdx9qaQQplznjBW45CWIDt7/vdZapn6zjQmzMjlSXMrATs1IjauvYBcJE8GbPqHoo4dhx/cw+k1o1Mrtao4ra18+905fyeIf95Ke3JjHL9GgL5Fwo3D3lXXzYelzTstjx/Pdrua4Sko9XP7SUvbnFTFhRBeu7J1MlAZ9iYQdhbsvHNwB7/0OmnWDcye4XU2FNu/No3WTetSIjuKJUc6gr1aNNQ9GJFypz/1keUph2k3ObfNGTYaaddyu6BeKSz089+l6Bj+96KdBX33axinYRcKcVu4na/FE2LzYuVApvr3b1fzCqm253D1lBZk7DnJ+txZc0L2l2yWJSIAo3E/G1qWw4DHoOsoZ5RtE/v35Jh6d/QNN6tfihat6MbRr+NynVUQqp3CvriP7nbsqxbaGC54OmvECRwd9dWnZiJE9Ern//M40qlfT7bJEJMAU7tVhLcz4PRzaATd8AHUaul0RhwtLeGLeGmpFR3H/BZ3JSG1CRqpGB4hEKp1QrY7lr8APM50bXCf2crsaFqzdzZCnF/HG0i1Y0KAvEdHKvcp2rYZ590LaIDjjdldL2Z9XxITZmUz7ZhtpCQ2YcksfeiU3drUmEQkOCveqKMqDd6+DurFw0QsQ5e4PPvvzi/hg9S7+cE4at52TRu0aGvQlIg6v0skYM9QYs9YYs94YM66C1+80xmQaY1YYYz42xiT7vtQgMG8c7F0HF78IDeJdKWH3wQImLdqAtZY28Q34fOw53Dm4g4JdRH6h0nA3xkQDzwHDgM7A5caYzsds9i2Qbq3tDkwBQvcu0Mezaip88zqc9SdoOyDgH2+t5Z1lWQycuJCnPljH5px8AHXCiEiFvDkskwGst9ZuBDDGvAWMADKPbmCt/bTc9kuBq3xZpOv2b4aZf4JWp8GA+wL+8Vn78rln2ko+W7+XjNQmPD6ymwZ9icgJeRPuiUBWucfZQO8TbH8DMLeiF4wxY4AxAElJSV6W6LLSYphyA2DgklcgOrAr5aODvg7kF/PoRV25IiNJg75EpFLehHtFSVJhr50x5iogHehX0evW2knAJID09PTQ6Nf75FHYthwufRUaB+5Uwqa9eSSVDfp6ctQpJDetR8vYugH7fBEJbd6cUM0GWpd73ArYfuxGxphBwH3AcGttoW/Kc9n6j+HzZ6DXtdDl4oB8ZHGph//7+EeGPL2I177YDMAZbZsq2EWkSrxZuS8D2hljUoFtwGjgivIbGGN6AC8CQ621u31epRsO74bpt0B8JxjyWEA+ckX2Ae6esoI1Ow9x4SktGX6qBn2JSPVUGu7W2hJjzO3AfCAamGytXW2MeQRYbq2dATwJNADeNc6Mla3W2uF+rNu/PB6YfjMUHoTfvg+1/D8ed/J6ct8DAAAKnklEQVRnm3h0dibxMbV56bfpnNu5md8/U0TCl1cXMVlr5wBzjnlufLmvB/m4Lnct+T/Y8IkzEKzZsV2fvnV00Ff3Vo34zWmtGTesE43qqr1RRE6OrlA9VvbX8PEj0Gk49LrObx9zqKCYx+euoXaNaMZf2Jn0lCakp2jQl4j4hgaHlVeQC1Oug5gWMPxZv43x/XTNbgY/vYj/fbWVGtFGg75ExOe0cj/KWph1B+Rmw3Vzoa7vB3DtyyvikZmree+77bRv1oB/XdmHHkka9CUivqdwP+rb/zgjBs55AJJOdI1W9eUeKebjH3bzx4HtuG1AGrVq6AcnEfEPhTvAnrUw5y5I7Qtn3eHTt96ZW8B7323j5r5tSI2rz2fjztEJUxHxO4V7cQFMud5pd7x4EkT5ZrqitZa3lmXxt9k/UOzxMLRLc1Li6ivYRSQgFO4f3A+7VsEV70LDFj55yy05eYybupIlG3M4vU0THh/ZnRQN+hKRAIrscP9hJix7ybmjUvvBPnnLklIPV7z0JblHivnbxd0YfVprDfoSkYCL3HA/kAXv3w4tTnXuhXqSNuw5THLZoK+nLnMGfbVopHkwIuKOyGzXKC2BaTeBpwRGTYYatar9VkUlHp75aB1Dn1nE60u2AHB6m6YKdhFxVWSu3Bf+HbYugZEvQdO21X6b77IOMHbKCtbuOsSIU1tyUY9EHxYpIlJ9kRfumxbDoifhlCug+2XVfptXPtvEX2dnkhBTh1euSWdgJw36EpHgEVnhnpfjHI5p2hbOe7Jab3F00NeprRsxOiOJccM60rCO2htFJLhETrhbC+/fCvk5cMXbULtBlX77wYJiHpuzhjo1o3jwwi70Sm5Cr2QN+hKR4BQ5J1S/fAHWzYNzJ0CLU6r0Wz/K3MW5Exfy9rKt1KoRpUFfIhL0ImPlvuN7+HA8tB8GvW/2+rflHC7k4ZmZzPh+Ox2bxzDp6nROaR3rx0JFRHwj/MO98BC8ex3Ui4MRz1VpjO+hghI+XbubOwa153f922rQl4iEjPAP9zl3wf5NcM1MqN+00s23HzjC9G+3cWv/tqTE1efzcefohKmIhJzwDvfv34bv/wf9xkLKWSfc1OOxvPnVVh6fu4ZSj+X8bi1IiauvYBeRkBS+4Z6zAWbfCUl9oO/dJ9x00948xk1dwZeb9nFmWlMeu7g7SU39f1NsERF/Cc9wLyl0bpcXVQMueQmij7+bJaUernr5Sw4WFPPEJd25NL0Vxk+31xMRCZTwDPePHnY6ZEa/CY1aVbjJ+t2HSGlanxrRUTz9m1NJblqPZg3rBLhQERH/CL/2j3UfwNLn4LSboOP5v3q5sKSUiR+uY+gzi3mtbNBXRmoTBbuIhJXwWrkf3AHv3QLNusLgR3/18jdb9zN2ygp+3H2YkT0SGalBXyISpsIn3D2lztyY4iMw6t9Q85cr8ZcWbeRvc3+gRcM6/Pu60xjQIcGlQkVE/C98wv2zibB5MQz/J8S3/+lpj8cSFWXomRzLlb2TGDu0IzFqbxSRMBce4b51KXz6GHQdBT2uAiD3SDF/nZ1J3ZrRPDyiqwZ9iUhECf0Tqkf2w9QbIbY1XPA0GMP81Ts5d+JCpn6zjfq1a2jQl4hEnNBeuVsLM34Ph3bA9R+wt6Q2D/73G2av3EHnFg2ZfO1pdE1s5HaVIiIBF9rhvnwy/DDTGePbqheH9+ax+Mc93DWkA2P6tqFmdOj/YCIiUh1epZ8xZqgxZq0xZr0xZlwFr9c2xrxd9vqXxpgUXxf6K7tWw7x7KEgewD8LhmCtJSWuPl/cM5DbBqQp2EUkolWagMaYaOA5YBjQGbjcGNP5mM1uAPZba9OAp4G/+7rQXyjKx757HUeiGzB402ieW7CJLTn5ADSoHdo/jIiI+II3y9sMYL21dqO1tgh4CxhxzDYjgNfKvp4CDDT+GtCS9RUFLw/F7F3LTYfHkJycygd39CUlrr5fPk5EJBR5s8xNBLLKPc4Geh9vG2ttiTEmF2gK7PVFkT/J+gr76nnUKS2mhCiuG9CFcwZlaNCXiMgxvFm5V5Scx/YWerMNxpgxxpjlxpjle/bs8aa+X9q8GFNaCkC0MQyss07BLiJSAW/CPRtoXe5xK2D78bYxxtQAGgH7jn0ja+0ka226tTY9Pj6+6tWmnA01aoOJxkTXch6LiMiveHNYZhnQzhiTCmwDRgNXHLPNDOAaYAkwCvjE+uPKodYZcM0MZ8xAytnOYxER+ZVKw73sGPrtwHwgGphsrV1tjHkEWG6tnQG8ArxhjFmPs2If7beKW2co1EVEKuFV36C1dg4w55jnxpf7ugC41LeliYhIdelKHxGRMKRwFxEJQwp3EZEwpHAXEQlDCncRkTBk3LqRhTFmD7Clmr89Dl+PNgh+2ufIoH2ODCezz8nW2kqvAnUt3E+GMWa5tTbd7ToCSfscGbTPkSEQ+6zDMiIiYUjhLiIShkI13Ce5XYALtM+RQfscGfy+zyF5zF1ERE4sVFfuIiJyAkEd7kF5Y24/82Kf7zTGZBpjVhhjPjbGJLtRpy9Vts/lthtljLHGmJDvrPBmn40xl5V9r1cbY94MdI2+5sWf7SRjzKfGmG/L/nyf50advmKMmWyM2W2MWXWc140x5tmy/x8rjDE9fVqAtTYof+GMF94AtAFqAd8DnY/Z5lbghbKvRwNvu113APZ5AFCv7OvfRcI+l20XAywClgLpbtcdgO9zO+BboHHZ4wS36w7APk8Cflf2dWdgs9t1n+Q+9wV6AquO8/p5wFycO9mdDnzpy88P5pV7cN2YOzAq3Wdr7afW2vyyh0tx7owVyrz5PgNMAJ4ACgJZnJ94s883Ac9Za/cDWGt3B7hGX/Nmny3QsOzrRvz6jm8hxVq7iAruSFfOCOB161gKxBpjWvjq84M53Cu6MXfi8bax1pYAR2/MHaq82efybsD5lz+UVbrPxpgeQGtr7axAFuZH3nyf2wPtjTGfG2OWGmOGBqw6//Bmnx8CrjLGZOPcP+L3gSnNNVX9+14lXt2swyU+uzF3CPF6f4wxVwHpQD+/VuR/J9xnY0wU8DRwbaAKCgBvvs81cA7N9Mf56WyxMaartfaAn2vzF2/2+XLgVWvtU8aYM3Du7tbVWuvxf3mu8Gt+BfPK3Wc35g4h3uwzxphBwH3AcGttYYBq85fK9jkG6AosMMZsxjk2OSPET6p6+2f7fWttsbV2E7AWJ+xDlTf7fAPwDoC1dglQB2cGS7jy6u97dQVzuP90Y25jTC2cE6Yzjtnm6I25wZ835g6cSve57BDFizjBHurHYaGSfbbW5lpr46y1KdbaFJzzDMOttcvdKdcnvPmz/R7OyXOMMXE4h2k2BrRK3/Jmn7cCAwGMMZ1wwn1PQKsMrBnAb8u6Zk4Hcq21O3z27m6fUa7kbPN5wDqcs+z3lT33CM5fbnC++e8C64GvgDZu1xyAff4I2AV8V/Zrhts1+3ufj9l2ASHeLePl99kAE4FMYCUw2u2aA7DPnYHPcTppvgMGu13zSe7v/4AdQDHOKv0G4BbglnLf4+fK/n+s9PWfa12hKiIShoL5sIyIiFSTwl1EJAwp3EVEwpDCXUQkDCncRUTCkMJdRCQMKdxFRMKQwl1EJAz9f4pD27/cCaYZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# calculate roc curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, bag_reg_svm_pred)\n",
    "# plot no skill\n",
    "plt.plot([0, 1], [0, 1], linestyle='--')\n",
    "# plot the roc curve for the model\n",
    "plt.plot(fpr, tpr, marker='.')\n",
    "# show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4lFXexvHvLwldQJCIUgRUdI1dIkWkCIKAFAsiICKIVNF3XXVXV9dVXNeya1mloxQRKaJIB2kCIi2IoIAoIAhiiTQLIATO+8cJu5FEMsBMnszM/bmuXGZmHmfux4Tb4cx5zjHnHCIiElsSgg4gIiLhp3IXEYlBKncRkRikchcRiUEqdxGRGKRyFxGJQSp3EZEYpHIXEYlBKncRkRiUFNQLlylTxlWuXDmolxcRiUorVqz4wTmXnNtxgZV75cqVSUtLC+rlRUSikpltCeU4DcuIiMQglbuISAxSuYuIxCCVu4hIDFK5i4jEoFzL3cyGmtn3Zvbp7zxuZvaymW0ws9VmdkX4Y4qIyPEI5Z37cKDJMR5vClTN/OoGDDj5WMewdRksfN7/U0REcpTrPHfn3AIzq3yMQ1oBrzu/X98SMzvVzM50zn0Tpoz/s3UZjGgOGQcgqRDcMRkqVg/7y4iIRLtwjLmXB7Zmub0t875szKybmaWZWVp6evrxv9Lmhb7YcZCxH9KGnUheEZGYF45ytxzuy3HXbefcYOdcqnMuNTk516tns6tcB5IKgyX4l131Joy5DfZsO/7nEhGJYeFYfmAbUDHL7QrA9jA8b3YVq8Mdk/w7+Io14avFsOBf0Lc61H8IavaExAIReWkRkWgSjnfuk4COmbNmagJ7IjLefkTF6lDnfqhcG+o+AHcvhcpXw6y/waC68NWSiL20iEi0CGUq5GhgMXC+mW0zsy5m1sPMemQeMg3YBGwAhgC9IpY2J6UqQ/uxcOso2P8jDL0O3r0bftmRpzFERPIT85Nc8l5qaqoL+6qQB36B+c/C4n5QqDhc+wRcfjsk6FotEYkNZrbCOZea23Gx1XoFi0GjPtB9ISRfAJPv9e/kv/0k6GQiInkqtsr9iLIp0Hka3DAAdm6EQfVgxl/h15+CTiYikidis9wBzOCy9tA7Da7oCEv6Q98rYc0ECGgoSkQkr8RuuR9RtDS0eAm6zIJiZeCtTvDGzbBjY9DJREQiJvbL/YiKV0LX96HJM34Zg/614P1n4OD+oJOJiIRd/JQ7QGKSv9Cp93L4w/Xw/tMwoBZsnBt0MhGRsIqvcj+ixJlwyzDo8I6/PfJGP1zzY+SuvRIRyUvxWe5HnNsQei6G+n+Fz6b5D1yXDIBDGUEnExE5KfFd7gAFCkP9v0CvxX5pgxkPwZD6sHV50MlERE6Yyv2I086BDm/DLSPglx/gtUYw+f9g786gk4mIHDeVe1ZmcOEN/gPXmr3go5HQNxVWjtLceBGJKir3nBQqDk3+Cd3nQ+lzYGIvGNYMvl8XdDIRkZCo3I/ljIvhzpnQ4mVIXwcDr4ZZj/kFykRE8jGVe24SEqDaHdB7BVzaFhb9x28Osm6KhmpEJN9SuYeq2GnQqh90ngGFS8DY22B0W9i1OehkIiLZqNyPV6Va0H0BNHoSvlwI/WrCgn9nbtwtIpI/qNxPRGIBqH0v9F4GVa+FuU/CwNrw5YKgk4mIACr3k1OyAtz6BrR/CzJ+hREt4O2u8PP3QScTkTincg+H8xr7jbrrPujXi38lFZYNgcOHgk4mInFK5R4uBYpAg0f9MgblLoVpD8CrDeHrj4JOJiJxSOUebmWqQsdJcPNr8ON2GNIApj4A+3YHnUxE4ojKPRLM4OLWfhmD6l0h7TW/4uTqcZobLyJ5QuUeSYVLQrN/Qde5/sPXd7rC6y0h/fOgk4lIjFO554Vyl8Nds+H652H7KhhwFczpAwf2Bp1MRGKUyj2vJCTClXfBPWlw0c2w8HnoXwM+nxl0MhGJQSr3vHbK6XDTILhjCiQVgTfbwJjbYPfWoJOJSAxRuQelSh3o8QE0fAw2zIF+1f2iZIcOBp1MRGKAyj1ISQWhzv3+Aqgq9fxywgPrwJYPg04mIlFO5Z4flKoE7cdA29Fw4GcY1hTe7eW3+xMROQEq9/zkD838u/jaf4TVY+GVarBiOBw+HHQyEYkyIZW7mTUxs/VmtsHMHsrh8UpmNsfMVpvZ+2ZWIfxR40TBYtDoCT8eX/ZCv0n30Mbwzeqgk4lIFMm13M0sEegHNAVSgHZmlnLUYf8GXnfOXQL0AZ4Od9C4c/oF0Gkq3DAQdn4Jg+vB9Idg/49BJxORKBDKO/fqwAbn3Cbn3AFgDNDqqGNSgDmZ38/L4XE5EWZwWTu/jEG1TrB0oJ9V8+k7WsZARI4plHIvD2SdhL0t876sVgE3Z35/I1DczE47+XgCQNHS0PxFf5VrsWQY3xneuAl2bAw6mYjkU6GUu+Vw39FvGx8A6pnZSqAe8DWQke2JzLqZWZqZpaWnpx932LhXIRW6zoMmz8LW5dC/Fsx7Gg7uDzqZiOQzoZT7NqBiltsVgO1ZD3DObXfO3eScuxx4JPO+PUc/kXNusHMu1TmXmpycfBKx41hiEtTs4ZcxuKA5zH8GBtTyF0KJiGQKpdyXA1XNrIqZFQTaApOyHmBmZczsyHM9DAwNb0zJpvgZ0Hoo3D4BMD9MM+4Ov4a8iMS9XMvdOZcB9AZmAuuAcc65NWbWx8xaZh5WH1hvZp8DZYGnIpRXjnZOA7/70zWPwPrpft34xf3hULZRMRGJI+YCmnWRmprq0tLSAnntmLVzE0x7EDbMhrIXQ/MXoGL1oFOJSBiZ2QrnXGpux+kK1VhS+my4bTy0eR327oDXGsGke2HvzqCTiUgeU7nHGjNIaQW9l0Gt3rDyDeib6v+pZQxE4obKPVYVKg7XPQXdF8Bp58LEu2F4M/hubdDJRCQPqNxj3RkXQecZ0LIvpK+HgVfDe4/Crz8HnUxEIkjlHg8SEuCK26F3GlzWHj58BfrVgHWTtYyBSIxSuceTYqdBq75w50woXBLGdoA3b4Vdm4NOJiJhpnKPR2fVhO7zofFTsPkD/y5+wb8g49egk4lImKjc41ViAbiqt19x8rzrYO4/YEBt2DQ/6GQiEgYq93hXsryfF3/beDh8EF5vCW/fBT99F3QyETkJKnfxqjaCXkug7p9h7US/jMGyIXD4UNDJROQEqNzlfwoUgQaPQM/FUO4ymPYADGkAX38UdDIROU4qd8muzLnQcSLc/Br89I0v+Kn3w77dQScTkRCp3CVnZnBxa/+Ba43ukDbUL2OwaqzmxotEAZW7HFvhktD0Wb8D1KlnwYRuMKKFv9pVRPItlbuEptxl0GWW38v129V+2uTsJ+DA3qCTiUgOVO4SuoRESL0Teq/wQzYfvAD9a8D6GUEnE5GjqNzl+J2SDDcOhE5TIakIjL4VxtwGu7cGnUxEMqnc5cRVvhp6fADXPg4b50K/6vDBS3DoYNDJROKeyl1OTlJBuPo+uHspnH0NzP67X1Z486Kgk4nENZW7hMepZ0G7N6HdGP8h6/BmMKEn/PJD0MlE4pLKXcLr/KZw9xL/bv6TcfBKNT9HXlv8ieQplbuEX8Fifhy+xyIoexFMuc9v1v3NqqCTicQNlbtEzul/gE5T4MZBsHsLDK4P0x+C/T8GnUwk5qncJbLM4NK2fhmDap1h6UC/4uSnb2sZA5EIUrlL3ihSCpq/AHfNgeJlYfydMPJG2LEx6GQiMUnlLnmrQjW/Tk3Tf8HXK6B/TZj3Tzi4P+hkIjFF5S55LyERanTzQzUXtIT5z/qS3zA76GQiMUPlLsEpfga0fs2vHZ+QCG/cDOM6wo/bg04mEvVU7hK8s+tDzw/hmkfh85n+A9cP+8KhjKCTiUQtlbvkD0mFoN6Dfh/XSlfBe4/A4Hrw1dKgk4lEJZW75C+lq0D7cdBmJOzbBUMbw6R7YO/OoJOJRJWQyt3MmpjZejPbYGYP5fD4WWY2z8xWmtlqM2sW/qgSN8wgpSXcvQyuugdWjvLLGHw0UssYiIQo13I3s0SgH9AUSAHamVnKUYc9Coxzzl0OtAX6hzuoxKFCp0Djf0CPhVDmPJjUG4Y1he/WBJ1MJN8L5Z17dWCDc26Tc+4AMAZoddQxDiiR+X1JQNMdJHzKXgidp0OrfvDD5zCwDrz3KPz6c9DJRPKtUMq9PJB1i51tmfdl9TjQwcy2AdOAe3J6IjPrZmZpZpaWnp5+AnElbiUkwOUd4J4VcPlt8OErfnOQtZO0jIFIDkIpd8vhvqP/NLUDhjvnKgDNgJFmlu25nXODnXOpzrnU5OTk408rUrQ0tHwF7nzPL2kw7nZ4sw3s/DLoZCL5Sijlvg2omOV2BbIPu3QBxgE45xYDhYEy4QgokqOzakC3+XDdP2HLh/4K1/n/goxfg04mki+EUu7LgapmVsXMCuI/MJ101DFfAQ0BzOwCfLlr3EUiKzEJat3tlzE4rwnM+wcMqA2b3g86mUjgci1351wG0BuYCazDz4pZY2Z9zKxl5mH3A13NbBUwGujknAZCJY+UKAdtRsBtb8PhDHi9FYzvAj99G3QykcBYUB2cmprq0tLSAnltiWEH98EHL/qvpMLQ4FG48i6/do1IDDCzFc651NyO0xWqElsKFIFr/uqXMShfDab/GYZc45cXFokjKneJTaedA7dPgNbD4KfvYEhDmPInv6SBSBxQuUvsMoOLbvIfuNboASuG+RUnV43R3HiJeSp3iX2FS0DTZ6Db+3BqJZjQHYY3h+8/CzqZSMSo3CV+nHkpdJkFzV+C7z6FgbVh9uNwYG/QyUTCTuUu8SUhAVI7Q+80uLiNn1XTrwasnx50MpGwUrlLfDolGW4cAJ2mQcFiMLotjG4Hu78KOplIWKjcJb5Vru2XFG7Ux1/Z2q+GfzefcSDoZCInReUuklgAav+f3xzknAZ+HH5QHdj8QdDJRE6Yyl3kiFMrQttR0G4sHNwLw6+HCT3gZy2TJNFH5S5ytPObQK+lUOd++GQ89K0Gy1/TFn8SVVTuIjkpWBQaPgY9F8EZl8DUP8Fr18I3q4JOJhISlbvIsSSfD3dMhpuG+Jk0g+vD9L/A/j1BJxM5JpW7SG7M4JI2fm586p2wdBD0re6HbLSMgeRTKneRUBU5Fa5/HrrOgeJnwNtdYOQN8MOGoJOJZKNyFzle5atB17nQ7N/w9UcwoBbMfcqvJS+ST6jcRU5EQiJU7+qHalJugAXP+X1cv5gVdDIRQOUucnKKl4Wbh0DHSZBQAEa1hrG3w56vg04mcU7lLhIOZ9fz0yYb/A2+eA/6VYcP+8Khg0EnkzilchcJl6RCUPcBuHspVKoN7z0Cg+rBV0uCTiZxSOUuEm6lKkP7sXDrG7B/Nwy9Dib2hl92BJ1M4ojKXSQSzOCCFn4xsqvuhVWjoW8qfPS6ljGQPKFyF4mkQqdA4yeh+0JI/gNMugeGNYFvPw06mcQ4lbtIXiibAp2nQav+sGMDDKoLMx+BX38KOpnEKJW7SF4xg8tv83PjL+8Ai/v6ZQzWTtQyBhJ2KneRvFa0NLR8GbrMhqKnwbiOfn78zk1BJ5MYonIXCUrFK6Hb+9DkGfhqKfSvBfOfg4xfg04mMUDlLhKkxCSo2RN6L4Pzm8K8p3zJb5wXdDKJcip3kfygRDm4ZTh0eAdwfrXJ8XfCT98GnUyilMpdJD85tyH0XAz1H4Z1U6DvlX79+EMZQSeTKBNSuZtZEzNbb2YbzOyhHB5/0cw+zvz63Mx2hz+qSJwoUBjqPwS9FkOFVJj+ZxhyDWxbEXQyiSK5lruZJQL9gKZACtDOzFKyHuOcu885d5lz7jLgFeCdSIQViSunneOHaW4ZDr+kw6sNYfIfYd+uoJNJFAjlnXt1YINzbpNz7gAwBmh1jOPbAaPDEU4k7pnBhTf6ZQxq9oSPRsArqfDxaM2Nl2MKpdzLA1uz3N6WeV82ZlYJqALMPfloIvJfhUtAk6eh23woXQXe7QHDm8P3nwWdTPKpUMrdcrjv994ytAXGO+cO5fhEZt3MLM3M0tLT00PNKCJHnHkJ3PketPgPfPcpDKwNs/4OB34JOpnkM6GU+zagYpbbFYDtv3NsW44xJOOcG+ycS3XOpSYnJ4eeUkT+JyEBqnWCe1bAJW1h0UvQrwZ8NjXoZJKPhFLuy4GqZlbFzAriC3zS0QeZ2flAKWBxeCOKSI6KlYEb+kHnGVCoOIxpD2+2hV1bgk4m+UCu5e6cywB6AzOBdcA459waM+tjZi2zHNoOGOOcPuURyVOVakH3BdDoSfhygX8Xv/AFyDgQdDIJkAXVxampqS4tLS2Q1xaJWXu2wfS/wGdToMz5cP3zUKVO0KkkjMxshXMuNbfjdIWqSCwpWQHajoL24yBjH4xoDu90h5+/DzqZ5DGVu0gsOu866LUU6jwAn77tt/hb/iocznEim8QglbtIrCpYFBr+zS9jcOalMPV+6F8Tpj4AW5cFnU4iTOUuEuvKVIWOk6Dew/DD57B8CAxrCls0sS2WqdxF4oEZJBUAS/S3D2fAhB7w4+9dsiLRTuUuEi8q14HEgr7gEwrAT9/AgNqwfkbQySQCkoIOICJ5pGJ1uGMSbF7oi75IKRjfGUbfCjV6QKM+kFQo6JQSJip3kXhSsbr/OuKuOTDrMVg6ELYsgtbD/Bi9RD0Ny4jEs6RC0PRZaDcG9nwNg+rBylFaTjgGqNxFxG/O3XMRlL8CJvaCd7rC/h+DTiUnQeUuIl6JctBxIlzzqL/waVBd+Fpb+0UrlbuI/E9CItR7EDpP99MlX2sMi16Gw4eDTibHSeUuItmdVRN6LPTDNbP+BqNaa32aKKNyF5GcFSkFbUbC9S/4mTQDasNG7aAZLVTuIvL7zODKLtB1HhQtDSNv9FMnDx0MOpnkQuUuIrkrm+ILvlpnWPQfGHod7Pwy6FRyDCp3EQlNwaLQ4iW4ZQT8sMHPpvlkfNCp5Heo3EXk+Fx4A/T8AE6/AN7uAhPvhgO/BJ1KjqJyF5Hjd+pZ0Gma3wxk5SgYXB++/SToVJKFyl1ETkxikt8MpONEfzXrkIawdLCWLsgnVO4icnLOrueXLji7Pkx/EMa0h707g04V91TuInLyipWB9mPhuqfhi1l+TvzmD4JOFddU7iISHmZQqxfcNRsKFIERLWDe03AoI+hkcUnlLiLhVe4y6D4fLmkL85/xJb9nW9Cp4o7KXUTCr1BxuHEA3DgYvl3th2nWTQk6VVxRuYtI5Fx6K3RfAKUqw9jbYOr9cHBf0KnigspdRCLrtHOgyyyo1RuWv+qnTH7/WdCpYp7KXUQiL6kgXPcU3DYefv7OX/S0YoTmxEeQyl1E8k7VRn5O/Fk1YPK9ML4z7NsddKqYpHIXkbxV/AzoMAGufRzWTYZBdWDr8qBTxRyVu4jkvYQEuPo+6DzD3x56HSx8Qdv5hVFI5W5mTcxsvZltMLOHfueYNma21szWmNmb4Y0pIjGp4pXQfSGktIQ5T8DIG+Cnb4NOFRNyLXczSwT6AU2BFKCdmaUcdUxV4GGgtnPuQuCPEcgqIrGoyKnQehi0fAW2LvNz4r+YFXSqqBfKO/fqwAbn3Cbn3AFgDNDqqGO6Av2cc7sAnHPaSVdEQmcGV3T0V7YWP8NvyD3zEcg4EHSyqBVKuZcHtma5vS3zvqzOA84zs0VmtsTMmuT0RGbWzczSzCwtPT39xBKLSOxKPh/umgNXdoXFfeG1RrBjY9CpolIo5W453Hf05NQkoCpQH2gHvGpmp2b7l5wb7JxLdc6lJicnH29WEYkHBQrD9f+GW0fBrs1+O79VY4NOFXVCKfdtQMUstysA23M4ZqJz7qBz7ktgPb7sRUROzAXN/Zz4My6BCd1gQg/49aegU0WNUMp9OVDVzKqYWUGgLTDpqGPeBa4BMLMy+GGaTeEMKiJxqGQFuGMy1H8YVo+FQfVg+8qgU0WFXMvdOZcB9AZmAuuAcc65NWbWx8xaZh42E9hhZmuBecCDzrkdkQotInEkMQnqPwR3TPGLjr3aCBb309IFuTAX0H+g1NRUl5aWFshri0iU2rsTJvaG9VOhamO4YYDfBSqOmNkK51xqbsfpClURiR5FS0PbUdDs37Bpvp8Tv2l+0KnyJZW7iEQXM6jeFbrOgcIl4PVWMKcPHDoYdLJ8ReUuItHpjIuh2/tweQdY+DwMawa7tgSdKt9QuYtI9CpYDFr1hdZDIf0zGFgH1kwIOlW+oHIXkeh30c3QYyGUqQpvdYLJ/wcH9gadKlAqdxGJDaUqw50zoPYfYcVwGHINfLcm6FSBUbmLSOxILACNnoDbJ8C+XTCkgd+3NQ7nxKvcRST2nNMAeiyCylfD1Pth3O1+jnwcUbmLSGw6JRnavwWN/wHrZ/gPW7csDjpVnlG5i0jsSkiAq+6BLu/5IZvhzWD+c3D4UNDJIk7lLiKxr/wV0H0BXNQa5j3lL3z68ejFbWOLyl1E4kPhEnDTYL8ezdcfwYCr4LNpQaeKGJW7iMQPM7isvX8XX7IijGkH0/4MB/cHnSzsVO4iEn/KnAt3zYaavWDZIHjtWvjhi6BThZXKXUTiU1IhaPI0tBvrx98H1YWVb8TMnHiVu4jEt/Ob+Dnx5avBxLvh7btg/49BpzppKncRkRJnQseJ0OBRv/DYoDqwbUXQqU6Kyl1EBCAhEeo+CJ2n+3nwQxvDov/A4cNBJzshKncRkazOquFXmDy/Gcx6DEa1hp+/DzrVcVO5i4gcrUgpaPM6NH8Rtizyc+I3zAk61XFRuYuI5MQMUu/0uz0VLQNv3ATv/Q0yDgSdLCQqdxGRYzn9Aug2zxf9hy/DsCaw88ugU+VK5S4ikpsCRfwQTZvXYccGv8LkJ+ODTnVMKncRkVCltIIeH0DZC+HtLvDu3XDgl6BT5UjlLiJyPE49CzpN9dMmPx4Fg+rBN6uDTpWNyl1E5HglJvkLnu6YBAd+hlcbwpKB+WrpApW7iMiJqlLXL11wTgOY8RcY3Q5+2RF0KkDlLiJycoqdBu3GQJNnYeMcGHg1fLkw6FQqdxGRk2YGNXv4ZYQLFoURLWDuU3AoI7BIKncRkXA581LoNt9vCLLgORjRHHZvDSRKSOVuZk3MbL2ZbTCzh3J4vJOZpZvZx5lfd4U/qohIFCh0CtzQH256Fb791A/TrJuc5zFyLXczSwT6AU2BFKCdmaXkcOhY59xlmV+vhjmniEh0ueQW6LEASleBsR1gyp/g4L48e/mkEI6pDmxwzm0CMLMxQCtgbSSDiYhEvdJnw53vwdwn/dIFXy2Gq++DPVuhch2oWD1iLx1KuZcHsg4abQNq5HDczWZWF/gcuM85F7GBplsHLc52X/NLzuT2WpXZd+AQnYYty/Z462oVuCW1Ijt/OUDPN7Ivwt+hZiVaXFqO7bv3cd/Yj7M93rXO2VybUpaN6T/z13c+yfb4PQ2qcnXVMqzZvoc+k7P/f+/PTc6nWqXSrNiyk+dmrM/2+GMtUriwXEk++OIHXpmbfS/Hf950Meckn8Lstd8xZOGmbI+/eOtllDu1CJNXbeeNJVuyPT6gQzVKFyvIW2lbGb9iW7bHh3euTpGCiYxcvJkpq7/J9vjY7rUAGLxgI3PW/Xb508IFEhlxp/8lfXnOFyza8MNvHi9VtCADb68GwLMzPuOjLbt+8/iZJQvzUtvLAXhi8hrWbv/tLjhnJxfj6ZsuAeDhd1azKf23VwSmlCvB31tcCMAfx6zkmz2/3ez4ikql+EuTPwDQY+QKdu397cJPtc8tw70NqwJwx9Bl7D946DePN7zgdLrVPQfQ755+907wd6/xk7z8ZXk6f/skp7zTFcMgqbCfJx+hgg9lzN1yuO/omfqTgcrOuUuA2cCIHJ/IrJuZpZlZWnp6+vElFRGJYmuLXsmsotdn3nJw6ABsjtyUSXO5XFFlZrWAx51z12XefhjAOff07xyfCOx0zpU81vOmpqa6tLS0EwotIhKVti6DES19sScWPKF37ma2wjmXmttxoQzLLAeqmlkV4GugLdD+qBc70zl35O9TLYF1x5VWRCQeVKzuC33zwuDH3J1zGWbWG5gJJAJDnXNrzKwPkOacmwTca2YtgQxgJ9ApYolFRKJZxeoRLfUjch2WiRQNy4iIHL9Qh2V0haqISAxSuYuIxCCVu4hIDFK5i4jEIJW7iEgMCmy2jJmlA9mvVQ5NGeCHXI+KLTrn+KBzjg8nc86VnHPJuR0UWLmfDDNLC2UqUCzROccHnXN8yItz1rCMiEgMUrmLiMSgaC33wUEHCIDOOT7onONDxM85KsfcRUTk2KL1nbuIiBxDvi73EDbmLmRmYzMfX2pmlfM+ZXiFcM5/MrO1ZrbazOaYWaUgcoZTbuec5bjWZubMLOpnVoRyzmbWJvNnvcbM3szrjOEWwu/2WWY2z8xWZv5+NwsiZ7iY2VAz+97MPv2dx83MXs7877HazK4IawDnXL78wi8vvBE4GygIrAJSjjqmFzAw8/u2+E26A88e4XO+Biia+X3PeDjnzOOKAwuAJUBq0Lnz4OdcFVgJlMq8fXrQufPgnAcDPTO/TwE2B537JM+5LnAF8OnvPN4MmI7f7a4msDScr5+f37n/d2Nu59wB4MjG3Fm14n9b+o0HGppZTtsCRotcz9k5N885tzfz5hKgQh5nDLdQfs4ATwLPAftzeCzahHLOXYF+zrldAM6574luoZyzA0pkfl8S2J6H+cLOObcAv7/F72kFvO68JcCpZnZmuF4/P5d7Thtzl/+9Y5xzGcAe4LQ8SRcZoZxzVl3w/+ePZrmes5ldDlR0zk3Jy2ARFMrP+TzgPDNbZGZLzKxJnqWLjFDO+XGgg5ltA6YB9+RNtMAc75/34xLKNntBCWVj7lCOiSYhn4+ZdQBSgXoRTRR5xzxnM0sAXiS2dvcK5eechB+aqY//29lCM7vIObc7wtkiJZRzbgcMd849n7l388jMcz4c+XiBiGh/5ed37tuAilluVyD7X9P+e4yZJeH/Knesvwbld6GcM2Z2LfAI0NI592seZYuU3M65OHAR8L6ZbcaPTU6K8g9VQ/3dnuicO+ic+xJYjy/7aBXKOXcBxgE45xYDhfHMNiEXAAABK0lEQVRrsMSqkP68n6j8XO7/3ZjbzAriPzCddNQxk4A7Mr9vDcx1mZ9URKlczzlziGIQvtijfRwWcjln59we51wZ51xl51xl/OcMLZ1z0bxHYyi/2+/iPzzHzMrgh2k25WnK8ArlnL8CGgKY2QX4ck/P05R5axLQMXPWTE1gj3Pum7A9e9CfKOfyaXMz4HP8p+yPZN7XB/+HG/wP/y1gA7AMODvozHlwzrOB74CPM78mBZ050ud81LHvE+WzZUL8ORvwArAW+ARoG3TmPDjnFGARfibNx0DjoDOf5PmOBr4BDuLfpXcBegA9svyM+2X+9/gk3L/XukJVRCQG5edhGREROUEqdxGRGKRyFxGJQSp3EZEYpHIXEYlBKncRkRikchcRiUEqdxGRGPT/ErAsUmMQwWQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, bag_reg_svm_pred)\n",
    "# plot no skill\n",
    "plt.plot([0, 1], [0.5, 0.5], linestyle='--')\n",
    "# plot the precision-recall curve for the model\n",
    "plt.plot(recall, precision, marker='.')\n",
    "# show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, '2-class Precision-Recall curve: AP=0.62')"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAG2dJREFUeJzt3XmYJXV97/H3xxkWlXFQR4wOA6MsUUQUHUFiohiQC0TBq15l3MAQcCPqdUm810RHjNftGq+JuGAwKqIIPsaMihJFFDSiM1wWBYWMCDKCERAGWWT95o+qZo5Nd/XpZqpPz/B+PU8/fU7V79T5nt9ZPlW/OlUnVYUkSZO5z6gLkCTNbQaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0GxEUtyWJLvjrqODS3JBUn2nqLNdkluSDJvlsrqXZJLk+zbXl6R5DOjrkkCg2LWJdkiyXFJLkvy2yTnJDlg1HUNo/0gu7n9gP7PJP+cZKsNfT9V9Ziq+vYUbX5RVVtV1R0b+v7bD+nb2sd5XZJ/T7LXhr6fe4skn0xye5KHj5u+Qfo5yQvb99ONSb6U5EEdbecl+bskVwy8/7Zu5x2a5Owk1ydZm+S9SeZP/xFvegyK2TcfuBx4GrAQ+FvgpCRLR1jTdDyrqrYCngA8Cfib8Q3S2NhfW59vH+ci4HTg5BHXs8HNxodgkvsDzwXWAS+aoMlYPz8E+C7wxSSZxvIfA3wMeAnwUOAm4MMdN3k78EfAXsAD2tv9rp13P+B1NM/5nsA+wBuHrWVTtrG/mTc6VXVjVa2oqkur6s6q+grwc+CJk90myZIkX0xyVZJrknxoknYfTHJ5u0Z0dpI/GZi3R5LV7bz/TPL37fQtk3ymXe51SVYleegQj+OXwNeAXdvlfDvJO5N8j+bN+sgkC9utpyuT/LJdk7trqCjJEUl+0q7ZXZjkCe30wSGYyepemqTGPuySPDzJyiS/SbImyRED97MiyUlJPt3e1wVJlk31GNvHeTtwArA4yUMGlvnMJOcOrAnvNjBvwucryQ5JvtVOuzrJCWNrs9OV5OD2/q9P8rMk+4/vu4HH/plxfXZ4kl8A30ry9SRHjVv2eUme015+VJJvtP16UZLnT7PU5wLXAUcDh07WqKpuAz4F/AHw4Gks/0XAl6vqjKq6gWbF6zlJFoxvmOSBNEFwRFVdVo0fV9Xv2ho+UlVnVtWt7ev7BOAp06hlk2VQjFj7obwzcMEk8+cBXwEuA5YCi4ETJ1ncKuDxwIOAzwInJ9mynfdB4INV9QBgB+CkdvqhNFs2S2jeoK8Abh6i7iXAgcA5A5NfAhwJLGjr/RRwO7AjsDuwH/AX7e3/B7ACeCnNmt1BwDUT3NVkdY/3OWAt8HDgecD/SbLPwPyDaPpta2AlMGHYTvA4N29rvAa4tp32BOATwMtp+uxjwMo0w4pdz1eAd7U1Ppqmz1cMU8e4mvYAPg28qX08TwUuncYintbe/3+jeZ0sH1j2LsD2wFfbrYFvtG22adt9uF2LHxvyOX+K+zqU5rk5EXjU2MrABI9pC+AwYG1VXZ3kj9sQnuzvj9ubPgY4b2w5VfUz4Faa99R4j6V5PT4vya+SXJzk1R21P5VJ3pf3OlXl34j+gM2AbwIf62izF3AVMH+CeYcB3+247bXA49rLZ9Bsdi8a1+bPgX8Hdhui3kuBG2jWEC+j2cS/bzvv28DRA20fCtwyNr+dthw4vb18KvDajvvZd4q6lwJFM5S3BLgDWDAw/13AJ9vLK4BvDszbBbi543GuoPmwua5d7jXA3gPzPwK8Y9xtLqL5AJ70+Zrgfp4NnDPJ414BfGaS230M+MBUfTd+OQN99siB+QuAG4Ht2+vvBD7RXn4BcOYE9/22IV/f2wF3Ao8feM4/OEk//xr4FvDEab6HTgNeMW7aLwefr4HpL2wf/3HAfYHd2ufqGRO0fRnNisei6dSzqf65RTEiacbwj6d5oxw1MP1raXbu3ZDkRTQfgpdVMwQy1TLf0A7lrEtyHc2WwqJ29uE0a1k/bYeXntlOP57mDXximh18702yWcfdPLuqtq6q7avqVVU1uPVx+cDl7WmC8MqxtUCaD5lt2vlLgJ9N9Zg66h70cOA3VfXbgWmX0azNj/nVwOWbgC2TzE/yooH+/tpAm5OqamuawPsxvz80uD3whsE13PbxPJyO5yvJNklObIfhrgc+w/rnZzqG7bvJ3PU8tX32VeCQdtIhNEMu0DzOPcc9zhfRDA8N4yXAT6rq3Pb6CcALx72+TmpfT9tU1Z9W1dnTfCw30GyRDnoA8NsJ2o69Vo+uqpur6nyaLZ0DBxsleTbwbuCAqrp6mvVsktyjPwJJQrNW81DgwGrGZwGoqgPGtd0L2C7J/K6wSLM/4q9pdsBdUFV3JrmWZriDqvoPYHkbUM8BvpDkwVV1I80a+9vT7FA/hWbt+LgZPLTBUxFfTrNFsWiSui+nGUrqXuAkdY9rdgXwoCQLBsJiO5o1y6mWfwLrPxgnmn91kpcDq5J8tqqubGt/Z1W9c3z7KZ6vd9H00W5VdU37gTTUENg4XX13I81O2TETfaiPP2X054C3JTmDZk379IH7+U5VPWMGNUIzZLddkrGQnk8zVHcAzfDfpNrX89c6mhxQVWfSDA09buB2jwS2AC6e4DZjw2STnjK73dfzceDPqupHXTXem7hFMRofoRkjfta4NfKJ/BC4Enh3kvun2fk80Q62BTTjr1cB85O8lYE1rSQvTvKQqrqTZlMf4I4kT0/y2HZs/XrgNprhlnuk/UD9N+D9SR6Q5D5pduY+rW3yT8AbkzwxjR2TbD9+OZPVPe6+LqcZPntX2z+70WyJTBoA03wsP6XZ6vqrdtLHgVck2bOt/f5J/qzdgdr1fC2gHbpLsphmH8NMHAe8LMk+bb8uTvKodt65wCFJNkuzw/55QyzvFJqth6NpvoV0Zzv9K8DOSV7SLm+zJE9K8uipFtgG5g7AHjT7zR5P88WHz9KxU3tMNTuVt+r4O7NtegLwrCR/0u5TORr44rity7Fl/gw4E3hLmv1Jj6YZXvtKW/Oftst7blX9cKoa700MilnWfhi+nOaN86txw0x3U81xAs+i2SH8C5px0xdM0PRUmjWwi2mGXX7H7w8F7Q9ckOQGmh3Eh1TzbY8/AL5AExI/Ab5DMySyIbwU2By4kGZ/yReAh7WP62Sa8fDP0gwTfIlmJ/x4k9U93nKaMfgrgH+hGUf/xgZ6HADvA45Msk1VrQaOoNkauBZYQ7O/aKrn6+00XyteRzPc88WZFNJ+iL0M+EC7rO/QfNBD862fHdq63k7Tv1Mt75a2ln0H27cftvvRDEddQTN89x6aNXbaYbvJdvYeCvxrVf2oqn419kfzHD4zHcc6TEdVXUDzBYwTaPZzLABeNTa/Hcr93wM3WU7TV9fQPAd/W1WntfP+lma49pRJhiPvtVLlDxdJkibnFoUkqZNBIUnqZFBIkjoZFJKkThvdcRSLFi2qpUuXjroMSdqonH322VdX1UOmbnl3G11QLF26lNWrV4+6DEnaqCS5bKa3dehJktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHXqLSiSfCLJr5P8eJL5SfIPaX7f+PxM8hOJkqTR6nOL4pM0p4iezAHATu3fkTS/0SBJmmN6C4qqOgP4TUeTg4FPV+MsYOskD5tqubfeuqEqlCQNY5T7KBbz+z+ss5bf/43juyQ5MsnqJKuvvPLaWSlOktQYZVBkgmkT/opSVR1bVcuqatnChQ/suSxJ0qBRBsVaYMnA9W1pfm5RkjSHjDIoVgIvbb/99GRgXVVdOcJ6JEkT6O3ssUk+B+wNLEqyFngbsBlAVX0UOAU4kOaH6W+i+bF4SdIc01tQVNXyKeYX8Oq+7l+StGF4ZLYkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE69BkWS/ZNclGRNkjdPMH+7JKcnOSfJ+UkO7LMeSdL09RYUSeYBxwAHALsAy5PsMq7Z3wAnVdXuwCHAh/uqR5I0M31uUewBrKmqS6rqVuBE4OBxbQp4QHt5IXBFj/VIkmagz6BYDFw+cH1tO23QCuDFSdYCpwB/OdGCkhyZZHWS1evWXdtHrZKkSfQZFJlgWo27vhz4ZFVtCxwIHJ/kbjVV1bFVtayqli1c+MAeSpUkTabPoFgLLBm4vi13H1o6HDgJoKq+D2wJLOqxJknSNPUZFKuAnZI8IsnmNDurV45r8wtgH4Akj6YJiqt6rEmSNE29BUVV3Q4cBZwK/ITm200XJDk6yUFtszcARyQ5D/gccFhVjR+ekiSNUDa2z+Udd1xWa9asHnUZkrRRSXJ2VS2byW09MluS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdZo/6gKm68474eKLR12FpDEPehAsWjTqKtSnjS4oAM44Y9QVSAK45ZYmKJYvH3Ul6tNGFxTz58Puu4+6CkkAl10Gv/nNqKtQ39xHIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOg19Co8ki4HtB29TVZ51SZI2cUMFRZL3AC8ALgTuaCcX0BkUSfYHPgjMA/6pqt49QZvnAyva5Z1XVS8ctnhJUv+G3aJ4NvCHVXXLsAtOMg84BngGsBZYlWRlVV040GYn4H8BT6mqa5NsM3zpkqTZMOw+ikuAzaa57D2ANVV1SVXdCpwIHDyuzRHAMVV1LUBV/Xqa9yFJ6tmwWxQ3AecmOQ24a6uiql7TcZvFwOUD19cCe45rszNAku/RDE+tqKqvD1mTJGkWDBsUK9u/6cgE02qC+98J2BvYFjgzya5Vdd3vLSg5EjgSYJtttptmGZKke2KooKiqTyXZnHYLALioqm6b4mZrgSUD17cFrpigzVntsn6e5CKa4Fg17v6PBY4F2HnnZePDRpLUo6H2USTZG/gPmp3THwYuTvLUKW62CtgpySPakDmEu2+VfAl4ensfi2iC6JKhq5ck9W7Yoaf3A/tV1UUASXYGPgc8cbIbVNXtSY4CTqXZ//CJqrogydHA6qpa2c7bL8nY127fVFXXzPzhSJI2tGGDYrOxkACoqouTTPktqKo6BThl3LS3Dlwu4PXtnyRpDho2KFYnOQ44vr3+IuDsfkqSJM0lwwbFK4FXA6+h+TbTGTT7KiRJm7hhv/V0C/D37Z8k6V6kMyiSnFRVz0/yI+5+DARVtVtvlUmS5oSptihe2/5/Zt+FSJLmps7jKKrqyvbi1cDlVXUZsAXwOO5+8JwkaRM07EkBzwC2bH+T4jTgZcAn+ypKkjR3DBsUqaqbgOcA/1hV/x3Ypb+yJElzxdBBkWQvmuMnvtpOG/rX8SRJG69hg+J1ND8w9C/taTgeCZzeX1mSpLli2OMovgN8Z+D6JTQH30mSNnFTHUfx/6rqdUm+zMTHURzUW2WSpDlhqi2KsXM7/d++C5EkzU2dQVFVYyf+Ww3cXFV3AiSZR3M8hSRpEzfsN5dOA/YFbmiv3xf4N+CP+ihK0sbjllvg4otHXYX6NGxQbFlVYyFBVd2Q5H491SRpI7FwIfzqV3DGGaOuRFNbcP+Z3nLYoLgxyROq6v8DJHkicPNM71TSpmHrrWHPPUddhYYzb95MbzlsULwOODnJ2PmdHga8YKZ3KknaeAx7HMWqJI8C/pDmh4t+WlW39VqZJGlOGOrI7HZ/xF8Dr62qHwFLk3jqcUm6Fxj2FB7/DNwK7NVeXwv8XS8VSZLmlGGDYoeqei9wG0BV3UwzBCVJ2sQNGxS3Jrkv7Wk8kuwA3NJbVZKkOWPYbz29Dfg6sCTJCcBTgMP6KkqSNHdMGRRJAvyU5keLnkwz5PTaqrq659okSXPAlEFRVZXkS1X1RNb/aJEk6V5i2H0UZyV5Uq+VSJLmpGH3UTwdeEWSS4EbaYafqqp266swSdLcMGxQHNBrFZKkOWuqX7jbEngFsCPwI+C4qrp9NgqTJM0NU+2j+BSwjCYkDgDe33tFkqQ5Zaqhp12q6rEASY4Dfth/SZKkuWSqLYq7zhDrkJMk3TtNFRSPS3J9+/dbYLexy0mun2rhSfZPclGSNUne3NHueUkqybLpPgBJUr86h56qasa/iJRkHnAM8Ayas82uSrKyqi4c124B8BrgBzO9L0lSf4Y94G4m9gDWVNUlVXUrcCJw8ATt3gG8F/hdj7VIkmaoz6BYDFw+cH1tO+0uSXYHllTVV7oWlOTIJKuTrF637qoNX6kkaVJ9BsVEv1dRd81M7gN8AHjDVAuqqmOrallVLVu48CEbsERJ0lT6DIq1wJKB69sCVwxcXwDsCny7PTXIk4GV7tCWpLmlz6BYBeyU5BFJNgcOAVaOzayqdVW1qKqWVtVS4CzgoKpa3WNNkqRp6i0o2uMujgJOBX4CnFRVFyQ5OslBfd2vJGnDGvakgDNSVacAp4yb9tZJ2u7dZy2SpJnpc+hJkrQJMCgkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUyaCQJHUyKCRJnQwKSVIng0KS1MmgkCR1MigkSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLUqdegSLJ/kouSrEny5gnmvz7JhUnOT3Jaku37rEeSNH29BUWSecAxwAHALsDyJLuMa3YOsKyqdgO+ALy3r3okSTPT5xbFHsCaqrqkqm4FTgQOHmxQVadX1U3t1bOAbXusR5I0A30GxWLg8oHra9tpkzkc+NpEM5IcmWR1ktXr1l21AUuUJE2lz6DIBNNqwobJi4FlwPsmml9Vx1bVsqpatnDhQzZgiZKkqczvcdlrgSUD17cFrhjfKMm+wFuAp1XVLT3WI0magT63KFYBOyV5RJLNgUOAlYMNkuwOfAw4qKp+3WMtkqQZ6i0oqup24CjgVOAnwElVdUGSo5Mc1DZ7H7AVcHKSc5OsnGRxkqQR6XPoiao6BThl3LS3Dlzet8/7lyTdcx6ZLUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSepkUEiSOhkUkqROBoUkqZNBIUnqZFBIkjoZFJKkTgaFJKmTQSFJ6mRQSJI6GRSSpE4GhSSpk0EhSerUa1Ak2T/JRUnWJHnzBPO3SPL5dv4Pkiztsx5J0vT1FhRJ5gHHAAcAuwDLk+wyrtnhwLVVtSPwAeA9fdUjSZqZPrco9gDWVNUlVXUrcCJw8Lg2BwOfai9/AdgnSXqsSZI0TfN7XPZi4PKB62uBPSdrU1W3J1kHPBi4erBRkiOBI9trty1b9sBLe6l4o3PLQthi3airmBvsi/Xsi/Xsi/Wu33amt+wzKCbaMqgZtKGqjgWOBUiyuuraZfe8vI1f0xc32RfYF4Psi/Xsi/WSrJ7pbfsceloLLBm4vi1wxWRtkswHFgK/6bEmSdI09RkUq4CdkjwiyebAIcDKcW1WAoe2l58HfKuq7rZFIUkand6Gntp9DkcBpwLzgE9U1QVJjgZWV9VK4Djg+CRraLYkDhli0cf2VfNGyL5Yz75Yz75Yz75Yb8Z9EVfgJUldPDJbktTJoJAkdZqzQeHpP9Yboi9en+TCJOcnOS3J9qOoczZM1RcD7Z6XpJJssl+NHKYvkjy/fW1ckOSzs13jbBniPbJdktOTnNO+Tw4cRZ19S/KJJL9O8uNJ5ifJP7T9dH6SJwy14Kqac380O79/BjwS2Bw4D9hlXJtXAR9tLx8CfH7UdY+wL54O3K+9/Mp7c1+07RYAZwBnActGXfcIXxc7AecAD2yvbzPqukfYF8cCr2wv7wJcOuq6e+qLpwJPAH48yfwDga/RHMP2ZOAHwyx3rm5RePqP9absi6o6vapuaq+eRXPMyqZomNcFwDuA9wK/m83iZtkwfXEEcExVXQtQVb+e5RpnyzB9UcAD2ssLufsxXZuEqjqD7mPRDgY+XY2zgK2TPGyq5c7VoJjo9B+LJ2tTVbcDY6f/2NQM0xeDDqdZY9gUTdkXSXYHllTVV2azsBEY5nWxM7Bzku8lOSvJ/rNW3ewapi9WAC9OshY4BfjL2Sltzpnu5wnQ7yk87okNdvqPTcDQjzPJi4FlwNN6rWh0OvsiyX1ozkJ82GwVNELDvC7m0ww/7U2zlXlmkl2r6rqea5ttw/TFcuCTVfX+JHvRHL+1a1Xd2X95c8qMPjfn6haFp/9Yb5i+IMm+wFuAg6rqllmqbbZN1RcLgF2Bbye5lGYMduUmukN72PfIv1bVbVX1c+AimuDY1AzTF4cDJwFU1feBLYFFs1Ld3DLU58l4czUoPP3HelP2RTvc8jGakNhUx6Fhir6oqnVVtaiqllbVUpr9NQdV1YxPhjaHDfMe+RLNFx1IsohmKOqSWa1ydgzTF78A9gFI8miaoLhqVqucG1YCL22//fRkYF1VXTnVjebk0FP1d/qPjc6QffE+YCvg5HZ//i+q6qCRFd2TIfviXmHIvjgV2C/JhcAdwJuq6prRVd2PIfviDcDHk/xPmqGWwzbFFcskn6MZalzU7o95G7AZQFV9lGb/zIHAGuAm4GVDLXcT7CtJ0gY0V4eeJElzhEEhSepkUEiSOhkUkqROBoUkqZNBIY2T5I4k5yb5cZIvJ9l6Ay//sCQfai+vSPLGDbl8aUMzKKS7u7mqHl9Vu9Ico/PqURckjZJBIXX7PgMnTUvypiSr2nP5v31g+kvbaeclOb6d9qz2t1LOSfLNJA8dQf3SPTYnj8yW5oIk82hO+3Bce30/mnMl7UFzcrWVSZ4KXENznq2nVNXVSR7ULuK7wJOrqpL8BfBXNEcISxsVg0K6u/smORdYCpwNfKOdvl/7d057fSua4Hgc8IWquhqgqsZOTrkt8Pn2fP+bAz+fleqlDcyhJ+nubq6qxwPb03zAj+2jCPCudv/F46tqx6o6rp0+0blw/hH4UFU9Fng5zYnopI2OQSFNoqrWAa8B3phkM5qTzv15kq0AkixOsg1wGvD8JA9up48NPS0EftlePhRpI+XQk9Shqs5Jch5wSFUd356i+vvtWXpvAF7cnqn0ncB3ktxBMzR1GM2vqp2c5Jc0pzx/xCgeg3RPefZYSVInh54kSZ0MCklSJ4NCktTJoJAkdTIoJEmdDApJUieDQpLU6b8AN/2SzLwBfqYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument\n",
    "step_kwargs = ({'step': 'post'})\n",
    "plt.step(recall, precision, color='b', alpha=0.2,\n",
    "         where='post')\n",
    "plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)\n",
    "\n",
    "from sklearn.metrics import average_precision_score\n",
    "average_precision = average_precision_score(y_test, bag_reg_svm_pred)\n",
    "\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEGCAYAAADscbcsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF/1JREFUeJzt3X+wX3Wd3/HniwQQihgg4moSN6lEdwOjCDEya7dF2CbBsYbtQCfpdMlqZhkp7Gj3Rwlrp3RVprjuyCwtMmWHDMFaAotSMm1ojMgsa4cAWcRARMgVq1yhRkhkGVl+JPfVP87n1m8u3/s95yT3m/vj+3rMnLnn+z6fc87ney958/mczzmfI9tERERvR012BSIipoMky4iIBpIsIyIaSLKMiGggyTIiooEky4iIBpIs+0jSSklPShqStH6y6xP1JG2QtEfS45Ndl5hakiz7RNIs4AbgAmAJsEbSksmtVTRwC7BysisRU0+SZf8sA4ZsP237NWATsGqS6xQ1bN8P7J3sesTUk2TZP/OAZzo+D5dYRExDSZb9oy6xPFsaMU0lWfbPMLCg4/N84NlJqktEHKYky/55GFgsaZGkY4DVwOZJrlNEHKIkyz6xvR+4AtgKPAHcYXvX5NYq6ki6DXgAeI+kYUnrJrtOMTUoU7RFRNRLyzIiooEky4iIBpIsIyIaSLKMiGggyfIIkHTpZNch2snfLMZKsjwy8g9v+snfLA6SZBkR0cCUus9y7smzvHDB0ZNdjQn3sxcO8NZTZk12NfriqZ3HT3YV+uJ1XuVojp3saky4V/gFr/nVbvMWNLbiw//AL+w90Kjs3+58davtGTHl3ezJrkCnhQuO5qGtC+oLxpSx4h1nTnYVooUHfe9hH+OFvQd4aOs7G5Wd9fbdcw/7hFPElEqWETH1GRhhZLKrccQlWUZEK8a87mbd8JkkyTIiWkvLMiKihjEHptDA8JGSZBkRrY0M4KT/SZYR0YqBA0mWERH10rKMiKhh4PVcs4yI6M043fCIiFqGA4OXK5MsI6Kd6gmewZNkGREtiQMc1lwc01KSZUS0Ug3wDF6yzHyWEdFKdZ+lGi29SFog6T5JT0jaJelTJf5FSd+XtFPSXZLmdOxzlaQhSU9KWtERX1liQ5LWd8QXSXpQ0m5Jt0s6psSPLZ+HyvaFdd87yTIiWhuxGi019gN/aPvXgXOAyyUtAbYBZ9h+L/AUcBVA2bYaOB1YCXxZ0ixJs4AbgAuAJcCaUhbgC8B1thcD+4B1Jb4O2Gf7NOC6Uq6nJMuIaGWiWpa2n7P9SFl/CXgCmGf7G7b3l2LbgfllfRWwyfartn8IDAHLyjJk+2nbrwGbgFWSBJwH3Fn23whc2HGsjWX9TuD8Un5cSZYR0YoRBziq0QLMlbSjY+n6bqPSDX4/8OCYTZ8A7inr84BnOrYNl9h48VOAn3ck3tH4Qccq218s5ceVAZ6IaK1BF3vU87aX9iog6QTga8Cnbf9dR/wzVF31r46Guuxuujf63KN8r2ONK8kyIlox4jVPzDulJB1NlSi/avvrHfG1wEeB8/3LF4UNA53vnZkPPFvWu8WfB+ZIml1aj53lR481LGk28BZgb6+6phseEa1UN6Uf1WjppVwjvBl4wvaXOuIrgSuBj9l+uWOXzcDqMpK9CFgMPAQ8DCwuI9/HUA0CbS5J9j7gorL/WuDujmOtLesXAd9yzdsb07KMiNYm6Kb0DwG/Azwm6dES+xPgeuBYYFsZc9lu+5O2d0m6A/geVff8crt6v4WkK4CtwCxgg+1d5XhXApskfR74DlVypvz8iqQhqhbl6rrKJllGRCu2OODD75Ta/jbdrx1u6bHPNcA1XeJbuu1n+2mq0fKx8VeAi9vUN8kyIlobyeOOERG9VQM8g5c6Bu8bR8RhGR3gGTRJlhHR2oEBnEgjyTIiWhl9gmfQJFlGRGsjEzAaPt0kWUZEK9VEGkmWERE9GfH6BD3uOJ0kWUZEKzYTclP6dJNkGREtKTelR0TUMWlZRkQ0kgGeiIgaptH7dWacJMuIaKV6Fe7gpY7B+8YRcZjqX0Y2EyVZRkQrJk/wREQ0kpZlREQNW2lZRkTUqQZ48rhjRESNiXkHz3STZBkRrVQDPLlmGRFRK0/wRETUyBM8EREN5YVlERE1bHh9JMkyIqKnqhueZBkRUWsQn+AZvP89RMRhGb11qMnSi6QFku6T9ISkXZI+VeInS9omaXf5eVKJS9L1koYk7ZR0Vsex1pbyuyWt7YifLemxss/1ktTrHL30NVlKWinpyVLR9f08V0QcKVU3vMlSYz/wh7Z/HTgHuFzSEmA9cK/txcC95TPABcDislwK3AhV4gOuBj4ILAOu7kh+N5ayo/utLPHxzjGuviVLSbOAG6i+4BJgTflFRMQ0N1Lew1O39GL7OduPlPWXgCeAecAqYGMpthG4sKyvAm51ZTswR9LbgRXANtt7be8DtgEry7YTbT9g28CtY47V7Rzj6uc1y2XAkO2nASRtoqrg9/p4zojos2o0vPGz4XMl7ej4fJPtm8YWkrQQeD/wIPA2289V5/Jzkk4txeYBz3TsNlxiveLDXeL0OMe4+pksu32BD44tJOlSqmYy75yX8aaIqa7lTenP217aq4CkE4CvAZ+2/XflsmLXol2r0z5+SPp5zbJRRW3fZHup7aVvPWXwZjKJmI4mohsOIOloqkT5VdtfL+Gfli405eeeEh8GFnTsPh94tiY+v0u81znG1c9kOd4XiIhpbAJHwwXcDDxh+0sdmzYDoyPaa4G7O+KXlFHxc4AXS1d6K7Bc0kllYGc5sLVse0nSOeVcl4w5VrdzjKuf/d6HgcWSFgE/AVYD/7KP54uII2SCbkr/EPA7wGOSHi2xPwGuBe6QtA74MXBx2bYF+AgwBLwMfBzA9l5Jn6PKOQCftb23rF8G3AIcB9xTFnqcY1x9S5a290u6girrzwI22N7Vr/NFxJFhi/0TkCxtf5vul+sAzu9S3sDl4xxrA7ChS3wHcEaX+AvdztFLX0dUbG+h+r9BRMwgmXUoIqJGJv+NiGgoyTIiokYm/42IaKjJPZQzTZJlRLRiw/5M/hsRUS/d8IiIGrlmGRHRkJMsIyLqZYAnIqKGnWuWERENiAMZDY+IqJdrlhERNfJseEREE66uWw6aJMuIaC2j4RERNZwBnoiIZtINj4hoIKPhERE17CTLiIhGcutQREQDuWYZEVHDiJGMhkdE1BvAhmWSZUS0lAGeiIiGBrBpmWQZEa0NYsty8K7SRsRhMTAyokZLHUkbJO2R9HhH7ExJ2yU9KmmHpGUlLknXSxqStFPSWR37rJW0uyxrO+JnS3qs7HO9JJX4yZK2lfLbJJ1UV9cky4hox4DVbKl3C7ByTOzPgD+1fSbw78tngAuAxWW5FLgRqsQHXA18EFgGXN2R/G4sZUf3Gz3XeuBe24uBe8vnnpIsI6I1u9lSfxzfD+wdGwZOLOtvAZ4t66uAW13ZDsyR9HZgBbDN9l7b+4BtwMqy7UTbD9g2cCtwYcexNpb1jR3xceWaZUS013yAZ66kHR2fb7J9U80+nwa2Svpzqgbdb5T4POCZjnLDJdYrPtwlDvA2288B2H5O0ql1XyTJMiJaUpsBnudtL215gsuAf2P7a5L+BXAz8FvQdRJNH0L8kKQbHhHtueFyaNYCXy/rf0V1HRKqluGCjnLzqbroveLzu8QBflq66ZSfe+oqlWQZEe0YPKJGyyF6FvgnZf08YHdZ3wxcUkbFzwFeLF3prcBySSeVgZ3lwNay7SVJ55RR8EuAuzuONTpqvrYjPq50wyPiEEzMfZaSbgPOpbq2OUw1qv17wF9Img28QjWaDbAF+AgwBLwMfBzA9l5JnwMeLuU+a3t00OgyqhH344B7ygJwLXCHpHXAj4GL6+qaZBkR7U3QEzy214yz6ewuZQ1cPs5xNgAbusR3AGd0ib8AnN+mrkmWEdFeHneMiKgxelP6gEmyjIjWMvlvD5KOtf1qPysTEdPEoY90T1u1tw5JWibpMcrwvaT3SfpPfa9ZRExZcrNlJmlyn+X1wEeBFwBsfxf4cD8rFRFTWNMb0mdYsmzSDT/K9o/KzEajDvSpPhEx5TWeUWhGaZIsnynzyVnSLOD3gaf6W62ImNJmWKuxiSbJ8jKqrvg7gZ8C3yyxiBhUI5NdgSOvNlna3gOsPgJ1iYjpIPdZdifpL+nS6LZ9aZfiETEAZtpIdxNNuuHf7Fh/E/DbHDzRZkQMmiTLN7J9e+dnSV+hmrY9ImJgHMrjjouAX53oigA8tfN4VrzjzH4cOvpk67OPTnYVooVlK16ekOOkG96FpH38stF9FNXLhWrfhBYRM5QZyMcdeybLMrvw+4CflNBImVMuIgbZAGaBno87lsR4l+0DZRnAX1FEjJVnw7t7SNJZfa9JREwfeTb8lyTNtr0f+EfA70n6AfALqpdv2HYSaMSgmmGJsIle1ywfAs4CLjxCdYmIaWAmdrGb6JUsBWD7B0eoLhExXWQ0/CBvlfQH4220/aU+1CcipoG0LA82CziBiXpBcETMHEmWB3nO9mePWE0iYnrINcs3SIsyIrpLsjzI+UesFhExrWgAJ/8d96Z023uPZEUiIqayJk/wREQcbIKe4JG0QdIeSY+Pif++pCcl7ZL0Zx3xqyQNlW0rOuIrS2xI0vqO+CJJD0raLel2SceU+LHl81DZvrCurkmWEdFOw+fCGw4C3QKs7AxI+jCwCniv7dOBPy/xJVSvuDm97PNlSbPKixRvAC4AlgBrSlmALwDX2V4M7APWlfg6YJ/t04DrSrmekiwjor0Jalnavp9q2sdOlwHX2n61lNlT4quATbZftf1DYAhYVpYh20/bfg3YBKwqs6adB9xZ9t/IL59IXFU+U7afrzHv+x4ryTIi2mueLOdK2tGxNHl317uB3yzd47+W9IESn8fBr7QZLrHx4qcAPy9zXHTGDzpW2f5iKT+uQ5kpPSIGmGg1Gv687aUtTzEbOAk4B/gAcIekf0j32xlN90afe5SnZtu4lYqIaK7/N6UPA18v8+c+JGkEmFviCzrKzQeeLevd4s8DczpmUOssP3qsYUmzgbfwxssBB0k3PCLa6+98lv+d6lojkt4NHEOV+DYDq8tI9iJgMdXsaA8Di8vI9zFUg0CbS7K9D7ioHHctcHdZ31w+U7Z/q25y87QsI6K9CWpZSroNOJfq2uYwcDWwAdhQbid6DVhbEtkuSXcA3wP2A5fbPlCOcwWwlWpOiw22d5VTXAlskvR54DvAzSV+M/AVSUNULcrVdXVNsoyI1iaqG257zTib/tU45a8BrukS3wJs6RJ/mmq0fGz8FeDiNnVNsoyI9vJseEREDQ/ms+FJlhHRXlqWERH1Mp9lREQTSZYRETVm4DvBm0iyjIhWRLrhERGNJFlGRDSRZBkR0UCSZUREjbwKNyKioSTLiIh6edwxIqKBdMMjIurkpvSIiIaSLCMiessTPBERDWlk8LJlkmVEtJNrlhERzaQbHhHRRJJlRES9tCwjIppIsoyIqJG3O0ZE1Mt9lhERTXnwsmWSZUS0Nogty6MmuwIRMc24xVJD0gZJeyQ93mXbH0mypLnlsyRdL2lI0k5JZ3WUXStpd1nWdsTPlvRY2ed6SSrxkyVtK+W3STqprq59S5a9fgkRMb1ppNnSwC3AyjccX1oA/FPgxx3hC4DFZbkUuLGUPRm4GvggsAy4uiP53VjKju43eq71wL22FwP3ls899bNleQtdfgkRMf1NVLK0fT+wt8um64B/y8Ht01XAra5sB+ZIejuwAthme6/tfcA2YGXZdqLtB2wbuBW4sONYG8v6xo74uPp2zdL2/ZIW9uv4ETFJTJsBnrmSdnR8vsn2Tb12kPQx4Ce2v1t6zaPmAc90fB4usV7x4S5xgLfZfg7A9nOSTq37IpM+wCPpUqpmMm/i+EmuTUQ00WKA53nbSxsfVzoe+AywvNvmLjEfQvyQTPoAj+2bbC+1vfRojp3s6kREExM0wNPFu4BFwHcl/R9gPvCIpF+hahku6Cg7H3i2Jj6/Sxzgp6WbTvm5p65ik54sI2J6Gb0pvcnSlu3HbJ9qe6HthVQJ7yzb/xfYDFxSRsXPAV4sXemtwHJJJ5WBneXA1rLtJUnnlFHwS4C7y6k2A6Oj5ms74uOa9G54REwz9oRN/ivpNuBcqmubw8DVtm8ep/gW4CPAEPAy8PGqOt4r6XPAw6XcZ22PDhpdRjXYfBxwT1kArgXukLSOasT94rq69i1ZtvwlRMR0MkE3pdteU7N9Yce6gcvHKbcB2NAlvgM4o0v8BeD8NnXt52h4z19CRExfg/gET7rhEdGOgbyDJyKigcHLlUmWEdFeuuEREQ3kVbgREXXyKtyIiHrVTemDly2TLCOivbyDJyKiXlqWERF1cs0yIqKJiXs2fDpJsoyI9tINj4io4cbv15lRkiwjor20LCMiGhi8XJlkGRHtaWTw+uFJlhHRjslN6RERdYRzU3pERCNJlhERDSRZRkTUyDXLiIhmMhoeEVHL6YZHRNQySZYREY0MXi88yTIi2st9lhERTQxgsjxqsisQEdOMDQdGmi01JG2QtEfS4x2xL0r6vqSdku6SNKdj21WShiQ9KWlFR3xliQ1JWt8RXyTpQUm7Jd0u6ZgSP7Z8HirbF9bVNckyItqzmy31bgFWjoltA86w/V7gKeAqAElLgNXA6WWfL0uaJWkWcANwAbAEWFPKAnwBuM72YmAfsK7E1wH7bJ8GXFfK9ZRkGRHtTVCytH0/sHdM7Bu295eP24H5ZX0VsMn2q7Z/CAwBy8oyZPtp268Bm4BVkgScB9xZ9t8IXNhxrI1l/U7g/FJ+XEmWEdGOgRE3W2CupB0dy6Utz/YJ4J6yPg94pmPbcImNFz8F+HlH4h2NH3Sssv3FUn5cGeCJiJYMbnzv0PO2lx7KWSR9BtgPfHU01L0yXRt97lG+17HGlWQZEe2YRoM3h0PSWuCjwPn2/+/PDwMLOorNB54t693izwNzJM0urcfO8qPHGpY0G3gLYy4HjJVueES0N3EDPG8gaSVwJfAx2y93bNoMrC4j2YuAxcBDwMPA4jLyfQzVINDmkmTvAy4q+68F7u441tqyfhHwrY6k3FValhHR3gTdZynpNuBcqmubw8DVVKPfxwLbypjLdtuftL1L0h3A96i655fbPlCOcwWwFZgFbLC9q5ziSmCTpM8D3wFuLvGbga9IGqJqUa6uq2uSZUS0NHETadhe0yV8c5fYaPlrgGu6xLcAW7rEn6YaLR8bfwW4uE1dkywjoh0DmaItIqKBAXzcMckyIlpy30fDp6Iky4hox+Dm91nOGEmWEdHeSLrhERH1cs0yIqKGndHwiIhG0rKMiKhjfODAZFfiiEuyjIh2RqdoGzBJlhHRXm4diojozYDTsoyIqOFWk//OGEmWEdHaIA7wqGa+yyNK0s+AH012PfpgLtWszTF9zNS/2a/afuvhHEDS/6L6/TTxvO2xb2+clqZUspypJO041PeQxOTI3yzGymslIiIaSLKMiGggyfLIuGmyKxCt5W8WB0myPAJsT+o/PEkHJD0q6XFJfyXp+MM41rmS/kdZ/5ik9T3KzpH0rw/hHP9B0h8dah0nwmT/zWLqSbIcDH9v+0zbZwCvAZ/s3KhK6/8WbG+2fW2PInOA1skyYipKshw8fwOcJmmhpCckfRl4BFggabmkByQ9UlqgJ0D1HmdJ35f0beCfjx5I0u9K+s9l/W2S7pL03bL8BnAt8K7Sqv1iKffHkh6WtFPSn3Yc6zOSnpT0TeA9R+y3EdFQkuUAkTQbuAB4rITeA9xq+/3AL4B/B/yW7bOAHcAfSHoT8JfAPwN+E/iVcQ5/PfDXtt8HnAXsAtYDPyit2j+WtBxYTPVq0jOBsyX9Y0lnU723+f1UyfgDE/zVIw5bnuAZDMdJerSs/w3Ve5nfAfzI9vYSPwdYAvzv8mL7Y4AHgF8Dfmh7N4Ck/wpc2uUc5wGXAJQX378o6aQxZZaX5Tvl8wlUyfPNwF22Xy7n2HxY3zaiD5IsB8Pf2z6zM1AS4i86Q8C2sS+9l3Qm1dwJE0HAf7T9X8ac49MTeI6Ivkg3PEZtBz4k6TQAScdLejfwfWCRpHeVcmvG2f9e4LKy7yxJJwIvUbUaR20FPtFxLXSepFOB+4HflnScpDdTdfkjppQkywDA9s+A3wVuk7STKnn+mu1XqLrd/7MM8Iz37P6ngA9Legz4W+B02y9Qdesfl/RF298A/hvwQCl3J/Bm248AtwOPAl+julQQMaXk2fCIiAbSsoyIaCDJMiKigSTLiIgGkiwjIhpIsoyIaCDJMiKigSTLiIgG/h+XuKug8j+LMgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test, bag_log_reg_svm.predict(X_test))\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "cax = ax.matshow(cm)\n",
    "fig.colorbar(cax)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision bagging svm:  0.6778406528068233\n",
      "Recall bagging svm:  0.6835304333443599\n",
      "F1 score bagging svm:  0.6806736530028206\n"
     ]
    }
   ],
   "source": [
    "print('Precision bagging svm: ', precision_score(y_test, bag_reg_svm_pred))\n",
    "print('Recall bagging svm: ', recall_score(y_test, bag_reg_svm_pred))\n",
    "print('F1 score bagging svm: ', f1_score(y_test, bag_reg_svm_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}