A1 Ensemble Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img align=\"center\" src=\"http://sydney.edu.au/images/content/about/logo-mono.jpg\">\n",
    "<h1 align=\"center\" style=\"margin-top:10px\">Statistical Learning with Python</h1>\n",
    "<h2 align=\"center\" style=\"margin-top:20px\">Appendix 1: Ensemble Learning</h2>\n",
    "<br>\n",
    "\n",
    "\n",
    "<a href=\"#1.-Twitter-Airline-Sentiment Data\">Twitter Airline Sentiment Data</a> <br>\n",
    "<a href=\"#2.-Classification-Models\">Classification Models</a> <br>\n",
    "<a href=\"#3.-Voting-Classifiers\">Voting Classifiers</a> <br>\n",
    "<a href=\"#4.-Model-Stacking\">Model Stacking</a> <br>\n",
    "<a href=\"#5.-Model-Evaluation\">Model Evaluation</a> <br>\n",
    "\n",
    "This notebook relies on the following imports and settings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot settings\n",
    "sns.set_context('notebook') # optimises figures for notebook display\n",
    "sns.set_style('ticks') # set default plot style\n",
    "crayon = ['#4E79A7','#F28E2C','#E15759','#76B7B2','#59A14F', \n",
    "          '#EDC949','#AF7AA1','#FF9DA7','#9C755F','#BAB0AB']\n",
    "sns.set_palette(crayon) # set custom color scheme\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (9, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
    "from sklearn.metrics import accuracy_score, recall_score, precision_score, roc_auc_score, confusion_matrix, log_loss\n",
    "\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.linear_model import LogisticRegression, LogisticRegressionCV\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import  RandomForestClassifier\n",
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Twitter Airline Sentiment Data\n",
    " \n",
    "In this lesson we revisit the Twitter airline sentiment dataset. To save time, we directly load the processed dataset that we constructed in the earlier lesson. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['X_test', 'X_train', 'y_test', 'y_train']>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import h5py\n",
    "f = h5py.File('data/tweets-matrices.h5', 'r')\n",
    "f.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "y_train = np.ravel(f.get('y_train'))\n",
    "y_test = np.ravel(f.get('y_test'))\n",
    "\n",
    "X_train = csr_matrix(f.get('X_train'))\n",
    "X_test = csr_matrix(f.get('X_test'))\n",
    "\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = pd.read_csv('data/tweets_features.csv').iloc[:,0].values.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  2. Classification Models\n",
    "\n",
    "**2.1 Naive Bayes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nbc= BernoulliNB()\n",
    "nbc.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**2.2 Regularised Logistic Regression**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=50, class_weight=None, cv=5, dual=False,\n",
       "           fit_intercept=True, intercept_scaling=1.0, max_iter=100,\n",
       "           multi_class='warn', n_jobs=None, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logit_l1= LogisticRegressionCV(Cs = 50, penalty='l1', solver='liblinear', scoring='neg_log_loss', cv=5)\n",
    "logit_l1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.2067926406393288, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='warn', n_jobs=None, penalty='l1', random_state=None,\n",
       "          solver='liblinear', tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C = logit_l1.C_\n",
    "logit_l1= LogisticRegression(C = C[0], penalty='l1', solver='liblinear')\n",
    "logit_l1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "170"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.abs(logit_l1.coef_) == 0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=50, class_weight=None, cv=5, dual=False,\n",
       "           fit_intercept=True, intercept_scaling=1.0, max_iter=100,\n",
       "           multi_class='warn', n_jobs=None, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='lbfgs', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logit_l2= LogisticRegressionCV(Cs = 50, penalty='l2', scoring='neg_log_loss', cv=5)\n",
    "logit_l2.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.2067926406393288, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='warn', n_jobs=None, penalty='l2', random_state=None,\n",
       "          solver='warn', tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C = logit_l2.C_\n",
    "logit_l2= LogisticRegression(C = C[0], penalty='l2')\n",
    "logit_l2.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3.2 Random Forest**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters found by randomised search: {'min_samples_leaf': 5, 'max_features': 75} \n",
      "\n",
      "Wall time: 2min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "model = RandomForestClassifier(criterion = 'entropy',  n_estimators=100)\n",
    "\n",
    "tuning_parameters = {\n",
    "    'min_samples_leaf': [1, 5, 10, 20, 50],\n",
    "    'max_features': np.arange(25, X_train.shape[1], 25),\n",
    "}\n",
    "\n",
    "rf_search = RandomizedSearchCV(model, tuning_parameters, cv = 5, n_iter= 64, scoring='neg_log_loss', n_jobs=4)\n",
    "rf_search.fit(X_train, y_train)\n",
    "\n",
    "rf = rf_search.best_estimator_\n",
    "\n",
    "print('Best parameters found by randomised search:', rf_search.best_params_, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
       "            max_depth=None, max_features=75, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=5, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=1000, n_jobs=None,\n",
       "            oob_score=False, random_state=1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.n_estimators = 1000\n",
    "rf.random_state = 1\n",
    "rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3.3 XGBoost**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from skopt.space import Real, Categorical, Integer\n",
    "from skopt import BayesSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters found by randomised search: {'colsample_bynode': 0.01, 'learning_rate': 0.10566347884368385, 'max_depth': 2, 'n_estimators': 2000, 'subsample': 0.8017562932483941} \n",
      "\n",
      "Wall time: 3min 20s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "model = XGBClassifier(reg_lambda=0)\n",
    "\n",
    "space = {\n",
    "    'learning_rate': Real(0.005, 0.15),\n",
    "    'n_estimators' : Integer(100, 2000),\n",
    "    'max_depth' : Integer(1, 4),\n",
    "    'subsample' : Real(0.5, 1.0),\n",
    "    'colsample_bynode' : Real(0.01, 1.0),\n",
    "}\n",
    "\n",
    "np.random.seed(87)\n",
    "xgb_search =  BayesSearchCV(model, space, cv = 5,  n_iter=32, scoring = 'neg_log_loss', n_jobs=4)\n",
    "xgb_search.fit(X_train, y_train)\n",
    "\n",
    "xgb = xgb_search.best_estimator_\n",
    "\n",
    "print('Best parameters found by Bayesian estimation:', xgb_search.best_params_, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlUAAAF3CAYAAACMir4SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd5hdVdn+8W+IAaSDEJBmkHIjNZTQBAlNKYogRbqooPiCCD+wIAoBRUBBXrBFjBCkGhslVAkEpCtICZDbAokvRREQFARCIL8/1h5yMpmZDMk5c2bOuT/XNdecs/fae6+9GfFhrbWfZ9CMGTOIiIiIiHkzX7M7EBEREdEKElRFRERE1EGCqoiIiIg6SFAVERERUQcJqiIiIiLqIEFVRERERB0kqIqIGGAkLSxpaLP7ERGzSlAVEQ0laYKkc7rYPkjS45I+MRfntKTte9HuNkmHdbPvm5Ium4trj5F00ts9rs5uBzZsch8iopN3NLsDEdHyzgW+L+kY26/XbN8OWBwY93ZPaFv16txcXPuQZl27xtLN7kBEzC5BVUQ02m+A7wG7AJfXbP8UcIHtVyQtBHwX2AF4N/AEcIztq6oRqbOrbSOAXYHLgENsXyfpg8AoYA1gfuBa4GDbr1TX2UDS/cCKwHXAEbZf6NxJSUcARwFLAhOB/7H9jy7aXQQ8Yfsrkm4DxgMHAKtU1z4b+BHwHuBK4EDbb0p6ghJgHgosWt3DF2y/Jmkx4NvAbsAM4CrgWNv/lvRNYL3q/hYG/gIsD/xG0jHAj4FvAHsAKwDPAyfbHiNpNeCu6tl+obqFC20fW93LylVftwL+DXzH9tnVvr2q57o88Afgc7b/0vl5RMRMmf6LiIayPQ24AHhrmk/SEpQA4sfVpi8DqwEbUAKOC4HaKcO1gIuBlShBQsd5FgV+CZxie2lgbWALYO+aY3cB9gPeC7yLEvTMQtK+wLHARyiByf8Bl/TyFj8BfBBYFdge+AElOFwH2Kn63OHjwPuBNYFNgK9X28dU/Vu7+lkJ+GHNcdsCuwPr2N4GeArY3fYPgYMogeZWlGf3deAcSe+sjn0XJTBaGfgYcKSkEdW+XwN/A5atrnG8pG0lbU4JAA8BhgLXA+Ml5T/EI3qQoCoi+sK5wE6S3lV93x+42/bk6vs5lIDjZcr/+f+HEtx0eAO41PbLtqfXbP8vMNz21ZIWp4xy/bPTsWfbfsT2vykBx96SOv+779PAmbYftf0q8BVgK0nv7cW9XWT7Kdt/Bx4FLrH9tO2pwGTKiFWHk23/rWr7LWBfSQtTgp0v2n7O9vOUAG8fSQtUx91b9e3FLq7/K0ow13HfrwDvpIy4dTjd9mu2bwf+DKwuaXXKuqwv2n7F9p+AbYAHq+dxnu07bb9u+wxgIeADvXgeEW0r/9UREQ1n+8+S7gD2Bb5Pmfr7dk2TJSgjMyOAvwKPM+t/9D3XaT1Wx3nfkLSbpKOAN4EHKFNktcdOrfn8BLBgdb1aKwOnSvpGzbY3KAHRY3O4vec7HVM7tfhmp77UTp89ASwHLAUM7tTPqdW2d1ff/97D9eenTK9uRxl1eqDaXnvdf9Z8fr3atyzwou2XOnbYfhjemhbcUtKnO11n5R76EdH2MlIVEX3lXOAgSetR1jf9pmbfT4CHgKG2RwCjOx07o6sTSvoAcDywje1htj8KdF4HtVzN5/cAL1WjQbWepqyhWqLjhzKKc3sv7qvLvnVj+U59+b/q2q8Dw2r2rUIJ0J7txTW+TQne3m17ONDbNxOfBBaXtEjHBkn7V2vUngZO6/Q8hgM/7+W5I9pSgqqI6Cu/pgQOX6VMLU2r2bcYZdrqjWqU5CRgsKRBczjnYpTg4xVJ75B0MGXN0pCaNl+QtJqkJYFTgPO6OM8FwBclrSppvmrk607KNFo9HSdpGUnLA8dRFupPp6zf+rakd0laihIoXVk7itTJa5R7h1mf3dLAd6rtQ7o6sIPtxyn3+C1JC0gScCYlwLsA+Kyk4VXqiz0pQe8K3Z8xIhJURUSfqFmwvjdl1KrWFyjriv4N3Ex5a+4V4H1zOO01lBGvRyijKx+vrlF73HjKQus/U6bfjuviPOdXPzdQpu/2BXbuZg3TvHgIuJuybuk6ZgZARwJTgIerPj4NfLKH81wAnC/pK5R1YmsB/wL+SFnXNZU5Pzsoz2tlysL3G4ATbN9s+ybKywOXUP6ZnAjsmbf/Ino2aMaMtzNyHRERc6NKqXCI7eua3ZeIaIyMVEVERETUQYKqiIiIiDrI9F9EREREHSRPVQNV2YdXpJS0mD6n9hERETFwJahqrBWBxydMmNDsfkRERER9dJvqJWuqIiIiIuogQVVEREREHbTFQvWq0OpY4Ghgou1hdTjnRGCU7Yk9tBkGPP7ORd/FfIMHz+slIyIiYg6WX24o46/4VSMv0e30X7usqVoS2KBZF1/zg4ew4KJLNevyERERbePBy89s2rXbJag6h1LI9CzgnZIuA9ahlHXYzfZzko4ADqRUuJ8G7GvbkqYAFwIfqvYdZPvejhNLGgrcBBxv+4q+u6WIiIjoT9plTdWRlNpWRwPLAN+1vQ6lmv0+khYDdgNGVtvHA0fUHP+c7U2A0ZRisB0WB66mTAMmoIqIiGhj7RJU1XrK9j3V54eBpW3/G9iPEmCdCnwEWKTmmI5aXZOA2nm8H1Mqwf+6sV2OiIiI/q4dg6raJJwzgEGSVgLuBJYArqUsaq9diPZqbfua7acD/wQ+16jORkRExMDQLkHVdHpePzYC+Ivts4DfA7sDvXld74/A/wAnSlphnnsZERERA1a7LFT/B/A34Pxu9t8AfE7SI5SRqFsoC9nnyPafJf0A+D4lGJvN5BvGJKVCREREH1h+uaFNu3Zb5Klqlo48VRMmTGDFFVdsdnciIiJi3qVMTUREREQjJaiKiIiIqINM/9WJpEWA/wDr2p5UbRtGytRE1FUflKCIiOhJ25ep6QszOv1+S8rURNRPM0tQRET0JNN/9fNf4E3gpWZ3JCIiIvpegqo6sT0DeIFSTzAiIiLaTIKq+nqGsq4qIiIi2kyCqvp6rBqxioiIiDaToKqObO/S7D5EREREc+TtvzqSdD+ws+2narenTE1E/TSzBEVERE+Sp6qBUqYmIiKi5aRMTUREREQjZaSqgZJRPZohGccjIhoqGdUbSdLNtrfpbn8yqkdfSsbxiIjmyPRffYxsdgciIiKiudpupErSqcCewLPA08CVwFeq768AOwLfoQRKg4Gxts+S9A7gR8A6wLLAg8C+wOnVee+2vWmf3kxERET0G201UiXpI8CWwNrAzsAGHbuAA2zvABwKYHtDYBPgo5K2ArYAptneHFgNWIKSPuHIqn0CqoiIiDbWbiNVOwDjbE8Dpkm6vNr+jO0p1eftgeGStq2+LwKsa/uHkp6TdDiwJrB6tS8iIiKi7YKqN+h6dO6Vms+DgS/Z/jWApKWBlyTtCpwMnA2cDyxND28ARERERHtpq+k/4EZgD0nzS1oM+DDwnk5tbgIOlTRE0iLAbcBmlBGscbbPB14AtqEEYABvVGuuIiIiok21VSBg+2pJmwN/BJ4HnmLWUSqA0ZSpvT9Sns/5tidKeg64RNK+wDTgdmCV6pgrgAckbWT71c7XTZma6Esp4xIR0RxtlfyzCqjWsH2BpCHAncCnbD/YoOsNI2VqIiIiWknK1FQM7CvpAeA+4LJGBVQRERHRXtpqpKqvpUzNwJHSLhER0UspU9NMKVPT/6W0S0REzKsBF1R1k9l8YeC7tq+V9C1gA9s7SXo38Fvb60g6CDiKMuV5L3A4JcXCedW5AH5o+yeSlgV+CqwMTAe+avs6SaOqbWsAywCnANsBmwIPAPvYztBfREREGxqIa6q6ymw+kRLcAGwFvE/SYErJmaslrU3JlL6F7eHAM8Cx1bmWsr0BsEt1LMD3gJtsr0cpaXNeFWgBrEspYfMZSr6q0ylB2YbAeg2654iIiOjnBlxQZftW4IdVZvOzKekPngC2k7Ro1ewBSpCzEzCeklNqdeAuSfcDH6VkRZ8ESNL1wF7AF6vjt6WMVGH7MeBuymgUlJGv6cBU4Gnbj1TfnwSWbNiNR0RERL824IKqKrP5xcB/KSNFt1IWjc0H7EHJHzWRMnK1EXAHJUnnONvDq5GqTYAjbD9HqQP4PUr9v/skLcHsz2UQM6dKp9Vsn17v+4uIiIiBacAFVXSf2fxa4GuUgOom4PPAXbbfqLbtLmmopEGUNVlHVQHahcDVwJHAS8BK1fGfBpD0XuD9lJxWEREREV0aiEHVTyi5ph4CfsHMzOZXU0rO3EZZvD4/ZeoP2w8AJ1GCpYcpQdhplEDslWrbPcBFth+iBFjbVte4HDjE9tN9dYMREREx8CRPVQMlT9XAkTxVERHRS8lT1UzjLx+XMjUREREtbiBO/0VERET0O209/SdpJDDK9sgGnX8Ymf7rtzLlFxERcyHTf82UMjX9U0rTREREPSWogmUkXQOsCpiSBHQ/4BhgBqWkzRG2X5I0w/YgAEkHAyNtHyxpCiVB6HBgK9vP9PldRERERFNlTVWp5Xc48D5gOeAw4Hhga9vrAi8DJ/biPNfaVgKqiIiI9pSgCh6w/bjtN4FHKbUEr6qyrQOcy8y6gj25u1EdjIiIiP4vQdWspWZmAP/qtL+2RA1VRnaAIZ3avVL/rkVERMRAkaCqa7tK6lhZfihwc/X5WWDtKrDatSk9i4iIiH4pC9Vn92/gVOAWSUMoC9UPq/Z9hVL65u+UcjhLN6WHERER0e+0dZ6qRkueqv4teaoiImIuJE9VM6VMTUREROvLmqqIiIiIOmjb6T9J5wEHA6/bXkDSycAfbF/ZwzFvJf/stP0k4Ebbv+u0fRiZ/muaTO9FREQDZPqvCwcDawI3ANg+YR7OtTUz3xCcTcrUNEfK0ERERF9qy6BK0pWUSPMeYP5q21hgou2xko4EPg+8AEwG/mp7VNVuNLB5dao9gC2AjYExkna3/VAf3kpERET0E225psp2R46p4cAsZWUkrUcpW7MRsBWweqfDb7S9PvBb4LO2fwb8ATgkAVVERET7asugag62B8bb/rftV4FLO+2/vPr9MMlTFREREZUEVbN7gx6ei+2OsjYz6GGxWkRERLSXBFWzmwDsLGkxSfNT1k3N6RXJ6bTp+rSIiIgoEgh0YnuSpHOAO4GXKPX+5lQs+TpgtKSDbN/ReefkG8YkpUITLL/c0GZ3ISIi2kjb5qnqjqQ1gF1sn1V9vwIYY/uquTjXMODxCRMmJKN6REREa0ieqrdhKjBC0iTKtN/1lCLKEREREd1KUNWJ7deA/Zrdj4iIiBhYMv0HSBoJjLI9ss7nHUbK1DREStBERESTZPqvmVKmpv5SgiYiIvqbBFU1qkXq5wJLAS8DRwKPURJ9rmT7dUnrABfbXl/SQcBRlNQU9wKHVwlDIyIios0kT9WsLgLOsb0ecDTwS0pahbuBD1Vt9gUukrQ2cCiwhe2OcjfH9n2XIyIioj9IUDXTIsBqtn8NYPsu4HlAlGBrn6rd3sAlwDaUuoB3Sbof+CiwZl93OiIiIvqHBFUzdfUsBlGmSK8Etpb0AeBvtp8EBgPjbA+vRqo2AY7os95GREREv5KgaqZ/A49J+hiApM2A5YBJVZqF64D/pYxaAUwEdpc0VNIg4EeU9VURERHRhrJQfVYHUMrNnAS8BnzM9rRq34XV/l8B2H6gancTJTi9Hzitq5OmTE39pQRNRET0N8lT1UApUxMREdFyus1Tlem/iIiIiDrISFUDJaP625Ms6RERMQAko3ozJaN67yRLekREDGSZ/ouIiIiog7YZqZL0Dkrag3WAZYEHgeOAS4HJwNrAfcAdwMHAksDuth+VtBdwDPBOYAHgU8AfgHtqLrEKcKHt5KqKiIhoQ+00UrUFMM325sBqwBLAzsB6wOnA+sD7gWFVm0uBz0iaDzgM+LDt9YFvA8fZnlaT+POzwN+BUX18TxEREdFPtM1Ile1bJT0n6XBKOZnVKaVp/m77jwCSngAmVIdMBVax/aak3YGPSBIwEnij47ySVqCUrdnT9rN9dkMRERHRr7TNSJWkXYGLgf8C5wO3UgKnaZ2aTu903CKUab5VqmPOoVr5L2lB4HLgxI7ALCIiItpT2wRVwPaUWn3nAy9QCiL3Js/BGsAM4FvAzcDHao77KXCr7Yu6OTYiIiLaRNtM/wE/AS6RtC9ldOp2SmA1Jw9QStBMBt4Erge2lLQFsB/wB0l/pIxePWx7/84nSJma3knpmYiIGMiS/LOBUqYmIiKi5aRMTUREREQjzfVIVbVI+whgL2AIZZ3Rc8BZtq+qVwclbQLsYfvLkg4GRto+uF7n7+aahwIv2b60hzaHAdge3UObYaRMTbdSliYiIgag+papkbQYcCVwLfBB2y9W21cFzpC0lu3T5+bcXViLkqyzL70fmNhTg56Cqc5SpqZrKUsTERGtZG4Xqp8OfI+SYuAiSSsC11HejPsQMFbStcCpwHLVMSfZvlLSGsC5wFLAy8CRtn8vaSww0fZYAEkzKFnNTwYWkXQ88CSwmqSJwMrABNuHSnoI2LvKfn4J8KLtz0naHPia7V0kfQXYmzKidj3wZWBRSpLPt/pISbmwK7CtpKera36PktNqKHCq7dGSRgHYHiXpn5QM6+8GRth+fS6fa0RERAxQb3tNlaSlgFVt/woYB1xmewPgJWBJ21Mob9rtDkyxvRHwaWCr6hQXAefYXg84GvilpAW6upbtF4ATgCttn1JtXpkSvL0P2EnS2sDVwHbV/nWBLavPOwLjJe0IbASMADYAVgD276qPtm+kjMKdYPt64BDgm7ZHUN4W/E4XXV0aOL3KsJ6AKiIiog3NzUL1TYEJkjYAhtq+uNp+b/UDJQ/Uq8Buki6nBDPfqBJprmb71wC27wKeB/Q2rn+r7edtvwb8lRLQXANsJ2kt4GHgDUlDgZ0oAdf2Vb/vpdT325hS6++Ozn3s4nrHAAtKOg74JmXEqit3v417iIiIiBYzN0HV4sB/KIHQpJrtmzEzqNoZuItSDuZiyijVPd1cbxBlGnIGMzOVD+nh+rUZzzuOuYNSu297ylqoW4A9gSG2/0aZ8vvfmlp9mwKn2P5z5z5Wtf5qjaOMaD0CHN9dp2y/0kOfIyIiosXNTVD1OGWU50/ACEmLVW/oHQU8JGkfyhTdupR1VL8A/oeyHmk+4DFJHwOQtBllPdMk4NnqvAC71VxvOnNY+2V7OiVoO5ISVN1ECYCuqZrcBBwoaRFJ76CUltlT0hFd9HGxTtfcgTIVeAVl5AtJeZUvIiIiZjE3QdU9wCbAv4BfAo8BX6AEKqMpgccngasAVYvIfwd8sVojdQBwZLX9+8DHbE+rjh0p6UHK23dP11xvM0mnzaFfVwML255MGalaFhgPUKV4+BVlim4SJUP6BcDPuunjjcBXJe0JjAJuk/QIZTRrCqUOYERERMRb5ipPlaSNgDGUenhX2X612j4Y+ABlbdJJtn9bx74OOMlT1bPkqYqIiAGo2zxV85L8c3ngi8BIypqlGZTaeHcD37b92FyduIWkTE1ERETLqW/yTwDbT1FSIkRERES0vbYvqCxpJDDK9sh5OMdbpXQ6bR9Gpv+6lKm/iIgYoOo/UhWz6LGUTsrUzC4laiIiotUkqCqWlnQdJdP63cDhlOzpJ1OKRT8OHGr7OUlnUNIsvEl54/Fsakrp1GR+j4iIiDYyNykVWtEqwOeB9Sj1AL8CnAZ8qCrBcz1wuqT3ADvZXp+S9mEtSub4zqV0IiIios1kpKq4tcqujqSLKTmsZgA3S4LyduPzlOLKr0i6nZID68u2X63aRERERBvLSFVRW/qm45ncVlPWZgRlIfp0SombrwPvAu6UtEbfdjUiIiL6owRVxZaSVq7q/h0EnAVsXhMwfR04oyoifQtlZOtYSj1A0YtSOhEREdHaElQVDwPnAQ9RpvhOAT4FjKtK2GwIHGP7j8CdwCRJ91GCqmvpfSmdiIiIaFFtn6eqkZKnqnvJUxUREQNU8lQ10/jLx6VMTURERIvL9F9EREREHWT6bx5JWgT4GbCX7Tc67RtGpv9mk6m/iIgYwDL91yi2X5J0I/BZ4IddtUmZmlmlRE1ERLSiTP/NJUn3S1q++nopcJSkbqPXiIiIaG0JquZSlRj0qerzv4CXKGVuIiIiog0lqKqfqcDqze5ERERENEeCqvqZDrzZ7E5EREREcySoqp9hwF+a3YmIiIhojrz9N5ck3Q/sbPspSUsAi9t+sKu2k28Yk5QKNZZfbmizuxAREVF3Carmku3hNV8/AZzdXdtkVI+IiGh9CarmUZX8c3tg92b3JSIiIponQdU8sv0S8JFm9yMiIiKaK2Vq5oGkKcBI21O62T+MlKmZRUrURETEAJcyNc2UMjUzpURNRES0qrYPqiSNBL4NDAYeA6YB61NyTp1h+2eSFgR+AGwJvA58w/bPa86xBnA1cKDtu/r2DiIiIqI/aPugqrIG8B7geGBB2+tIWhq4p0qd8CFgEeB9wFBggqTfVMeuBIwGPpmAKiIion0l+Wdh2y8C2wI/rTY8C1wBjAS2Bi62/abtv9te2/a06thfAI/Zvq0J/Y6IiIh+IkFV8Ur1u/PzGEQZzXsdeGtFv6TVJM1ffT0SeK+kXRrey4iIiOi3ElTN6ibg0wDV9N9uwETgVuDjkgZJGgrcAixQHXMP8DngB5IW7vMeR0RERL+QNVWzOhn4oaSHKAvXT7F9n6SHgXOAB6p2n7f9H0kA2L5V0s3AN4GjO580ZWpmSomaiIhoVclT1UAdeaomTJiQMjURERGtods8VZn+i4iIiKiDjFQ1UDKqzyrZ1CMiogUko3ojSToMwPborvYno3qRbOoREdHKElTVQXfBVERERLSPAR9USXoH8CNgHWBZ4EHgOODnwGRgbWAqcIDt5yU9A/wa2AL4D7C/7SlVceS7geHAVsCOwFGUdWf3AofbflXSfsDXKHmrfg8cSsnEju1Rjb/jiIiI6I9aYaH6FsA025sDqwFLADsD6wI/tL028Cgwqmq/DHCn7fWAyyipEjpca1tVm0OBLWwPB54BjpW0AnAW8MHqvIOBJP2MiIiIgT9SVeWIek7S4cCawOqUOn1/sj2xanYBcEn1+VXgZzXbT6053d3V722q89xV5aKaH7gP2By43fYT1bUPBJA0vP53FhEREQPJgA+qJO1KSdp5NnA+sDRlum96TbP5ar6/aXtGF9thZrmawcA420dW11iE8qy2ZtZyNcvU9WYiIiJiwGqF6b/tKQHQ+cALlFGmwYBqRpA+CVxbfV5I0ke62F5rIrC7pKGSBlHWbB1FWUO1maTlqnZnAR+t8/1ERETEADTgR6qAnwCXSNoXmAbcTgmsngdOkrQaZfH6ITXH7CXpFOAp4BOdT2j7AUknUWoBzgfcD5xWLVT/AnC9pMHAnZTRsa/31MGUqSlSoiYiIlpZSyb/rJJuTrQ9rIt9M2x3m7irAf1ImZqIiIjWkTI1EREREY3UkiNVfUHSwcBI2wf30GYYbV6mJqVpIiKixaRMTTO1c5malKaJiIh20TZBlaSRlMzn04BVgCuBl4DdKFHnzsDfO9Zb1Y5ESdoeOJMyXToV2K867WqSJgIrAxNsH9pX9xMRERH9S7utqdoUOAzYGDgC+KftjSlvB+7T1QGSFgAuBj5he13gIWa+Mbgy8DHgfcBOktZubPcjIiKiv2qbkarKJNv/ByDpWWBCtX0qsGQ3x6wLPGn7fgDbx1XHHwzcavv56vtfKYlHIyIiog21W1A1rdP36Z0bSBpUZVwfUm16nVmzqC8OLNrF8TPoYfFaREREtLZ2m/6bk2eBtass6rtW2wwMlbRW9f1LlCnEiIiIiLckqJrVV4DxlEzpBrD9KnAA8DNJDwJrAac1rYcRERHRLyVPVQMlT1XyVEVERMtJnqpmGn/5uJSpiYiIaHGZ/ouIiIiog0z/NVC7Tf9lqi8iItpApv96S9JngOU7bZ7f9lfn9pztUqYmJWkiIqKdJaia3R22J9VukLROszoTERERA0PWVEVERETUQYKqiIiIiDpIUBURERFRBwmqIiIiIuogC9Vnt4WkPTttmx+Y67f/IiIiovUlqOrE9rn1PufkG8a0TZ6qiIiIdpWgqg+kTE1ERETry5qqiIiIiDpoqzI1ks4DDgZet72ApJOBP9i+sodjZtieLSW9pJOAG23/TtJEYJTtiZ3aDCNlaiIiIlpJytRUDgbWBG4AsH3CPJxra+Dm3jRMmZqIiIjW1zZBlaQrKdHlPZS3+ZA0Fphoe6ykI4HPAy8Ak4G/2h5VtRsNbF6dag9gC2BjYIyk3fvwNiIiIqKfaps1VbZ3rT4OB56p3SdpPeBwYCNgK2D1ToffaHt94LfAZ23/DPgDcIjthxra8YiIiBgQ2iaomoPtgfG2/237VeDSTvsvr34/DCzdpz2LiIiIASFBVfEGPTwL29OrjzPoYYFaREREtK8EVcUEYGdJi0man7Juak6vRU6njdakRURERM8SFAC2J0k6B7gTeAl4FnhlDoddB4yWdNCczp+M6hEREa2vrfJUdUfSGsAuts+qvl8BjLF91Tyedxjw+IQJE5JRPSIiojUkT9UcTAVGSJpEmfa7Hhjf3C5FRETEQJKgCrD9GrBfs/sRERERA1evp/8kLQgcAewFDAEGA88BZ/U0TdabUjCd2m8MHGb7kF51rEEk7QpsPC9Z11OmJiIiouV0O/3Xq6BK0mLAlcC1wGjbL1bbVwXOAO6yfXp9+to6OoKqDfb4ctuUqbnv7t81uxsRERGNNM9rqk4HvgfcClwkaUXK228fAz4EjJW0KXC77TMBJP0KuAj4KDCx+rmOmW/W7QyMBrYEnqSsZfpGdb1RtkdWhYrvoWQ5Xwb4vO1rJa1T9WcRYChwqu3RkkYBKwNrVO1PAbYDNgUeAPaxPUPSV4C9KaNt1wNfBt7TqX8XAyNtHyxpe+BMSgqKqcycKvwpsCKwPHAjJcN6Vv5HRES0oTnmqZK0FLCq7V8B44DLbG9AST2wpO0pwE+Ae4F9q2MWpdTKu7rz6YADbO8AHAYsTClw/ICdlhEAABmqSURBVElgRDddmN/25sDRwDerbYcA37Q9AtgG+E5N+3WBkcBngPMpAeE6wIbAepJ2pJSjGQFsAKwA7N9F/zrufwFKgPUJ2+sCDwGfAHYB7q/6tjqlwPKG3T3HiIiIaG29GanaFJggaQNgqO2Lq+33Vj9QihBPBxaUtBql4PBVtqdJqj3XM1UQBrAD8JNqZGeqpAndXP+66vckoGMO7RhgR0nHUYKoRWra/9b2dElTgadtPwIg6UlgSUpJmk1r+v5O4G/AbZ3612Fd4Enb9wPYPq5jh6RNJB0FvA94V6d+RERERBvpTVC1OPAfyijOpJrtmzEzMNmZMoo1CPg4Jag6rYtz1SbU7LE0TI1Xq9+1JWLGAf8CrgIuoxohq0yr+Tyd2Q0G/tf2dwEkLVG1W5quE36+Tk12dUmLA4sCuwN7AudSpv7WISVsIiIi2lZvgprHgbWBP1FyOS0maRPgKOAhSfsAK9u+hTJN9nFgNcrIT09uBPaRNEjS8pQpu96uR9oBOMH2FcBOAJJ6+3rdTcCBkhaR9A5KseQ9e2hvYKiktarvX6JMXe4A/LgauVsQGE4J2CIiIqIN9Wak6h7g+5S3/H4JPEZZ3H05ZaH5lcAnJS1r+/8kPQvc2YsF2+cC61PWKD1NWQD+CrBQL/o0CrhN0quUBehTgFV6cRy2r5K0PnA3JQi6DriAslC9q/avSjoA+FlVF/CvwIHAJsCPqinIF4E7uutDytRERES0vt6mVNgIGAN8i7JW6tVq+2DgA5S39k6y/dveXljSLsAg2+OrKbU/UvJCPf/2b6N/SpmaiIiIljNvKRVs31sFQV8EvloFUzOANykjPgfZfuxtduoR4EJJHW/0ndBKAVVERES0lxRUbqB2yqiebOoREdEmUlC5O5JGUiUb7WX7GbZne6CSplCShU7pvG/NDx7S8hnVH7z8zGZ3ISIioql68/ZfRERERMxB249UVZaRdA2wKiWFwl6UUjTHUNaO3QscYfuljgOqTPMXAStR1oct2NedjoiIiP4jI1XFysDhlMzoy1HyUB0PbF2VpnkZOLHTMScD91X7fwAs23fdjYiIiP4mQVXxgO3Hbb8JPAosQUkd8Vy1/1xKYeZaI4GfA9i+lZK/KyIiItpUgqqitpzNDEoJnFqDmH2qtLZsTudzRERERJtJUNW9Xat1UwCHAjd32n8jJbM6kkZQSvNEREREm8pC9a79GzgVuEXSEMpC9cM6tTkRGCvpYWAyPUz/tUOZmpSoiYiIdpfknw2UMjUREREtp9vkn5n+i4iIiKiDth2pkjQWmGh7bDf7u8yc/javMYw2KFOTEjUREdFGUqammVq9TE1K1ERERLRRUCVpEHAm8GHgKWAwMFHSQcBRlKnQe4HDbb9ac9wKwE8puauWB8baPkHS74CTbf+2OvefKMlCn+rL+4qIiIj+oZ3WVO0BbACsTSlDsxqwMCVdwha2hwPPAMd2Om5f4FLbmwHrAkdJWho4jyqlArAV8JcEVBEREe2rnYKqkcCvbb9u+5/ANZR50dWBuyTdD3wUWLP2INtnAH+TdCxwNjA/JRgbB+wgaWHgE8DYPrqPiIiI6IfaKajqKgP6YGCc7eHVSNUmwBG1B0k6EzgSmAp8E3gWGGT7ZUpgtielhM0VDb+DiIiI6LfaKai6Edhb0gKSlgR2rLbvLmlotS7qR5T1VbV2AL5j+xeAgBUowRiUKcBTgGtr12FFRERE+2mboMr2FcBEYBJwJfAI8CJwEnAT8DAlWDqt06GnAhdKmkQZxfoDsEp1ztspI2DnN/4OIiIioj9r2zxV86oa2VoH+JntDbppM4zkqYqIiGglyVPVAEcBX6S8Sdij8ZePS5maiIiIFpegai7ZPgs4q9n9iIiIiP4h038VSSOBUbZH1vGcw2jx6b9M/UVERJvJ9F8ztXKZmpSoiYiIKBJUzWppSddR0ibcDRxOybh+ICXh5zRgX9uuRra+R8l3dSewVj1HuSIiImJgaZuUCr20CvB5YD1gUeAwYDdgpO11gPHAEZKGABcC+1dv/r3epP5GREREP5Ggala32v6z7RnAxZTSNvsB+0g6FfgIsAilBuAzth+sjjuvGZ2NiIiI/iNB1aym13yeD1iCMrW3BHAtpb7fIOAN8uwiIiKiRgKDWW0paWVJ8wEHUQKpv1TpE34P7E7Juv4osKSkdavj9qNkVo+IiIg2laBqVg9TpvIeAp4EfgzMJ+kR4D5gMrCK7WnAAcDPJN0LrAS80pwuR0RERH+Qt/8qticCW3axa4fOG6qRrF2BLW2/LOn/Ud4Y7NLkG8a0dJ6qiIiISFA1V2y/Kel54PeSpgFTgE931z5laiIiIlpfgqq5ZPs04LRm9yMiIiL6h5SpmQeSxgITO35sD+u0fxgtXKYmJWoiIqINpUxNM7VqmZqUqImIiJgpQVUnkgZRpvV2p+St+jFwP3AKsBAlZ9XRtq9oWicjIiKi30lQNbs9gfdTsqYPAW4DngUOsT1Z0rbA2UCCqoiIiHhLgqrZbQ2Ms/0a8BowXNKCwIcl7QVsRilVExEREfGWJP+c3evUZEevFpv/DtgEuJcyDdjtIrWIiIhoTxmpmt2twBckjaZM/91AyZi+FWXk6jRKqZqIiIiItySo6sT2byRtTClLMx9wFrA6pYTN68BNwEKSFu7tOVs1o3qyqUdERMyUPFUN1JGnasKECcmoHhER0Rq6XQKUNVURERERdZCgKiIiIqIOWmL6r0p5cASwF2Vx+WDgOeAs21dJWh4YY3vnHs6xCbCH7S9LOhgYafvgTm2G0UU5mh7OOYwWLVOTEjUREdGmWrdMjaTFgCuBa4EP2n6x2r4qcIaktWyfDnQbUFXWApZtRB9bsUxNStRERETMasAHVcDpwPcoqRAukrQicB3wMeBDwFhJP6caYZK0LKX0zErAm8BxwB+Ak4FFJB0PPAmsJulWYClgfNXuLR3FlG2Prb7PsJ38VREREW1qQK+pkrQUsKrtXwHjgMtsbwC8BCxpewrwE2YdpTobOM/2RsCulADrDeAE4Erbp1TtVgH2ADYEtqzaRkRERHRpoI9UbQpMkLQBMNT2xdX2e6sfgBeA2nwG2wNrSjq5+j4EWLWLc19p+58AksYBI4EH6tv9iIiIaBUDeqQKWBz4DyBgUs32zZgZVO0M3FWzbzCwre3htodTArOHujj39JrP81ESf9aaQbVYTdKQub2BiIiIaA0DPah6HFgb+BMwQtJi1Vt8RwEPSdoHWBmYWnPMTcD/AEhaixKMLUQJompH7naWtET1ZuE+wI2drv1sdW2A3ep6VxERETHgDPTpv3uA7wNnAL8EHgOuBy4HRlPeCvwks07vfR44V9KDlJGmA2z/R9I9wChJpwGTq59rgCWAS2zfUKVI6DAa+Hl1npuAp7vrZCuWqUmJmoiIiFkN+DxVkjYCxgDfAq6y/Wq1fTDwAeAbwC+AQ22v08d9G0bK1ERERLSS1s1TZfteSbsAXwS+WgVTMyjpEu4Gvgz8ijKaFREREdEQA36kqj8biBnVkyk9IiKiR607UtWXJI0ERtke+XaOG0gZ1ZMpPSIiYu4M9Lf/IiIiIvqFjFS9fctIuobyRqEpa7muoKRYeMX2Ds3sXERERDRHgqq3b2Xgw5TcV3dRMrQL2LEqixMRERFtKEHV2/eA7ccBJD0KLA08k4AqIiKivWVN1dtXW75mBmXE6pUm9SUiIiL6iQRVEREREXWQ6b8+MJDK1KT8TERExNxJ8s8GSpmaiIiIltNt8s9M/0VERETUQUaq6qC7TOspUxMREdFyUqammVKmJiIiovW1ZVAl6VRgT0oW9KeBKymR5zGUNAn3AkfYfknSh4FvUqZKHwM+a/sfkj4InAW8Ckzu+7uIiIiI/qTt1lRJ+giwJbA2sDOwAbAocDywte11gZeBEyUNBX4M7GZ7PeB24PuSFgAuAPa0vRHJUxUREdH22i6oAnYAxtmeZvtfwOWU0amrbD9XtTkX2A7YBLinJlt6x/Z1gadsP1ptv6CvOh8RERH9UzsGVW8w+313/j6IMjXa3fYZzLpQbToRERHR1toxqLoR2EPS/JIWoxRHXhLYVVLHavJDgZuBu4HNqrf4AD5TbX8QWFbS+tX2ffuq8xEREdE/tV1QZftq4Fbgj8DVwFPAo8CpwC2SJgNLAF+z/Q9KIPUbSQ8DI4HDbL9OCaQulHQfsFCf30hERET0K22Xp0rS5sAati+QNAS4E/iU7QcbcK1hJE9VREREK0meqhqmvNn3/ygjdRc0IqCqNf7ycSlTExER0eLaLqiy/TywY7P7EREREa2l7ab/GkXSFGBkTfqFATX9l2m/iIiIXsn0XzMNhDI1KU8TERExb1oyqJI0CDgN2J2SQ+rHwP3AKZQ39ZYAjrZ9haSxwIvARsAKwMm2z6/SK/wUWBN4Dfh/tm+StCNwMjAEeBw4tCZpaERERLSpVk2psCfwfkrm802ATwJfBw6xvSFwCKWeX4eVgK2AXYEzqm3fAP5i+33AgcApkpahBGsfsr0BcD1weuNvJyIiIvq7lhypAramlKJ5jTLKNFzSgsCHJe0FbAYsUtP+BtszJE0Clqo5x34Ath8CNq+KK68M3CwJYDDwfF/cUERERPRvrTpS9TqllAzw1oLx31FGre6lTAPWLjR7FcB27ar9zudYkxJE3WZ7uO3hwAhgj8bcQkRERAwkrRpU3UopRTNE0kLADcA6wAnAtcBHKQHSnM6xL7wVUF0H3EMZsVqjavN1Zk4XRkRERBtryaDK9m+A24H7gN8DZwE/Ah6mlKRZFFhI0sI9nOZEYHVJDwAXAwfafhr4FDBO0kPAhsAxDbuRiIiIGDCSp6qBkqcqIiKi5SRPVTOlTE1ERETra8npv4iIiIi+lum/BhoI03+Z9ouIiHhbMv3XTP25TE3K00RERNTHgAyqJK1IeSNvYeBN4Mhq19nAgsCzwGdt/0XSRMpbgFtW+74MfAFYCzjL9lmSFgF+QEm7MBg43falkuYD/hfYjpKz6kLbp0saCXwV+C/wPuAhYD/b0xp97xEREdE/DdQ1VZ8GxtvemJJ76gPAZcARttcHRgOX1rQfZHsT4FfA94CPUcrSnFDt/xpwr+2NqnMdL+m9wGGUEjbrURKH7iFpl+qYLYAjKEHVysCHGnSvERERMQAM1KDqRuBYSZcA7wKuAf5l+/cAtn8BrCZp8ar9tdXvqcBdtv9reyqlsDLA9sBhku6nJP1cGFgb2BYYa/sN2/+ljI5tVx0zyfYTtt+k5L7qn/N7ERER0ScGZFBl+3bK9N31wMcpo0+dDWJm1vTaabnpXbQdDBxQU35mM0oG9c7PZxAzp0xfrdk+gx4WrkVERETrG5BBlaRvU4KgCyhTcMOBd0kaUe3fG5hqu7fFjm8CPlcd+27gQcqU3k3AJyQNrsrd7A/cXNebiYiIiJYwIBeqU0amLpH0SeAN4ADgeeD7VemZ5ykjWL11EvBDSZMoo1Zfsv1XST8G1gAeAIYAF9v+TbVQvdcm3zCmX6dUiIiIiHmXPFUN1JGnasKECcmoHhER0Rq6Xe4zIKf/IiIiIvqbBFURERERdZDpv7kgaXlgjO2dJX0YWMP2d7toN4x+XKYmJWoiIiLetpSpqSfbTwE7V183nlP7/lqmJiVqIiIi6qftgypJDwF72360Sib6ou3PSdqckmn9KUr5mmUpqRb2rT5PpARWh1XnmWr7/CbcQkRERPQDWVMFVzMzS/q6lBqBADsC9wDTbG8OrEbJwN4xQoXtRyglcUYnoIqIiGhvbT9SRSlxc7Skm4CHgTUlDQV2AvYEFpV0OLAmsDqwSNN6GhEREf1WRqrgDmB9Sv2/icAtlGBqCCVT+8XAf4HzKXUBU44mIiIiZtP2QZXt6ZRpviMpQdVNwPGUEaztgXHV1N4LwDbMrCfYYToZ8YuIiGh7CQaKq4GtbU+W9HfKQvTxwEuUcjj7Uooy3w6s0unYW4ELJP3DdleFnfttmZqUqImIiKif5KlqoJSpiYiIaDkpUxMRERHRSBmpaqD+nFE92dQjIiLmSjKqN5Kk84FRtqd2tb8/ZlRPNvWIiIj6yvRffWxDUi1ERES0tbYZqepFOZrbgQOAN4AbgC8BKwHXAc8CrwDHAOdSnturwCeBPYDlgWskbWX7ub69s4iIiOgP2mmkqqdyNOOBXSnFkTeglKQ5rNov4ADbOwBHA2fa3hj4CbCZ7dMo9QF3TkAVERHRvtopqLoG2E7SWpRyNG/UlKMZAVxq+79VMtDzmBmAPWN7SvX5auD7kn4KvAhc0pc3EBEREf1XOwVVPZWjeaFT20HMnBp9pWOj7V8CG1IysB9NKaYcERER0T5B1RzK0dwE7CvpnZLeQVkrdXPnc0j6OTDC9o+Br1MCLEipmoiIiLbXboFAl+VobN8paTjwB8ozuQH4HtA5Dfq3gDGSTqCUrflctX08ZaH6h2w/3vmi/bFMTUrURERE1FeSfzZQytRERES0nJSpiYiIiGikjFRVJC0OjLW9ew9tZtieLUKVNAUYWfOWYMf2YfSzMjUpTxMRETFPUqamF5ak5Kiqu/5UpiblaSIiIhojQdVM5wDLS/oNcCUle/oM4F7gCNsvdTSUtBRwESXj+iPAgn3f3YiIiOhPsqZqpiMpmdFPoKRa2Nr2usDLwImd2p4M3Fft/wHlLcKIiIhoYwmqZrc1cFVNyZlzmZldvcNI4OcAtm8FHuuz3kVERES/lKBqdp2fSW129Q4zmHWh2vSG9igiIiL6vQRVM3VkRZ8I7FqtmwI4lNmzq98IHAggaQSlAHNERES0sSxUn+kfwN+As4FTgVskDaEsVD+sU9sTgbGSHgYmk+m/iIiItpc8VQ2UPFUREREtJ3mqmmn85eNSpiYiIqLFZU1VRERERB1k+m8eSToZ2JPyRuBPbX+3Zt8w+tH0X6b+IiIi5lmm/xpB0tbAtsB6wBDgEUlX23Ztu/5SpiYlaiIiIhon03/zwPYtwDa2pwNDKUHqy83tVURERDRDgqp5ZPt1SSdRagBOAJ5scpciIiKiCRJU1YHtE4FlKAWWD21ydyIiIqIJElTNA0lrShoOYPu/wK8p66siIiKizWSh+rx5L3CSpC0pb/99FDivuV2KiIiIZshI1TywfQ1wNfBHSjmbO2xf1txeRURERDMkT1UDJU9VREREy0meqiYZDDBm9Dkst9xyze4LAE888USzuxARETFgbbfddsOAJ6p0SrNIUNVYqwPsv//+ze5HRERE1MfjwCrAlM47ElQ11mPV762BvzWzI21oReB3wFZAhuf6Xp5/c+X5N0+efXP11fPv8twJqhprWvX7b7anNLMj7UZSx8cn8uz7Xp5/c+X5N0+efXM1+/nn7b+IiIiIOkhQFREREVEHCaoiIiIi6iBBVWO9AJxU/Y6+lWffXHn+zZXn3zx59s3V1Oef5J8RERERdZCRqoiIiIg6SFAVERERUQfJU9UgkvYDvgYMAf7X9g+a3KWWJ+lEYO/q69W2vyRpe+C7wDuBn9v+WtM62AYknQEsbftgScOBMcBiwK3AYV2VdYh5J+kjwInAwsANtr+Qv/2+I+kA4Ljq67W2j83ff2NJWgy4A/iw7Snd/b339T+HjFQ1gKQVgFOALYHhwGckrdXcXrW26n9QHwQ2oDzzjSTtC5wHfBR4HzBC0k7N62Vrk7Qd8ImaTRcBR9heg1KA9NCmdKzFSXovMBrYDVgP2LD6O8/ffh+QtBBwDqVyxvrAVtW/j/L33yCSNgVuA9aovr+T7v/e+/SfQ4Kqxvj/7d0/SFZRGMfxr/2BooYQgopMCOMZimgqoqEhQxoaokIwgv7Y1BJkU9QY0VBQQ7Q5RDkohQhFadGWQWCB5o+GkCyjoCUKUYyGcyUj3e4feP19QHjvfe9wOOe55z73nPN6moFnkr5L+gl0A0cqLlOtmwDOS5qSNA28I91w7yV9yN5M7gJHqyxkrYqIetKLxJXsuBFYKelldkknrvuiHCK9mY9nsd8K/MKxX5alpGfpKtLMxHJgGsd/kc4AZ4HP2fFO5on3KvohT/8VYwPpIT9rgtToVhBJw7OfI2ILaRrwFv+3w8aSi7ZY3AEuAg3Z8Xz3gOu+GE3AVET0ApuAPmAY138pJP2IiEvAKCmZfUHaosz1XxBJ7fDPljQL9Tel90MeqSrGEmDu/6qoA35XVJZFJSK2Ak+BC6QNrd0OBYuIduCjpIE5p30PlGcZaXT8NLAb2AVsxvVfiojYDpwCGkkP8RnSUgTXf3kW6m9K74c8UlWMcdIO2bPW8XeY0goSEXuAHuCcpK6I2Ausn3OJ26EYrcD6iBgC6oHVpI7MdV+OL0C/pG8AEfGANMUxM+ca139xWoABSV8BIqIT6MDxX6Zx5q/vhc4XxiNVxegH9kXE2mwR42HgccVlqmkR0QA8BNokdWWnB9NX0RQRS4E24FFVZaxVkvZL2iZpB3AZ6JV0EpjMEl2A47jui9IHtETEmizOD5DWcTr2y/EGaI6IVRFRBxwkTQE6/sszb18vaYyS28FJVQEkfSKtL3kODAH3JL2qtlQ1rwNYAVyPiKFs1ORE9tcDjJDWPHRXVcBF6BhwIyJGSaNXNysuT02SNAhcI/0aagQYA27j2C+FpCfAfeA18Ja0UP0qjv/SSJpk4XgvtR28TY2ZmZlZDjxSZWZmZpYDJ1VmZmZmOXBSZWZmZpYDJ1VmZmZmOXBSZWZmZpYDJ1VmZmZmOXBSZWZmZpYDJ1VmZmZmOfgDaibsvRH2M28AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from statlearning import plot_feature_importance\n",
    "plot_feature_importance(xgb, features, max_features=30)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3.4 LightGBM**\n",
    "\n",
    "I'm setting up the search space for LightGBM differently from XGBoost to encourage diversity between the two models. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightgbm import LGBMClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters found by Bayesian optimisation: {'colsample_bytree': 0.23280445410594658, 'learning_rate': 0.008975080497395874, 'n_estimators': 1643, 'num_leaves': 775, 'reg_alpha': 0.0004654357597697516, 'subsample': 0.6207993249264564} \n",
      "\n",
      "Wall time: 4min 31s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "model = LGBMClassifier()\n",
    "\n",
    "space = {\n",
    "    'reg_alpha':  (1e-10, 1000, 'log-uniform'),\n",
    "    'learning_rate': Real(0.005, 0.15),\n",
    "    'n_estimators' : Integer(100, 2000),\n",
    "    'num_leaves' : Integer(2, 2**10),\n",
    "    'subsample' : Real(0.5, 1.0),\n",
    "    'colsample_bytree' : Real(0.1, 1.0),\n",
    "}\n",
    "\n",
    "np.random.seed(87)\n",
    "lgb_search =  BayesSearchCV(model, space, cv = 5,  n_iter=32, scoring = 'neg_log_loss', n_jobs=4)\n",
    "lgb_search.fit(X_train, y_train)\n",
    "\n",
    "lgb = lgb_search.best_estimator_\n",
    "\n",
    "print('Best parameters found by Bayesian optimisation:', lgb_search.best_params_, '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3.5 Linear Support Vector Classifier**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters found by grid search: {'C': 0.125} \n",
      "\n",
      "Wall time: 3.97 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "Cs = np.logspace(-10, 10, 81, base=2)\n",
    "\n",
    "model = LinearSVC(loss='hinge')\n",
    "\n",
    "tuning_parameters ={\n",
    "    'C': Cs,\n",
    "}\n",
    "\n",
    "svm_search = GridSearchCV(model, tuning_parameters, cv=5, return_train_score=False, n_jobs=4)\n",
    "svm_search.fit(X_train, y_train)\n",
    "\n",
    "svm = svm_search.best_estimator_\n",
    "\n",
    "print('Best parameters found by grid search:', svm_search.best_params_, '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Voting Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('clf1', BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True)), ('clf2', LogisticRegression(C=1.2067926406393288, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='warn', n_jobs=None, penalty='l1', ran...    silent=True, subsample=0.6207993249264564,\n",
       "        subsample_for_bin=200000, subsample_freq=0))],\n",
       "         flatten_transform=None, n_jobs=None, voting='hard', weights=None)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "clfs = [('clf1', nbc), ('clf2',  logit_l1), ('clf3',  logit_l2),  ('clf4',  xgb), ('clf5',  svm) \n",
    "        ,('clf6', lgb)]\n",
    "\n",
    "vhard = VotingClassifier(clfs)\n",
    "vhard.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('clf1', BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True)), ('clf2', LogisticRegression(C=1.2067926406393288, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='warn', n_jobs=None, penalty='l1', ran...    silent=True, subsample=0.6207993249264564,\n",
       "        subsample_for_bin=200000, subsample_freq=0))],\n",
       "         flatten_transform=None, n_jobs=None, voting='soft', weights=None)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clfs = [('clf1', nbc), ('clf2', logit_l1), ('clf3',  logit_l2), ('clf4',  xgb),  ('clf5',  lgb)]\n",
    "# We exclude SVM since it does not predict probabilities\n",
    "\n",
    "vsoft = VotingClassifier(clfs, voting='soft')\n",
    "vsoft.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Model Stacking\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 60 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "StackingCVClassifier(classifiers=[BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True), LogisticRegression(C=1.2067926406393288, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='warn', n_jobs=None, penalty='l1', random_state=None,\n",
       "  ...     silent=True, subsample=0.6207993249264564,\n",
       "        subsample_for_bin=200000, subsample_freq=0)],\n",
       "           cv=5, drop_last_proba=False,\n",
       "           meta_classifier=LogisticRegression(C=inf, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='lbfgs',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "           n_jobs=None, pre_dispatch='2*n_jobs', random_state=None,\n",
       "           shuffle=True, store_train_meta_features=False, stratify=True,\n",
       "           use_clones=True, use_features_in_secondary=False,\n",
       "           use_probas=True, verbose=0)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "from mlxtend.classifier import StackingCVClassifier\n",
    "\n",
    "stack = StackingCVClassifier([nbc, logit_l1, logit_l2, xgb, lgb], use_probas=True, \n",
    "                             meta_classifier = LogisticRegression(C=np.inf, solver='lbfgs'), cv=5)\n",
    "stack.fit(X_train, y_train) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Model Evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Error rate</th>\n",
       "      <th>Sensitivity</th>\n",
       "      <th>Specificity</th>\n",
       "      <th>AUC</th>\n",
       "      <th>Precision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Naive Bayes</th>\n",
       "      <td>0.056</td>\n",
       "      <td>0.844</td>\n",
       "      <td>0.964</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic $\\ell_1$</th>\n",
       "      <td>0.052</td>\n",
       "      <td>0.789</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.969</td>\n",
       "      <td>0.895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic $\\ell_2$</th>\n",
       "      <td>0.052</td>\n",
       "      <td>0.782</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Random Forest</th>\n",
       "      <td>0.076</td>\n",
       "      <td>0.659</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGBoost</th>\n",
       "      <td>0.052</td>\n",
       "      <td>0.778</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.971</td>\n",
       "      <td>0.905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LightGBM</th>\n",
       "      <td>0.046</td>\n",
       "      <td>0.800</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Linear SVC</th>\n",
       "      <td>0.054</td>\n",
       "      <td>0.763</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hard Voting</th>\n",
       "      <td>0.049</td>\n",
       "      <td>0.780</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Soft Voting</th>\n",
       "      <td>0.048</td>\n",
       "      <td>0.802</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stack</th>\n",
       "      <td>0.045</td>\n",
       "      <td>0.811</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.913</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Error rate  Sensitivity  Specificity    AUC  Precision\n",
       "Naive Bayes             0.056        0.844        0.964  0.965      0.829\n",
       "Logistic $\\ell_1$       0.052        0.789        0.981  0.969      0.895\n",
       "Logistic $\\ell_2$       0.052        0.782        0.982  0.970      0.899\n",
       "Random Forest           0.076        0.659        0.978  0.949      0.862\n",
       "XGBoost                 0.052        0.778        0.983  0.971      0.905\n",
       "LightGBM                0.046        0.800        0.986  0.973      0.919\n",
       "Linear SVC              0.054        0.763        0.983  0.967      0.904\n",
       "Hard Voting             0.049        0.780        0.986  0.000      0.922\n",
       "Soft Voting             0.048        0.802        0.983  0.972      0.908\n",
       "Stack                   0.045        0.811        0.984  0.973      0.913"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns=['Error rate', 'Sensitivity', 'Specificity', 'AUC', 'Precision']\n",
    "rows=['Naive Bayes', 'Logistic $\\ell_1$', 'Logistic $\\ell_2$', 'Random Forest', \n",
    "      'XGBoost', 'LightGBM', 'Linear SVC', 'Hard Voting', 'Soft Voting', 'Stack']\n",
    "\n",
    "results=pd.DataFrame(0.0, columns=columns, index=rows) \n",
    "\n",
    "methods=[nbc, logit_l1, logit_l2, rf, xgb, lgb, svm, vhard, vsoft, stack]\n",
    "\n",
    "y_prob = np.zeros((len(y_test), len(rows)))\n",
    "\n",
    "for i, method in enumerate(methods):\n",
    "    \n",
    "    y_pred = method.predict(X_test) \n",
    "   \n",
    "    if method not in [svm, vhard]: # svm and vhard do not predict probabilities\n",
    "            y_prob[:, i] = method.predict_proba(X_test)[:,1]\n",
    "            results.iloc[i,3]=  roc_auc_score(y_test, y_prob[:,i])   \n",
    "    elif method == svm:\n",
    "        y_df = method.decision_function(X_test)\n",
    "        results.iloc[i,3]=  roc_auc_score(y_test, y_df)  \n",
    "\n",
    "    confusion  = confusion_matrix(y_test, y_pred) \n",
    "    results.iloc[i,0]=  1 - accuracy_score(y_test, y_pred)\n",
    "    results.iloc[i,1]=  confusion[1,1]/np.sum(confusion[1,:])\n",
    "    results.iloc[i,2]=  confusion[0,0]/np.sum(confusion[0,:])\n",
    "    results.iloc[i,4]=  precision_score(y_test, y_pred)\n",
    "\n",
    "results.round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Additional Code\n",
    "\n",
    "The two cells below format the notebook for display online. Please omit them from your work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "@import url('https://fonts.googleapis.com/css?family=Source+Sans+Pro|Open+Sans:800&display=swap');\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<style>\n",
    "@import url('https://fonts.googleapis.com/css?family=Source+Sans+Pro|Open+Sans:800&display=swap');\n",
    "</style>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.CodeMirror pre {font-family: monospace; font-size: 12pt; line-height: 22.4px;}\n",
       "\n",
       "pre {font-family: monospace; font-size: 12pt; line-height: 22.4px;}\n",
       "\n",
       ".output_area pre {font-family: monospace; font-size: 12pt; line-height: 22.4px;}\n",
       "\n",
       ".rendered_html table {font-size: 10.5pt;}\n",
       "\n",
       "h1 {\n",
       "  font-family: 'Open Sans' !important;\n",
       "  color: #222;\n",
       "  font-size: 2.5em !important;\n",
       "  padding-bottom: 0px;  \n",
       "  margin-top: 14px;\n",
       "  margin-bottom: 6px;\n",
       "  font-weight: 800 !important;\n",
       "}\n",
       "\n",
       "\n",
       "h2 {\n",
       "  font-family: 'Open Sans' !important;\n",
       "  font-size: 1.6em !important;\n",
       "  font-weight: 800 !important;\n",
       "  color: #222;\n",
       "}\n",
       "\n",
       "\n",
       "h3 {\n",
       "  font-family: 'Open Sans' !important;\n",
       "  font-size: 25px !important;\n",
       "  color: #222 !important;\n",
       "  margin-bottom: 30px !important;\n",
       "  margin-top: 25px !important;\n",
       "}\n",
       "\n",
       "\n",
       "p {\n",
       "  font-family: 'Source Sans Pro', sans-serif;\n",
       "  color: #222;\n",
       "  font-size: 19px;\n",
       "  margin-bottom: 10px;\n",
       "  margin-top:0;\n",
       "  line-height: 1.4;\n",
       "}\n",
       "\n",
       "li {\n",
       "  font-family: 'Source Sans Pro', sans-serif;\n",
       "  color: #222;\n",
       "  font-size: 19px;\n",
       "  line-height: 1.4;\n",
       "}\n",
       "\n",
       "\n",
       "\n",
       "/*code {\n",
       "     padding: 2px 4px !important; \n",
       "     font-size: 90% !important;\n",
       "     color: #c7254e !important; \n",
       "     background-color: #f9f2f4 !important; \n",
       "     border-radius: 2px !important;\n",
       "     text-decoration: none !important;\n",
       "      font-size: 90% !important;\n",
       "}\n",
       "*/\n",
       "\n",
       "\n",
       "code {\n",
       "      font-size: 90% !important;\n",
       "/*      border-style: solid !important;\n",
       "      border-width: 1px !important;\n",
       "      border-color :#cccccc !important;*/\n",
       "}\n",
       "\n",
       "TT {\n",
       "      font-family: monospace;\n",
       "      font-size: 95% !important;\n",
       "}\n",
       "\n",
       "\n",
       "\n",
       "a.neutral {\n",
       "  text-decoration: none !important;\n",
       "  border-style: dotted !important;\n",
       "  border-width: 1px !important;\n",
       "  border-color :#cccccc !important;\n",
       "\n",
       "}\n",
       "\n",
       "</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.core.display import HTML\n",
    "style = open('jstyle.css', \"r\").read()\n",
    "HTML('<style>'+ style +'</style>')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}