02_bag-of-tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Bag of Tweets\n",
    "\n",
    "####  Eryk Wdowiak and Eric Adsetts\n",
    "\n",
    "Module 4 project -- sentiment in Tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import the libraries we need (and maybe a few that we didn;t need)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "# import re\n",
    "import string\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from  nltk import FreqDist\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import accuracy_score, f1_score, confusion_matrix\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the data. Remove tweet without text. Rename columns to something easier to work with and make the target data numeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>direction</th>\n",
       "      <th>emotion</th>\n",
       "      <th>target</th>\n",
       "      <th>company</th>\n",
       "      <th>product</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>.@wesley83 I have a 3G iPhone. After 3 hrs twe...</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>negative</td>\n",
       "      <td>0</td>\n",
       "      <td>Apple</td>\n",
       "      <td>device</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@jessedee Know about @fludapp ? Awesome iPad/i...</td>\n",
       "      <td>iPad or iPhone App</td>\n",
       "      <td>positive</td>\n",
       "      <td>1</td>\n",
       "      <td>Apple</td>\n",
       "      <td>software</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@swonderlin Can not wait for #iPad 2 also. The...</td>\n",
       "      <td>iPad</td>\n",
       "      <td>positive</td>\n",
       "      <td>1</td>\n",
       "      <td>Apple</td>\n",
       "      <td>device</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@sxsw I hope this year's festival isn't as cra...</td>\n",
       "      <td>iPad or iPhone App</td>\n",
       "      <td>negative</td>\n",
       "      <td>0</td>\n",
       "      <td>Apple</td>\n",
       "      <td>software</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@sxtxstate great stuff on Fri #SXSW: Marissa M...</td>\n",
       "      <td>Google</td>\n",
       "      <td>positive</td>\n",
       "      <td>1</td>\n",
       "      <td>Google</td>\n",
       "      <td>company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>@teachntech00 New iPad Apps For #SpeechTherapy...</td>\n",
       "      <td>nan</td>\n",
       "      <td>neutral</td>\n",
       "      <td>2</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>#SXSW is just starting, #CTIA is around the co...</td>\n",
       "      <td>Android</td>\n",
       "      <td>positive</td>\n",
       "      <td>1</td>\n",
       "      <td>Google</td>\n",
       "      <td>device</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Beautifully smart and simple idea RT @madebyma...</td>\n",
       "      <td>iPad or iPhone App</td>\n",
       "      <td>positive</td>\n",
       "      <td>1</td>\n",
       "      <td>Apple</td>\n",
       "      <td>software</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Counting down the days to #sxsw plus strong Ca...</td>\n",
       "      <td>Apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>1</td>\n",
       "      <td>Apple</td>\n",
       "      <td>company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Excited to meet the @samsungmobileus at #sxsw ...</td>\n",
       "      <td>Android</td>\n",
       "      <td>positive</td>\n",
       "      <td>1</td>\n",
       "      <td>Google</td>\n",
       "      <td>device</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                tweet           direction  \\\n",
       "0   .@wesley83 I have a 3G iPhone. After 3 hrs twe...              iPhone   \n",
       "1   @jessedee Know about @fludapp ? Awesome iPad/i...  iPad or iPhone App   \n",
       "2   @swonderlin Can not wait for #iPad 2 also. The...                iPad   \n",
       "3   @sxsw I hope this year's festival isn't as cra...  iPad or iPhone App   \n",
       "4   @sxtxstate great stuff on Fri #SXSW: Marissa M...              Google   \n",
       "5   @teachntech00 New iPad Apps For #SpeechTherapy...                 nan   \n",
       "7   #SXSW is just starting, #CTIA is around the co...             Android   \n",
       "8   Beautifully smart and simple idea RT @madebyma...  iPad or iPhone App   \n",
       "9   Counting down the days to #sxsw plus strong Ca...               Apple   \n",
       "10  Excited to meet the @samsungmobileus at #sxsw ...             Android   \n",
       "\n",
       "     emotion  target  company   product  \n",
       "0   negative       0    Apple    device  \n",
       "1   positive       1    Apple  software  \n",
       "2   positive       1    Apple    device  \n",
       "3   negative       0    Apple  software  \n",
       "4   positive       1   Google   company  \n",
       "5    neutral       2  unknown   unknown  \n",
       "7   positive       1   Google    device  \n",
       "8   positive       1    Apple  software  \n",
       "9   positive       1    Apple   company  \n",
       "10  positive       1   Google    device  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##  load data\n",
    "data = pd.read_csv('judge-1377884607_tweet_product_company_v2-clean.csv')\n",
    "data.columns = ['tweet','direction','emotion']\n",
    "# data.shape  # (9093, 3)\n",
    "\n",
    "##  remove rows without tweet\n",
    "data = data.dropna(subset=['tweet','emotion'],axis='index')\n",
    "# data.shape  # (9092, 3)\n",
    "\n",
    "##  clean emotions\n",
    "emo_dict = {'Negative emotion':'negative', \n",
    "            'Positive emotion':'positive',\n",
    "            'No emotion toward brand or product':'neutral', \n",
    "            \"I can't tell\":'neutral'}\n",
    "data['emotion'] = data['emotion'].replace(emo_dict)\n",
    "del emo_dict\n",
    "\n",
    "#Make target numeric\n",
    "target_dict = {'negative': 0,\n",
    "              'positive': 1,\n",
    "              'neutral': 2}\n",
    "data['target'] = data['emotion'].map(target_dict)\n",
    "##  define company and product\n",
    "##  first convert NaN to a string\n",
    "data['direction'] = data['direction'].map('{}'.format)\n",
    "\n",
    "##  define company\n",
    "comp_dict = {'iPhone':'Apple', \n",
    "             'iPad or iPhone App':'Apple', \n",
    "             'iPad':'Apple', \n",
    "             'Google':'Google', \n",
    "             'nan':'unknown', \n",
    "             'Android':'Google',\n",
    "             'Apple':'Apple',\n",
    "             'Android App':'Google', \n",
    "             'Other Google product or service':'Google',\n",
    "             'Other Apple product or service':'Apple'}\n",
    "data['company'] = data['direction'].replace(comp_dict)\n",
    "del comp_dict\n",
    "\n",
    "##  define product\n",
    "prod_dict = {'iPhone':'device', \n",
    "             'iPad or iPhone App':'software', \n",
    "             'iPad':'device', \n",
    "             'Google':'company', \n",
    "             'nan':'unknown', \n",
    "             'Android':'device',\n",
    "             'Apple':'company',\n",
    "             'Android App':'software', \n",
    "             'Other Google product or service':'other',\n",
    "             'Other Apple product or service':'other'}\n",
    "data['product'] = data['direction'].replace(prod_dict)\n",
    "del prod_dict\n",
    "\n",
    "##  let's take a look\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove stop words and tokenize and lemmatize tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  prepare stop word list\n",
    "stopwords_list = stopwords.words('english')\n",
    "stopwords_list += list(string.punctuation)\n",
    "stopwords_list += ['0','1','2','3','4','5','6','7','8','9']\n",
    "\n",
    "##  process tweets\n",
    "def process_tweets(tweet):\n",
    "    tokens = word_tokenize(tweet)\n",
    "    wnl = WordNetLemmatizer()\n",
    "    no_stop_lemmas = [wnl.lemmatize(token.lower()) for token in tokens if token.lower() not in stopwords_list]\n",
    "    ot_string = ' '.join(no_stop_lemmas)\n",
    "    return ot_string\n",
    "\n",
    "##  process tweets\n",
    "data['tweet'] = list(map(process_tweets, list(data['tweet'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'wesley83 3g iphone hr tweeting rise_austin dead need upgrade plugin station sxsw'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['tweet'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "perform train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  train test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(data[['tweet','company','product']], \n",
    "                                                    data['target'], \n",
    "                                                    test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>company</th>\n",
       "      <th>product</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8256</th>\n",
       "      <td>next life 'm coming back ipad woman ca n't kee...</td>\n",
       "      <td>Apple</td>\n",
       "      <td>device</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6516</th>\n",
       "      <td>rt mention rt mention best thing 've heard wee...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7256</th>\n",
       "      <td>would love meet u rt mention 'll austin conven...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2116</th>\n",
       "      <td>iphone crashed front sxsw apple pop-up bestwor...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1545</th>\n",
       "      <td>mention mention mention platformer ci di venue...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5735</th>\n",
       "      <td>rt mention notatsxsw sxsw 's link free downloa...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5192</th>\n",
       "      <td>rt mention quot future local contextual discov...</td>\n",
       "      <td>Google</td>\n",
       "      <td>company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5391</th>\n",
       "      <td>rt mention android may gaining market share 'd...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>861</th>\n",
       "      <td>mention love mention mention sxsw quot apple c...</td>\n",
       "      <td>Apple</td>\n",
       "      <td>company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7271</th>\n",
       "      <td>anyone know status ipad 2 austin pop-up store ...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7273 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  tweet  company  product\n",
       "8256  next life 'm coming back ipad woman ca n't kee...    Apple   device\n",
       "6516  rt mention rt mention best thing 've heard wee...  unknown  unknown\n",
       "7256  would love meet u rt mention 'll austin conven...  unknown  unknown\n",
       "2116  iphone crashed front sxsw apple pop-up bestwor...  unknown  unknown\n",
       "1545  mention mention mention platformer ci di venue...  unknown  unknown\n",
       "...                                                 ...      ...      ...\n",
       "5735  rt mention notatsxsw sxsw 's link free downloa...  unknown  unknown\n",
       "5192  rt mention quot future local contextual discov...   Google  company\n",
       "5391  rt mention android may gaining market share 'd...  unknown  unknown\n",
       "861   mention love mention mention sxsw quot apple c...    Apple  company\n",
       "7271  anyone know status ipad 2 austin pop-up store ...  unknown  unknown\n",
       "\n",
       "[7273 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tf-idf vectorize the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  create TF-IDF matrix\n",
    "vectorizer = TfidfVectorizer(ngram_range = (1,3))\n",
    "tf_idf_data_train = vectorizer.fit_transform(X_train['tweet'])\n",
    "tf_idf_data_test = vectorizer.transform(X_test['tweet'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify with naive bayes than run a grid search for optimal hyper parameters. Creates a new naive bayes classifier using these hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multinomial Naive Bayes\n",
      "Training Accuracy: 0.9428\n",
      "Testing Accuracy:  0.6712\n",
      "Training f1: 0.9427\n",
      "Testing f1:  0.6531\n"
     ]
    }
   ],
   "source": [
    "##  naive bayes classifier\n",
    "nb_classifier = MultinomialNB(alpha = .1)\n",
    "\n",
    "nb_classifier.fit(tf_idf_data_train, y_train)\n",
    "nb_train_preds = nb_classifier.predict(tf_idf_data_train)\n",
    "nb_test_preds = nb_classifier.predict(tf_idf_data_test)\n",
    "\n",
    "nb_train_score = accuracy_score(y_train, nb_train_preds)\n",
    "nb_test_score = accuracy_score(y_test, nb_test_preds)\n",
    "nb_train_f1 = f1_score(y_train, nb_train_preds, average='weighted')\n",
    "nb_test_f1 = f1_score(y_test, nb_test_preds, average = 'weighted')\n",
    "\n",
    "print(\"Multinomial Naive Bayes\")\n",
    "print(\"Training Accuracy: {:.4}\".format(nb_train_score))\n",
    "print(\"Testing Accuracy:  {:.4}\".format(nb_test_score))\n",
    "print(\"Training f1: {:.4}\".format(nb_train_f1))\n",
    "print(\"Testing f1:  {:.4}\".format(nb_test_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 5 candidates, totalling 25 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6693246737285932\n",
      "{'alpha': 0.5}\n",
      "0.6178412200378456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  25 out of  25 | elapsed:    1.5s finished\n"
     ]
    }
   ],
   "source": [
    "#hyper parameter tuning\n",
    "nb_grid = GridSearchCV(nb_classifier, {'alpha': [.01,.1,.25,.5,1]}, n_jobs = -1, verbose = 1)\n",
    "nb_grid.fit(tf_idf_data_train, y_train)\n",
    "print(nb_grid.best_score_)\n",
    "print(nb_grid.best_params_)\n",
    "nb_grid_preds = nb_grid.predict(tf_idf_data_test)\n",
    "nb_grid_f1 = f1_score(y_test, nb_grid_preds, average = 'weighted')\n",
    "print(nb_grid_f1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=0.5)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_optimal = MultinomialNB(alpha = .5)\n",
    "nb_optimal.fit(tf_idf_data_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify with random forest than run a grid search for optimal hyper parameters. Creates a new random forest classifier using these hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forests\n",
      "Training Accuracy: 0.9953\n",
      "Testing Accuracy:  0.6603\n",
      "Training f1: 0.9953\n",
      "Testing f1:  0.6188\n"
     ]
    }
   ],
   "source": [
    "##  random forests classifier\n",
    "rf_classifier = RandomForestClassifier(n_estimators=100, class_weight = 'balanced', max_depth = None, n_jobs = -1)\n",
    "\n",
    "rf_classifier.fit(tf_idf_data_train, y_train)\n",
    "rf_train_preds = rf_classifier.predict(tf_idf_data_train)\n",
    "rf_test_preds = rf_classifier.predict(tf_idf_data_test)\n",
    "\n",
    "rf_train_score = accuracy_score(y_train, rf_train_preds)\n",
    "rf_test_score = accuracy_score(y_test, rf_test_preds)\n",
    "rf_train_f1 = f1_score(y_train, rf_train_preds, average='weighted')\n",
    "rf_test_f1 = f1_score(y_test, rf_test_preds, average = 'weighted')\n",
    "    \n",
    "print('Random Forests')\n",
    "print(\"Training Accuracy: {:.4}\".format(rf_train_score))\n",
    "print(\"Testing Accuracy:  {:.4}\".format(rf_test_score))\n",
    "print(\"Training f1: {:.4}\".format(rf_train_f1))\n",
    "print(\"Testing f1:  {:.4}\".format(rf_test_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 48 candidates, totalling 240 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:  9.4min\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed: 15.6min\n",
      "[Parallel(n_jobs=-1)]: Done 240 out of 240 | elapsed: 16.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6741376555727298\n",
      "{'max_depth': None, 'min_samples_split': 6, 'n_estimators': 500}\n",
      "0.643211554738738\n"
     ]
    }
   ],
   "source": [
    "#hyper parameter tuning\n",
    "rf_grid = GridSearchCV(rf_classifier, {'n_estimators' : [100,250,500],\n",
    "                                       'max_depth': [None, 1,3,5],\n",
    "                                       'min_samples_split': [2,6,10,20]},\n",
    "                       n_jobs = -1, verbose = 1)\n",
    "rf_grid.fit(tf_idf_data_train, y_train)\n",
    "print(rf_grid.best_score_)\n",
    "print(rf_grid.best_params_)\n",
    "rf_grid_preds = rf_grid.predict(tf_idf_data_test)\n",
    "rf_grid_f1 = f1_score(y_test, rf_grid_preds, average = 'weighted')\n",
    "print(rf_grid_f1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_optimal = RandomForestClassifier(max_depth = None, min_samples_split = 6, n_estimators = 500, class_weight = 'balanced', n_jobs = -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify with logistic regression than run a grid search for optimal hyper parameters. Creates a new logistic regression classifier using these hyperparameters. This and svm are our best performing models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression\n",
      "Training Accuracy: 0.9229\n",
      "Testing Accuracy:  0.6526\n",
      "Training f1: 0.9234\n",
      "Testing f1:  0.6558\n"
     ]
    }
   ],
   "source": [
    "#Log Reg\n",
    "logreg = LogisticRegression(class_weight = 'balanced', n_jobs = -1)\n",
    "\n",
    "logreg.fit(tf_idf_data_train, y_train)\n",
    "lr_train_preds = logreg.predict(tf_idf_data_train)\n",
    "lr_test_preds = logreg.predict(tf_idf_data_test)\n",
    "\n",
    "lr_train_score = accuracy_score(y_train, lr_train_preds)\n",
    "lr_test_score = accuracy_score(y_test, lr_test_preds)\n",
    "lr_train_f1 = f1_score(y_train, lr_train_preds, average='weighted')\n",
    "lr_test_f1 = f1_score(y_test, lr_test_preds, average = 'weighted')\n",
    "    \n",
    "print('Logistic Regression')\n",
    "print(\"Training Accuracy: {:.4}\".format(lr_train_score))\n",
    "print(\"Testing Accuracy:  {:.4}\".format(lr_test_score))\n",
    "print(\"Training f1: {:.4}\".format(lr_train_f1))\n",
    "print(\"Testing f1:  {:.4}\".format(lr_test_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 13 candidates, totalling 65 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  50 tasks      | elapsed:   39.3s\n",
      "[Parallel(n_jobs=-1)]: Done  65 out of  65 | elapsed:   51.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6851351645183094\n",
      "{'C': 2}\n",
      "0.6674069043223244\n"
     ]
    }
   ],
   "source": [
    "#hyper parameter tuning\n",
    "lr_grid = GridSearchCV(logreg, {'C': [-1,-.5,-.1,.1,.5,1,1.5,2,2.5,10,20,25,50]},\n",
    "                       n_jobs = -1, verbose = 1)\n",
    "lr_grid.fit(tf_idf_data_train, y_train)\n",
    "print(lr_grid.best_score_)\n",
    "print(lr_grid.best_params_)\n",
    "lr_grid_preds = lr_grid.predict(tf_idf_data_test)\n",
    "lr_grid_f1 = f1_score(y_test, lr_grid_preds, average = 'weighted')\n",
    "print(lr_grid_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_optimal = LogisticRegression(class_weight = 'balanced', n_jobs = -1, C = 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify with decision tree than run a grid search for optimal hyper parameters. Creates a new decision tree classifier using these hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree\n",
      "Training Accuracy: 0.9953\n",
      "Testing Accuracy:  0.5899\n",
      "Training f1: 0.9953\n",
      "Testing f1:  0.5912\n"
     ]
    }
   ],
   "source": [
    "#Decision Tree\n",
    "tree = DecisionTreeClassifier(class_weight = 'balanced')\n",
    "\n",
    "tree.fit(tf_idf_data_train, y_train)\n",
    "dt_train_preds = tree.predict(tf_idf_data_train)\n",
    "dt_test_preds = tree.predict(tf_idf_data_test)\n",
    "\n",
    "dt_train_score = accuracy_score(y_train, dt_train_preds)\n",
    "dt_test_score = accuracy_score(y_test, dt_test_preds)\n",
    "dt_train_f1 = f1_score(y_train, dt_train_preds, average='weighted')\n",
    "dt_test_f1 = f1_score(y_test, dt_test_preds, average = 'weighted')\n",
    "    \n",
    "print('Decision Tree')\n",
    "print(\"Training Accuracy: {:.4}\".format(dt_train_score))\n",
    "print(\"Testing Accuracy:  {:.4}\".format(dt_test_score))\n",
    "print(\"Training f1: {:.4}\".format(dt_train_f1))\n",
    "print(\"Testing f1:  {:.4}\".format(dt_test_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 175 candidates, totalling 875 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:   25.5s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 434 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 784 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=-1)]: Done 875 out of 875 | elapsed:  3.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5784370169741488\n",
      "{'max_depth': None, 'min_samples_leaf': 1, 'min_samples_split': 4}\n",
      "0.5983203933920344\n"
     ]
    }
   ],
   "source": [
    "#hyper parameter tuning\n",
    "dt_grid = GridSearchCV(tree, {'max_depth': [None, 50,25,10,5], 'min_samples_split': [2,4,8,10,12,14,16],\n",
    "                             'min_samples_leaf': [1,5,10,15,20]},\n",
    "                       n_jobs = -1, verbose = 1)\n",
    "dt_grid.fit(tf_idf_data_train, y_train)\n",
    "print(dt_grid.best_score_)\n",
    "print(dt_grid.best_params_)\n",
    "dt_grid_preds = dt_grid.predict(tf_idf_data_test)\n",
    "dt_grid_f1 = f1_score(y_test, dt_grid_preds, average = 'weighted')\n",
    "print(dt_grid_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_optimal = DecisionTreeClassifier(class_weight = 'balanced', max_depth = None, min_samples_leaf = 1, min_samples_split = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify with knn than run a grid search for optimal hyper parameters. Creates a new knn classifier using these hyperparameters. Note: the optimal hyperparameters feel weird to me."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K Nearest Neighbors\n",
      "Training Accuracy: 0.7447\n",
      "Testing Accuracy:  0.6311\n",
      "Training f1: 0.7307\n",
      "Testing f1:  0.6079\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "knn = KNeighborsClassifier()\n",
    "\n",
    "knn.fit(tf_idf_data_train, y_train)\n",
    "knn_train_preds = knn.predict(tf_idf_data_train)\n",
    "knn_test_preds = knn.predict(tf_idf_data_test)\n",
    "\n",
    "knn_train_score = accuracy_score(y_train, knn_train_preds)\n",
    "knn_test_score = accuracy_score(y_test, knn_test_preds)\n",
    "knn_train_f1 = f1_score(y_train, knn_train_preds, average='weighted')\n",
    "knn_test_f1 = f1_score(y_test, knn_test_preds, average = 'weighted')\n",
    "    \n",
    "print('K Nearest Neighbors')\n",
    "print(\"Training Accuracy: {:.4}\".format(knn_train_score))\n",
    "print(\"Testing Accuracy:  {:.4}\".format(knn_test_score))\n",
    "print(\"Training f1: {:.4}\".format(knn_train_f1))\n",
    "print(\"Testing f1:  {:.4}\".format(knn_test_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 24 candidates, totalling 120 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    6.2s\n",
      "[Parallel(n_jobs=-1)]: Done 120 out of 120 | elapsed:   20.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6445766389200074\n",
      "{'n_neighbors': 1, 'p': 1}\n",
      "0.6093983363997821\n"
     ]
    }
   ],
   "source": [
    "#hyper parameter tuning\n",
    "knn_grid = GridSearchCV(knn, {'n_neighbors': [1,3,5,7,9,11,13,15], 'p': [1,2,3]},\n",
    "                       n_jobs = -1, verbose = 1)\n",
    "knn_grid.fit(tf_idf_data_train, y_train)\n",
    "print(knn_grid.best_score_)\n",
    "print(knn_grid.best_params_)\n",
    "knn_grid_preds = knn_grid.predict(tf_idf_data_test)\n",
    "knn_grid_f1 = f1_score(y_test, knn_grid_preds, average = 'weighted')\n",
    "print(knn_grid_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_optimal = KNeighborsClassifier(n_neighbors = 1, p = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify with support vector machine than run a grid search for optimal hyper parameters. Creates a new support vector machine classifier using these hyperparameters. This and logreg are our best performing models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Support Vector Machines\n",
      "Training Accuracy: 0.9432\n",
      "Testing Accuracy:  0.6658\n",
      "Training f1: 0.9436\n",
      "Testing f1:  0.6617\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "svm = SVC(class_weight = 'balanced')\n",
    "\n",
    "svm.fit(tf_idf_data_train, y_train)\n",
    "svm_train_preds = svm.predict(tf_idf_data_train)\n",
    "svm_test_preds = svm.predict(tf_idf_data_test)\n",
    "\n",
    "svm_train_score = accuracy_score(y_train, svm_train_preds)\n",
    "svm_test_score = accuracy_score(y_test, svm_test_preds)\n",
    "svm_train_f1 = f1_score(y_train, svm_train_preds, average='weighted')\n",
    "svm_test_f1 = f1_score(y_test, svm_test_preds, average = 'weighted')\n",
    "    \n",
    "print('Support Vector Machines')\n",
    "print(\"Training Accuracy: {:.4}\".format(svm_train_score))\n",
    "print(\"Testing Accuracy:  {:.4}\".format(svm_test_score))\n",
    "print(\"Training f1: {:.4}\".format(svm_train_f1))\n",
    "print(\"Testing f1:  {:.4}\".format(svm_test_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 28 candidates, totalling 140 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 140 out of 140 | elapsed:  5.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.684860061354623\n",
      "{'C': 10, 'kernel': 'linear'}\n",
      "0.6672589469062217\n"
     ]
    }
   ],
   "source": [
    "#hyper parameter tuning\n",
    "svm_grid = GridSearchCV(svm, {'C': [.5,1,5,10,15,25,50],\n",
    "                             'kernel': ['linear', 'poly', 'rbf', 'sigmoid']},\n",
    "                       n_jobs = -1, verbose = 1)\n",
    "svm_grid.fit(tf_idf_data_train, y_train)\n",
    "print(svm_grid.best_score_)\n",
    "print(svm_grid.best_params_)\n",
    "svm_grid_preds = svm_grid.predict(tf_idf_data_test)\n",
    "svm_grid_f1 = f1_score(y_test, svm_grid_preds, average = 'weighted')\n",
    "print(svm_grid_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_optimal = SVC(class_weight = 'balanced', C = 10, kernel = 'linear')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classify with xg boost than run a grid search for optimal hyper parameters. Creates a new xg boost classifier using these hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[21:45:22] WARNING: /Users/travis/build/dmlc/xgboost/src/learner.cc:480: \n",
      "Parameters: { num_classes } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "Support Vector Machines\n",
      "Training Accuracy: 0.8549\n",
      "Testing Accuracy:  0.6723\n",
      "Training f1: 0.8464\n",
      "Testing f1:  0.6357\n"
     ]
    }
   ],
   "source": [
    "xgb_clf = xgb.XGBClassifier(objective = 'multi:softmax', num_classes = 3, n_jobs = -1,\n",
    "                            verbosity = 1)\n",
    "\n",
    "xgb_clf.fit(tf_idf_data_train, y_train)\n",
    "xgb_train_preds = xgb_clf.predict(tf_idf_data_train)\n",
    "xgb_test_preds = xgb_clf.predict(tf_idf_data_test)\n",
    "\n",
    "xgb_train_score = accuracy_score(y_train, xgb_train_preds)\n",
    "xgb_test_score = accuracy_score(y_test, xgb_test_preds)\n",
    "xgb_train_f1 = f1_score(y_train, xgb_train_preds, average='weighted')\n",
    "xgb_test_f1 = f1_score(y_test, xgb_test_preds, average = 'weighted')\n",
    "    \n",
    "print('Support Vector Machines')\n",
    "print(\"Training Accuracy: {:.4}\".format(xgb_train_score))\n",
    "print(\"Testing Accuracy:  {:.4}\".format(xgb_test_score))\n",
    "print(\"Training f1: {:.4}\".format(xgb_train_f1))\n",
    "print(\"Testing f1:  {:.4}\".format(xgb_test_f1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: This cell took almost 3 HOURS to run. Do NOT run again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 405 candidates, totalling 2025 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:  9.6min\n",
      "[Parallel(n_jobs=-1)]: Done 434 tasks      | elapsed: 24.4min\n",
      "[Parallel(n_jobs=-1)]: Done 784 tasks      | elapsed: 48.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1234 tasks      | elapsed: 84.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1784 tasks      | elapsed: 137.0min\n",
      "[Parallel(n_jobs=-1)]: Done 2025 out of 2025 | elapsed: 164.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[17:43:11] WARNING: /Users/travis/build/dmlc/xgboost/src/learner.cc:480: \n",
      "Parameters: { num_classes } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "0.680185103778178\n",
      "{'colsample_bytree': 1, 'eta': 0.01, 'max_depth': 6, 'min_child_weight': 0.1, 'subsample': 1}\n",
      "0.6508281569125254\n"
     ]
    }
   ],
   "source": [
    "#hyper parameter tuning\n",
    "xgb_grid = GridSearchCV(xgb_clf, {'eta': [.01,.05,.1,.15,.2], 'min_child_weight': [.1, 1, 10], 'max_depth': [3, 6, 10],\n",
    "                                 'subsample': [.5, .75, 1], 'colsample_bytree': [.5,.75,1]},\n",
    "                       n_jobs = -1, verbose = 1)\n",
    "xgb_grid.fit(tf_idf_data_train, y_train)\n",
    "print(xgb_grid.best_score_)\n",
    "print(xgb_grid.best_params_)\n",
    "xgb_grid_preds = xgb_grid.predict(tf_idf_data_test)\n",
    "xgb_grid_f1 = f1_score(y_test, xgb_grid_preds, average = 'weighted')\n",
    "print(xgb_grid_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_optimal = xgb.XGBClassifier(objective = 'multi:softmax', num_classes = 3, n_jobs = -1, colsample_bytree = .5,\n",
    "                                eta = 0.01, max_depth = 6, min_child_weight = 0.1, subsample = 1,\n",
    "                            verbosity = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Created a voting classifier using all of our models. Still doesn't perform as well as svm and log reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Voting Classifier\n",
      "Training Accuracy: 0.9912\n",
      "Testing Accuracy:  0.6828\n",
      "Training f1: 0.9912\n",
      "Testing f1:  0.6564\n"
     ]
    }
   ],
   "source": [
    "voting_clf = VotingClassifier(estimators = [('nb', nb_optimal), ('rf', rf_optimal), ('lr', lr_optimal),('tree', dt_optimal), \n",
    "                                            ('knn', knn_optimal), ('svm', svm_optimal), ('xgb', xgb_optimal)],\n",
    "                              n_jobs = -1)\n",
    "\n",
    "voting_clf.fit(tf_idf_data_train, y_train)\n",
    "voting_train_preds = voting_clf.predict(tf_idf_data_train)\n",
    "voting_test_preds = voting_clf.predict(tf_idf_data_test)\n",
    "\n",
    "voting_train_score = accuracy_score(y_train, voting_train_preds)\n",
    "voting_test_score = accuracy_score(y_test, voting_test_preds)\n",
    "voting_train_f1 = f1_score(y_train, voting_train_preds, average='weighted')\n",
    "voting_test_f1 = f1_score(y_test, voting_test_preds, average = 'weighted')\n",
    "    \n",
    "print('Voting Classifier')\n",
    "print(\"Training Accuracy: {:.4}\".format(voting_train_score))\n",
    "print(\"Testing Accuracy:  {:.4}\".format(voting_test_score))\n",
    "print(\"Training f1: {:.4}\".format(voting_train_f1))\n",
    "print(\"Testing f1:  {:.4}\".format(voting_test_f1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot a confusion matrix of svm, one of our best performing models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'SVM Heatmap')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAEWCAYAAACE4zmnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmAUlEQVR4nO3dd3xUVfrH8c83oQaQXgSpoiIKIqKAoCBVsYJd1NUVsXfWtrqiq7uubd3VnwV1besqYi+gYC+L0gQRy6qIgBSRTqhJnt8f9wbHTDIZktyZCTzv12tec/t5ZhienHvuuefKzHDOuVhZ6Q7AOZd5PDE45+J4YnDOxfHE4JyL44nBORfHE4NzLo4nBudcHE8MlYCk3pL+K2m1pBWSPpa0v6QeknIl1S5mn88kXSipjSST9FmR9Y0kbZY0L0G5Jql9kWWjJf27Aj5TYVxVynssV/E8MWQ4STsBrwH3AA2AFsCNwCYz+wRYCBxXZJ+9gY7A0zGLc8LlhU4BfogwdFeJeWLIfLsDmNnTZpZvZhvMbKKZfR6ufxw4vcg+pwPjzWx5zLIngd8V2eaJ8gYnqYOkSWFN5htJJ8SsOzysuayRtEDS6JhdPwjfV0laJ6mnpDPC2tDfJa2SNFfSgeHyBZJ+lvS7ZI4fUyMZKWmRpMWSRpX38+4wzMxfGfwCdgKWEySAw4D6Rda3BPKAluF8FkEt4phwvg1g4fsCIJugNvE1MACYl6BsA9oXWTYa+Hc4XSs85plAFWBf4BegY7i+L9ApjKkzsLSYuKrEHPuM8LOcGcZ5MzAf+D+gOjAIWAvU3objPx3G2QlYBgxI979pZXh5jSHDmdkaoDfBj/whYJmkVyQ1DdcvAN4DTgt36U/wn+j1IodaCHxDkAxOJ6hBJGNG+Nd7laRVwNUx644gSCyPmlmemX0GPA8cH8b2npnNNrMCC2o4TwN9Sinvh/B4+cBYgsR3k5ltMrOJwGag/TYc/0YzyzWz2cCjwMlJfu4dmieGSsDMvjKzM8xsF2BvoDlwd8wmj/NrYjgNeMbMthRzqCcI/iqfTPKJoauZ1St8AbfGrGsNdC+SOIYDzQAkdZf0rqRlklYD5wKNSilvacz0BgAzK7qs9jYcf0HM9I8E350rhSeGSsbMvgYeI0gQhV4AdpF0CDCMIFEU53ngcGCumc2vgHAWAO/HJg4zq21m54Xr/wO8QnCaUxd4AFDhR6mA8hMdv1DLmOlWwKIKKHe754khw4WNe1dI2iWcb0nwF/+Twm3MLBd4jqCq/KOZTSvuWOF2/YARFRTea8Dukk6TVDV87S9pz3B9HWCFmW2UdADBlZBCy4ACoF05yk90/ELXS8qRtBdB28XYcpS3w/DEkPnWAt2BTyXlEiSEL4Arimz3OEHVPuGVBjObZmbfV0RgZraWoEHwJIK/xEuAvxG0cQCcD9wkaS3wJ+DZmH3XA7cAH4enIT3KEEKJx4/xPvAd8DZwR9hO4UqhsAXXue2KpDYE/TSqmllemsOpdLzG4JyL44nBORfHTyWcc3G8xuCci5Oxd7at3bLKqzIJVM2qlu4QMl6BFaQ7hIyXU6V20X4fgNcYnHPF8MTgnIvjicE5F8cTg3MujicG51wcTwzOuTieGJxzcTwxOOfieGJwzsXxxOCci+OJwTkXxxODcy6OJwbnXBxPDM65OJ4YnHNxPDE45+J4YnDOxfHE4JyL44nBORfHE4NzLo4nBudcHE8Mzrk4nhicc3E8MTjn4nhicM7F8cTgnIvjicE5F8cTg3MujicG51wcTwzOuTieGJxzcaqkO4BMtGnTJs7+3bls2byZ/Px8+g/sxzkXjmTE6SNZn7segBUrVrJXp47c+c/b0xxt+j35+L954bkXkcRuu7fnpltupHr16ukOK61GX3cjH7z/IQ0aNOC5l58F4Juv/8ctN/2FDevX07x5c2657WZq166d5kiLJzNLdwzFWrtlVdoCMzM2bNhATk4OeVvyOOv0kYy6+jI67dNp6zZ/uPQq+hzShyOOHpKWGKtmVUtLuUUtXfozZ5x6Ji+++jw1atTgD5ddSe+De3P00KPSHRoFVpC2sqdPm0FOTk2uv+aGrYlh+AmncdkfLqXb/vvx0gsv89PCn7jg4vPTFiNATpXaKm65n0oUQxI5OTkA5OXlkZeXh/Tr97du3TqmTZlO3/4HpyvEjJKfn8+mjZvIy8tjw8aNNG7SON0hpd1+3bpSt27d3yyb/+OP7NetKwA9enbn7UnvpCO0pESeGCS1ljQgnK4pqU7UZVaE/Px8Tjn2VAYefCjdex7A3p333rruvbc/YP/u3TK2GphKTZs24Xdnns7g/ocxoM9A6tSuzYG9eqY7rIzUrv2uvPfOewBMevMtli5Zmt6AEog0MUg6G3gOeDBctAvwUpRlVpTs7Gz+8/y/Gf/2q8yZPYfvvv1+67qJEyYyeMigNEaXOdasXsO777zH+EmvMem9iWzYsIHXXnk93WFlpNF//hPPPjOOU44fzvr166latWq6QypR1DWGC4BewBoAM/sWaFLSxpJGSpomadqjDz8WcWjJqbNTHbodsB+TP5oMwKqVq5gzew69D+6V5sgywyeTP6VFi+Y0aNCAqlWr0n9gP2bNnJXusDJS23Ztuf+h+/jPuKc4dMhgdmm5S7pDKlHUiWGTmW0unJFUBSixUdHMxphZNzPrduaIMyIOrWQrV6xk7Zq1AGzcuJFPJ0+hTds2ALw18R169+m9w7e6F2q2czM+nzWbDRs2YGZ8+skU2rZrm+6wMtKK5SsAKCgo4KEHH+G4E49Nc0Qli/py5fuSrgVqShoInA+8GnGZ5fbLsl+44Y83UZBfQIEVMHBwfw7q2xuAiRMmccaI09McYebovE8nBg4awEnHnUJ2djYd9uzAcSdk7g8+Va4edS3Tp05j1apVDO53GOdecA4b1q9n7NPjAOg34JCMuHJTkkgvV0rKAs4CBgEC3gQetiQKTeflysogUy5XZrJ0Xq6sLEq6XBl1YhgGvG5mm7Z1X08MiXliKJ0nhtKlqx/DkcD/JD0p6YiwjcE5l+EiTQxmdibQHhgHnAx8L+nhKMt0zpVf5H/BzWyLpAkEVyNqAscAI6Iu1zlXdlF3cDpM0mPAt8CxwMNAsyjLdM6VX9Q1htOBscA5ZWmAdM6lh99dWUn5VYnS+VWJ0pV0VSKSGoOkj8yst6S1/LanowAzs52iKNc5VzEiSQxm1jt8rxR3Ujrnfivqxscnk1nmnMssUXdw2it2JuzgtF/EZTrnyimSxCDpmrB9obOkNeFrLbAUeDmKMp1zFSfqeyX+ambXlGVfvyqRmF+VKJ1flShdWm6iApBUH9gNqFG4zMw+KG0/TwyJeWIonSeG0qX0cmUhSSOASwiGdJsJ9AAmA/2iLNc5Vz5RNz5eAuwP/GhmhwD7AqsiLtM5V05RJ4aNZrYRQFJ1M/sa2CPiMp1z5RT1vRILJdUjGBl6kqSVwI8Rl+mcK6eU3SshqQ9QF3gjdoDYknjjY2Le+Fg6b3wsXboaHxvEzM4O3/0/vHMZLuo2hhnAMuB/BGMyLAPmSZohyXtAOpehok4Mk4AhZtbIzBoChwGvEQwjf1/EZTvnyijqxNDDzN4snDGziUBPM/sE8Ce2OJehor4qsVjSVcAz4fyJwFJJ2YC3DDmXoaKuMZzCrw+yfRFoGS7LBk6IuGznXBml5HKlpFpmlrst+/jlysT8cmXp/HJl6dLywBlJB0r6EvgqnN9Hkjc6Opfhoj6V+DswGFgOYGazgIMjLtM5V05RJwbMbEGRRflRl+mcK5+or0oskHQgYJKqEtxt+VXEZTrnyinqGsO5wAVAC+AnoEs475zLYCVelShyn0McM1sRSUSh3Lw1flUigXtnextuaTo36pDuEDLeYS2P2eabqKYT3PBU3I4GtCtpR0l/SnBcM7M/J1jvnEuzEhODmbUtx3GL67NQCzgLaAh4YnAug5Xa+ChJwHCgrZn9WVIroJmZTSlpHzO7M2b/OgSNjmcSdI2+s6T9nHOZIZnGx/uAngRdmQHWAv9X2k6SGki6GficIAF1NbOrzOznsgbrnEuNZC5XdjezrpI+AzCzlZIS9seVdDswDBgDdDKzdeUP1TmXKsnUGLaEd0MagKTGlH5n5BVAc+A6YFHs06gkrSlXxM65yCVTY/gnwZ2RTSXdAhxH8B++RGYWeY9K51x0Sk0MZvaUpOlA/3DRMWbmvRed244l2yU6h2AMBQNqRheOcy4TlFrlDzsrPQ40ABoBj0pKeCrhnKvckqkxDAf2iXmi1K0Ez6G8OcK4nHNplEwj4SJinlRNMIjrT9GE45zLBCXWGCTdQ9CmsBqYI2lSOD8QKLHXo3Ou8kt0KjEtfJ9OcLmy0HuRReOcywiJbqJ6PJWBOOcyRzI3Ue0G/BXoSExbg5mVeNu1c65yS6bx8VHgfiAPOAR4Avh3lEE559IrmcRQ08zeJhjt6UczGw0cHm1Yzrl0SqYfwyZJWcC3ki4kuFRZO9qwnHPplEyN4RKCLtEXA/sBpwKnRxmUcy69krmJamo4uY5gFCYk3QF8GmFczrk0Kuvt0f5AWue2Y2VNDMUOOe2c2z4k6hJd0nMlhCcG57ZrZX2uxOZownHOZYKonivhnKvEfGxG51wcTwzOuTieGJxzccpyVQKI/mnXzrn0SfaqRCtgZThdD5gPeOOkc9upEk8lzKxtOObCW8CRZtbIzBoCRwATUxWgcy71krm7soeZnV04Y2YTJN0WYUxpt2TxEv50zWiWL1+BBMOOH8opp53MN199wy033crmTZvIrlKFa667ir0775XucFMib3Me42+cQP6WfKzAaNO9DV2P3xczY/rYGcz7ZB7KEh0GdmCvwzoCsHjOYj59YgoF+QXUqFOdITcMSfOniNbKn1fx1N/GsnblOiToeXh3+gzrzYTHJ/HJ+CnUqlcLgCN+fygdu3cAYNHcxYz9+wtsWr8RKYvL77uQqtWqpvNjAMklhkXhcyQKB2cZTjBy9HYru0oVLrvyUvbs2IHc3FyGH386PXp25x933cM554+g10G9+OiDj/nHXf/kocceTHe4KZFdNZvDrj+UqjWqUpBXwGs3vM4uXVqw6qfV5C7P5di7hqEssWH1BgA25W5i8r8mM+iaQdRuVHvr8u1ZVnYWR597BC13a8HG9Zu487x/ssd+uwHQ59je9Duhz2+2z8/P58m/PsOpV59Ii12bk7s6l+zs7HSEHieZqxInA40JBoR9IZw+Ocqg0q1x40bs2THI6LVq1aJtuzb8/PMyQKxblwvAurXraNy4cRqjTC1JVK0R/CUryC/A8gsA8fWkr9n32C4oK+ggW7Nu8KCyuR/PpfUBrandqPZvlm/P6jbciZa7tQCgRk51mrZqwupfVpe4/TfTvqV5u51psWtzAGrVrUVWdmZcKEzmtusVwCWSaplZ7rYcXNLuBMPCNTWzvSV1Bo4ys0rzsJpFPy3im6++Ye/OezHq6su5cORF3H3HPygoMB596pF0h5dSBQUFvHLNq6xZsoY9B3WgyW6NWbt0LXMn/8CPU3+kRp0a9DijO3V3rsvqxWsoyC9g/I0T2LJxCx0P68huB7dP90dImeVLVrDwu59o3aEVc7/4kQ9fnszUSTNoufsuHHPu4eTUyeHnhcuQ4P6rHiZ3dS77HrIP/U/sm+7QgeQeUXegpC+Br8L5fSTdl+TxHwKuAbYAmNnnwEkJyhopaZqkaf966NEki4jO+tz1jLr0Kq64+nJq167Nc2Of54qrLmfC269zxVWXcdP1f053iCmVlZXFMX87mhPvO4Fl3//CygUryd+ST3bVbI7+y1Hs0X93PnrgYwAsv4Dlc5cz8KoBDL5mELNemMnqRSX/9dyebNqwiUdv/DdDzz+KGrVq0PuoHlz/xJX84cFLqNuwDi898DoQ1LzmfjGP0649mYvvPo/PP5rD/2Z8l+boA8nUW/4ODAaWA5jZLODgJI+fY2ZFH06TV9LGZjbGzLqZWbffn31mkkVEY8uWPEZdehVDDj+U/gP7AfDay6/Rb+AhAAwcPIA5s79MZ4hpU71WdXbea2cWzlxIrYa1aHNAawBa79+aFfOD7i05DWvRYp8WVK1RlRo71aBph2Zb123P8vPy+dfoJ9mvfxf2OWhvAOrUr0NWdhZZWVn0GHIA879ZAEC9xnXZtVNbatetRbUa1ejYfQ8WfpsZD3lL6oTGzBYUWZSf5PF/kbQrQX8IJB0HLE4+vPQwM276059p264Np54xfOvyRk0aM33qDACmfDqVlq1bpivElNuwZiObcjcBwRWKRZ8vom7zerTq1orFc4J/0iVfLqHuznUBaN2tFUu/XkpBfgF5m/JY9t0y6rWol67wU8LMePqO52jaugmHHPfr387Vy9dsnZ790Rx2btMUgA7ddmfxD0vYvHEz+fn5fD/rB5q2bpLyuIuTzFWJBZIOBExSVYIxIL9K8vgXAGOADpJ+An4guKqR0WbOmMXrr4yn/e7tOWnYKQBceOkFXD/6j9x+653k5+VTvXo1rht9bZojTZ0NK9fzwf0fYgWGFRhte7al1X4tadqhCe/f+wFzxs+hSo2q9DqnFwD1WtRjly4teOnKl0Bij367U79l/fR+iIj98MU8pr01g53bNuO2c+4GgkuTM96dyU/fLQZBg2b1OeHSYQDk1Mmh73EHcdcF94BExwM6sFePPdP4CX4lM0u8gdQI+AcwgKDn40Tg4mS6REvKNrN8SbWALDNbm2xguXlrEge2g7t3drLNPDuuzo06pDuEjHdYy2OKHXQpmRrDHmb2m7/yknoBHyex7w+S3gDGAu8ksb1zLgMk08ZwT5LLitOBoEv1BQRJ4l5JvZMNzjmXHonuruwJHAg0lnR5zKqdgKS6Z5nZeuBZ4FlJ9QlOSd5Pdn/nXHokqjFUI3jiVBWgTsxrDXBcsgVI6hP2e5hO8FBcH3reuQyXaMzH94H3JT1mZj+W5eCS5gGfEdQa/rCtPSedc+mRTBvDw5LqFc5Iqi/pzSSP39nMhprZ054UnKs8krkq0cjMVhXOmNlKSQl7YUi60sxuA26RFHfZ0cwu3uZInXMpk0xiKJDUyszmA0hqTdiTMYHCDlDTyhOccy49kkkMfwQ+kvQ+QQeng4CRiXYws1fDyfVmNi52naTjyxKocy51Sm1jMLM3gK4EnZSeAfYzs2TbGK5JcplzLoMk6sfQwcy+ltQ1XFQ4alOr8NRiRoJ9DwOGAC0k/TNm1U4kuLvSOZcZEp1KXAGcDdxZzDoD+iXYdxFB+8JRBP0XCq0FLtvGGJ1zKZaoH8PZ4fsh23rQcMyGWZKeMjOvIThXySQ6lRiWaEczeyHBvs+a2QnAZ0UuVyrY1Tpvc6TOuZRJdCpxZPjehOCeicK7Iw8B/kswMGxJLgnfjyhXdM65tEh0KnEmgKSJQEczWxzO7ww8luighdsCvwAbzKwgHBi2AzChAuJ2zkUomS7RLWP+owMsJXhkXTI+AGpIakEwwMtplJJUnHPpl0wHp7fDeyOeDudPJBhjIRkys/WSzgLuM7PbJM0sQ5zOuRRK5rkSF0oayq8jQ48xsxeTPL7CcR2GA2eFy3wsBucyXDI1BoAZwFoze0tSjqQ6SY7feClBT8cXzWyOpHbAu2WM1TmXIqUmBklnE9wb0QDYFWgBPAD0L23fmDEdakuqbWZzAb+z0rkMl0zj4wVAL4KRmzCzbwkuYZZKUidJnwFzgC8lTZe0Yzwe2rlKLJnEsMnMNhfOSKpC6bddF3oQuNzMWptZK4Ju1g9te5jOuVRKJjG8L+laoKakgcA44NVS9ilUy8y2timY2XtArW2O0jmXUskkhquAZcBs4BxgPHBdksefK+l6SW3C13XA3LKF6pxLlYSNj5KygTlm1oGynQL8HriRoPu0AR+Gy5xzGSxhYggfL/dN7NBuyZBUAzgXaE9Q07jCzLaUL1TnXKok04+hPjBH0hRg60jPZnZUgn0eB7YQ1BAOA/Yk6NPgnKsEkkkM15fhuB3NrBOApEeAKWU4hnMuTRKNx1D0dOCRbRh0Zetpg5nlScU+UNc5l6ES1RiKng505NdxFkqzj6Q14bQILnWu4deBWnYqY7zOuRSQWfF9lSTNjjkdqAJMMbOuxW4cgVWbf0m2E9UOadbyEsfidaG+p/sFsNLYpIXFVucT9WP4zelAhUfknMtYiU4l/HTAuR1UoqHdfNwE53ZQyXSJds7tYDwxOOfieGJwzsXxxOCci+OJwTkXxxODcy6OJwbnXBxPDM65OJ4YnHNxPDE45+J4YnDOxfHE4JyL44nBORfHE4NzLo4nBudcHE8Mzrk4nhicc3E8MTjn4nhicM7F8cTgnIvjicE5F8cTg3MujicG51wcTwzOuTieGJxzcTwxOOfieGJwzsXxxOCci5Poadc7tGMGH0tOTg5Z2VlkZ2fz+Nh/AfDsU+N47pkXyMrOotfBB3LR5RekOdLUWPHzSv71lydZu3ItCA4+ohf9j+vLy4+8xsyPZyOJOvXrcObVp1KvUV0+nTSVN55+CzOjRk4Nhl92Ai3b75LujxG5S4eNYMRhJ2NmzJ73NWfefgUjhpzMpUNH0L5FGxod24nla1YCMOr4cxnefygAVbKy2bPVbjQ+fh9Wrl2Vxk8QkJmlO4Zirdr8S1oDO2bwsTz2zCPUq19v67JpU6bz2JgnuOu+26lWrRorlq+kQcP6aYlv1vIZKS1v1fLVrF6+hta7t2Tj+o3cPPI2zr/5bOo3rkfNWjUBePv591g8bwmnXnES338xl2atm1GrTg6zP53Dq49N4Nr7R6U05r6n/z6l5TVv2IyP/v4CHUf0Y+PmjYy97n7GT3mHWd9/ycp1q3nvjnF0u2DI1sQQ64geA7hs2Nn0v/LElMZskxaquOVeY9gGL4x9idPPOpVq1aoBpC0ppEO9hnWp17AuADVyarBz62as+mU1zdvsvHWbzRs3IwW/s133brd1ebuObVm1bFVK402XKtlVqFm9BlvytpBTvSaLli9l5vdzSt3v5EOO4el3X05BhMmJJDFIapBovZmtiKLcCiVx8TmXAWLo8Ucz9Pijmf/jfGbOmMUD94yhWrVqXDzqQjruvWe6I025XxYvZ/63C2m7Z2sAXnz4VT55cwo1a9Xkirsvitv+49cns/cBHVMdZsotWr6EO557kPlPfcqGTRuZOP0DJk3/oNT9alavwaHd+nLhvdelIMrkRFVjmA4YUFw1xYB2xSzPKGMev58mTRuzYvlKLhp5KW3atiY/P581q9fwyFNj+PKLr7h21PW8OGHc1r+SO4KN6zfxwA2PcOKFw7aeQgwdcSRDRxzJhKcm8u6LH3DUmYdv3f7rz/7HR+Mnc+U9l6Ur5JSpV7suR/ccRNvTerJq3RrGXf8Aw/sP46m3X0i435E9BvLxnKkZ0bZQKJKrEmbW1szahe9FXyUmBUkjJU2TNO2xh5+IIrSkNWnaGAhOF/r2P5g5X3xJk6ZN6DugD5LYq1NHsiRWrVyV1jhTKS8vnwdueJjuA7rR9eAucesPGNCNGe/P2jq/8PufeOL2p7nglpHUrlsrhZGmx4CuvflhyQJ+Wb2CvPw8XvhoAgd23K/U/U7qe3RGnUZACi5XSqov6QBJBxe+StrWzMaYWTcz63bGiNOjDq1EG9ZvIDc3d+v0p/+dwq7t29Gn30FMnxI0+s2fN58tW/J+0zi5PTMznrjtKXZu1YyBJ/Tbunzpwp+3Ts/6eDbNWjUFYPnSFdx//cOcde1pNG3ZJOXxpsP8nxfRY899qVm9BgD99+3NV/O/S7jPTjl16NO5By9PfjMVISYt0sZHSSOAS4BdgJlAD2Ay0C/Bbmm3YvkKrrz0WgDy8/MYPGQQPXv3YMuWLdx8/V84eeipVK1alRtuuW6HOY34bvZcPpk4lRbtmnPTWbcCMPTsI/lo/GSWzv8ZZYmGTRsw/PKgVf31x98gd00uT/39WQCys7P445gr0xZ/Kkz5+jOe+3A8M+57g7z8PD77fg5jxj/FRcf8nitPOI9mDRrz+ZhJjJ/yLmff9QcAhvY+lInT32f9xg1pjv63Ir1cKWk2sD/wiZl1kdQB+IuZDStt33Rfrsx0qb5cWRml+nJlZVTS5cqoTyU2mtlGAEnVzexrYI+Iy3TOlVPU/RgWSqoHvARMkrQS+DHiMp1z5RRpYjCzoeHkaEnvAnWBN6Is0zlXfpElBknZwBwz6wBgZu9HVZZzrmJF1sZgZvnAN5JaRVWGcy4aUbcx1AfmSJoC5BYuNLOjIi7XOVcOUSeG6yM+vnMuAlEnhiFmdlXsAkl/A7y9wbkMFnU/hoHFLDss4jKdc+UU1W3X5wHnA7tK+jxmVR3gv1GU6ZyrOFGdSvwHmAD8Fbg6ZvnaSjEWg3M7uEgSg5mtBlZLuqrIqtqSapvZ/CjKdc5VjKgbH1/n1wFbagBtgW+AvSIu1zlXDlF3ie4UOy+pK0Hbg3Mug6X0uRJmNgPonsoynXPbLuqBWi6Pmc0CugKLoizTOVd+Ubcx1ImZziNoc3g+4jKdc+UUdRvDjQCScsxsfZRlOecqTqRtDJJ6SvoS+Dqc30fSfVGW6Zwrv6gbH+8GBgPLAcxsFlDiKNHOucwQ+VUJM1tQZFF+1GU658on6sbHBZIOBExSVYKh5L+KuEznXDlFXWM4F7gAaAH8BHQJ551zGSzqqxK/AMOjLMM5V/Giuu36TwlWm5n9OYpynXMVI6oaQ24xy2oBZwENAU8MzmWwqG67vrNwWlIdgkbHM4FngDtL2s85lxmifK5EA+BygjaGx4GuZrYyqvKccxUnqjaG24FhwBigk5mti6Ic51w0orpceQXQHLgOWCRpTfhaK2lNRGU65ypIVG0MKR3nwTlXsfw/sHMujsws3TFUCpJGmtmYdMeRyfw7SqwyfT9eY0jeyHQHUAn4d5RYpfl+PDE45+J4YnDOxfHEkLxKcW6YZv4dJVZpvh9vfHTOxfEag3MujicG51yc7TIxSDJJsXd4jpI0OoJyri0y/9+KLiMVJOVLminpC0njJOVs4/7NJT0XTneRNCRm3VGSri5578qhIn9TkupJKtOjGiXNk9SoLPtui+0yMQCbgGEp+AJ/kxjM7MCIy4vKBjPrYmZ7A5sJhuRLmpktMrPjwtkuwJCYda+Y2a0VFmn6VORvqh4lPMNVUtTjsCZle00MeQQtwJcVXSGpsaTnJU0NX71ilk+SNEfSw5J+LPwRSHpJ0vRw3chw2a1AzfAv7VPhsnXh+zOSDo8p8zFJx0nKlnR7WO7nks6J/JvYdh8C7SU1CD/355I+kdQZQFKf8DPPlPSZpDqS2oS1jWrATcCJ4foTJZ0h6V5JdcPvNCs8Ti1JCyRVlbSrpDfC7/hDSR3S+PlLUpbf1GhJo2K2+0JSG+BWYNfwO7pdUt/wc78CfBluG/ebSykz2+5ewDpgJ2AeUBcYBYwO1/0H6B1OtwK+CqfvBa4Jpw8FDGgUzjcI32sCXwANC8spWm74PhR4PJyuBiwI9x0JXBcurw5MA9pmwvcVvlcBXgbOA+4BbgiX9wNmhtOvAr3C6drhPm2AL8JlZwD3xhx763x47EPC6ROBh8Ppt4HdwunuwDvp/k4q6Dc1GhgVc4wvwu9q6/cVLu9LMOpZ25hlJf3m5hX+LqN8ZUS1JQpmtkbSE8DFwIaYVQOAjpIK53eSVBvoTfAfGjN7Q1LsoDIXSxoaTrcEdiN8iE4JJgD/kFSdIMl8YGYbJA0COksqrHbXDY/1Q1k/ZwWpKWlmOP0h8AjwKXAsgJm9I6mhpJ2Aj4G7wlrSC2a2MOa7LM1YgoTwLnAScF/43R8IjIs5TvXyf6SKV4bf1LaYYmaxv4Nt/c1VqO02MYTuBmYAj8YsywJ6mNnG2A1L+nFL6kvwD9/TzNZLeg+okahQM9sYbjeY4D/CM4WHAy4ysze37WNEboOZdYldUNL3YWa3SnqdoB3hY0mDgY3FbhzvFeAvCkb32g94h2As0FVFy89gd5P8byqP356uJ/rdbB0ntSy/uYq2vbYxAGBmK4BnCQahLTQRuKhwRlKXcPJj4IRw2SCgfri8LrAy/AfqAPSIOdYWBQ/SKc5YgnEuDwLeCJe9CZxXuI+k3SXVKtuni9yHhEP/hz/UX8K/mLua2Wwz+xswFSjaHrCW3z7lfCsLRvKaCvwDeM3M8s1sDfCDpOPDsiRpnyg+UEXYxt/UPKBruKwr0DZcXuJ3FEr0m0uJ7ToxhO4EYluSLwa6hY1qX/JrC/yNwCBJXwDHA0sI/gHfAKpI+oqg0eiTmGONAT4vbHwsYiLQB3jLzDaHyx4maFyaEZbzIJlbaxsN7Cfpc4LP/btw+aVhI9rnwBaC06ZY7xJUq2dKOrGY444FTg3fCw0HzpI0C5gDHF1xHyMSyf6mngcaSJoDXAj8D8DMlhPUtr5QMAxiUYl+cynhXaJDYXtAvpnlSeoJ3F+JqrfOVahM/WuVDq2AZ8PLaZuBs9Mcj3Np4zUG51ycHaGNwTm3jTwxOOfieGJwzsXxxLADCHstFt7fsETSTzHz1SqojPckdUty276SXovq+K78/KrEDiC8bt4Fght7CO6NuKNwvaQqZpaXnuhcJvIaww5KwR2fD0j6FLgtwZ2ASDpV0pSwhvGgpOwky2gT3jU4I3zF3pa+k6TXJX0TxlF41+UgSZPD7ceV4Z4DVwE8MezYdgEONLPLS9pA0p4E93v0Cjt85RN2lU7Cz8BAM+saHuOfMesOIOhG3BHYlV/HOrgOGBDuM43giekuxfxUYsc2zszyS9mmP8ENT1PDG6tqEvyHT0ZV4N7w3oF8YPeYdVPMbC6ApKcJ7m7dSJAoPg7LqgZMTrIsV4E8MezYcmOmS7oTUARjS1xThuNfBiwF9gmPHXv3YdGedRaWNcnMTi5DWa4C+amEKzSP4u8EfBs4TlKTcF0DSa2TPGZdYLGZFQCnAbFtEwdIahu2LZwIfERws1AvSe3DsmpJ2r3oQV30PDG4QiXdCfglwXn/xPCOyknAziUc43VJC8PXOOA+4HfhXZMd+G0NZSrBqFlfEQxU86KZLSMY8enpsKzJxN/W7VLA75VwzsXxGoNzLo4nBudcHE8Mzrk4nhicc3E8MTjn4nhicM7F8cTgnIvz/2XalQiugk3eAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "matrix = confusion_matrix(y_test, svm_grid_preds)\n",
    "sns.heatmap(matrix.T, square=True, annot=True, fmt='d', cbar=False,\n",
    "            xticklabels=['Negative', 'Positive', 'Neutral'], yticklabels=['Negative', 'Positive', 'Neutral'], cmap = 'Greens')\n",
    "plt.xlabel('True Label')\n",
    "plt.ylabel('Predicted Label')\n",
    "plt.title('SVM Heatmap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}