Text Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'wordcloud'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-224cae8d7e5a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mwordcloud\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mWordCloud\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'wordcloud'"
     ]
    }
   ],
   "source": [
    "# data manipulation\n",
    "import pandas as pd\n",
    "\n",
    "# numpy arrays\n",
    "import numpy as np\n",
    "\n",
    "# data visualization\n",
    "import seaborn as sns\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import plotly\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.figure_factory as ff\n",
    "\n",
    "sns.set()\n",
    "\n",
    "# NLP\n",
    "import string\n",
    "\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "import nltk\n",
    "from nltk.probability import FreqDist\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from vectorizers.NLTKVectorizer import NLTKVectorizer\n",
    "\n",
    "import re\n",
    "\n",
    "# machine learning\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression  # Logistic Regression\n",
    "from sklearn.naive_bayes import MultinomialNB  # Naive Bayes\n",
    "from sklearn.svm import LinearSVC  # SVM\n",
    "from sklearn.ensemble import RandomForestClassifier  # Random Forest\n",
    "\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "\n",
    "# Model explainability\n",
    "from lime.lime_text import LimeTextExplainer\n",
    "\n",
    "# other\n",
    "from pprint import pprint\n",
    "from time import time\n",
    "import logging\n",
    "from functools import partial\n",
    "import joblib\n",
    "\n",
    "en_stop_words = list(set(stopwords.words(\"english\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Color palette generator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "This function is used to create a *palette* of `n` colors of `palette_name` colors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def get_n_color_palette(palette_name, n_colors, as_hex=False):\n",
    "    palette = sns.color_palette(palette=palette_name, n_colors=n_colors)\n",
    "    if as_hex:\n",
    "        palette = palette.as_hex()\n",
    "    palette.reverse()\n",
    "    return palette"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Plotly export chart"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "This function is used to export the HTML of plotly `fig_obj`, and save it in: `assets/file_name.html`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def save_fig_as_div(fig_obj, file_name):\n",
    "    with open(f\"assets/{file_name}\", \"w\") as fig_file:\n",
    "        fig_div_string = plotly.offline.plot(\n",
    "            figure_or_data=fig_obj, output_type=\"div\", include_plotlyjs=\"cdn\"\n",
    "        )\n",
    "        fig_file.write(fig_div_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Classification report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "This function generates the classification report for the model predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def get_classification_report(y_true, y_pred, target_names):\n",
    "\n",
    "    # calculate classification report, and convert it to DataFrame\n",
    "    clf_report = classification_report(\n",
    "        y_true=y_true, y_pred=y_pred, target_names=target_names, output_dict=True\n",
    "    )\n",
    "    clf_report_df = pd.DataFrame(data=clf_report)\n",
    "    clf_report_df = clf_report_df.T\n",
    "    clf_report_df.drop(columns=[\"support\"], inplace=True)\n",
    "\n",
    "    measures = clf_report_df.columns.tolist()\n",
    "    classes = clf_report_df.index.tolist()\n",
    "\n",
    "    # create plotly annotated heatmap, and update styling\n",
    "    fig = ff.create_annotated_heatmap(clf_report_df.values, x=measures, y=classes)\n",
    "    fig.update_layout(\n",
    "        autosize=False,\n",
    "        width=800,\n",
    "        height=800,\n",
    "        title_text=\"<i><b>Classification report</b></i>\",\n",
    "        xaxis_title=\"Measures\",\n",
    "        yaxis_title=\"Class\",\n",
    "        plot_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "        paper_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "        font={\n",
    "            \"family\": \"Courier New, monospace\",\n",
    "            \"size\": 14,\n",
    "            # 'color': \"#eaeaea\"\n",
    "        },\n",
    "    )\n",
    "    fig.update_xaxes(tickangle=-45)\n",
    "    fig[\"data\"][0][\"showscale\"] = True\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "This function generates the confusion matrix for the model predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def get_confusion_matrix(y_true, y_pred, labels):\n",
    "\n",
    "    # claculate confusion matrix\n",
    "    conf_matrix = confusion_matrix(y_true=y_true, y_pred=y_pred, labels=labels)\n",
    "    conf_matrix = np.flipud(conf_matrix)\n",
    "\n",
    "    # create annotated heat map of the confusion matrix\n",
    "    fig = ff.create_annotated_heatmap(\n",
    "        conf_matrix, x=labels.tolist(), y=labels.tolist()[::-1]\n",
    "    )\n",
    "    fig.update_layout(\n",
    "        autosize=False,\n",
    "        width=800,\n",
    "        height=800,\n",
    "        title_text=\"<i><b>Confusion matrix</b></i>\",\n",
    "        xaxis_title=\"Predicted category\",\n",
    "        yaxis_title=\"Real category\",\n",
    "        plot_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "        paper_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "        font={\n",
    "            \"family\": \"Courier New, monospace\",\n",
    "            \"size\": 14,\n",
    "            # 'color': \"#eaeaea\"\n",
    "        },\n",
    "    )\n",
    "    fig.update_xaxes(tickangle=-45)\n",
    "    fig[\"data\"][0][\"showscale\"] = True\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "newsgroups_data = fetch_20newsgroups(subset=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "categories = newsgroups_data.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(\n",
    "    data={\"text\": newsgroups_data.data, \"category\": newsgroups_data.target}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "df[\"category\"] = df[\"category\"].apply(lambda x: categories[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "The dataset consists of 18,846 samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What is this Data?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "The 20 Newsgroups data set is a collection of approximately 20,000 newsgroup documents, partitioned (nearly) evenly across 20 different newsgroups."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "The data is organized into 20 different newsgroups, each corresponding to a different topic. Some of the newsgroups are very closely related to each other (e.g. `comp.sys.ibm.pc.hardware` / `comp.sys.mac.hardware`), while others are highly unrelated (e.g `misc.forsale` / `soc.religion.christian`).\n",
    "\n",
    "Here is a list of the 20 newsgroups, partitioned (more or less) according to subject matter:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "<table style='font-family:\"Courier New\", Courier, monospace; font-size:80%'>\n",
    "    <tr>\n",
    "        <td>comp.graphics<br>comp.os.ms-windows.misc<br>comp.sys.ibm.pc.hardware<br>comp.sys.mac.hardware<br>comp.windows.x\n",
    "        </td>\n",
    "        <td>rec.autos<br>rec.motorcycles<br>rec.sport.baseball<br>rec.sport.hockey</td>\n",
    "        <td>sci.crypt<br>sci.electronics<br>sci.med<br>sci.space</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "        <td>misc.forsale</td>\n",
    "        <td>talk.politics.misc<br>talk.politics.guns<br>talk.politics.mideast</td>\n",
    "        <td>talk.religion.misc<br>alt.atheism<br>soc.religion.christian</td>\n",
    "    </tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "One quick *data-cleaning* step is to rename the category columns to something more readable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "df = df.assign(category=df[\"category\"].apply(lambda x: \"_\".join(x.split(\".\")[1:])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "df[\"category\"].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Calculate for each category in the dataset the number of articles, the average article length, the max article length, and the min article length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "categories_statistics_df = (\n",
    "    df.groupby(by=\"category\")[\"text\"]\n",
    "    .agg(\n",
    "        [\n",
    "            (\"count\", lambda x: x.size),\n",
    "            (\"mean\", lambda x: x.str.len().mean()),\n",
    "            (\"max\", lambda x: x.str.len().max()),\n",
    "            (\"min\", lambda x: x.str.len().min()),\n",
    "        ]\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=\"count\", ascending=False)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Categories article count:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Use a pie chart to the percentages of articles for each category:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "blue_palette = get_n_color_palette(\"Blues\", 20, True)\n",
    "\n",
    "fig = px.pie(\n",
    "    data_frame=categories_statistics_df,\n",
    "    names=\"category\",\n",
    "    values=\"count\",\n",
    "    color_discrete_sequence=blue_palette,\n",
    "    title=\"Categories Percentages\",\n",
    "    width=800,\n",
    "    height=500,\n",
    ")\n",
    "\n",
    "fig.update_layout(\n",
    "    {\n",
    "        \"plot_bgcolor\": \"rgba(0, 0, 0, 0)\",\n",
    "        \"paper_bgcolor\": \"rgba(0, 0, 0, 0)\",\n",
    "        \"font\": {\n",
    "            \"family\": \"Courier New, monospace\",\n",
    "            \"size\": 14,\n",
    "            # 'color': \"#eaeaea\"\n",
    "        },\n",
    "    }\n",
    ")\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "We can see that the dataset is *balanced*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "save_fig_as_div(fig, file_name=\"charts/categories-percentages-pie-chart.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Categories average article length:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Use a bar chart to show the average article length per category:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "chart_labels = {\"mean\": \"Average Article Length\", \"category\": \"Category Name\"}\n",
    "\n",
    "fig = px.bar(\n",
    "    data_frame=categories_statistics_df.sort_values(by=\"mean\"),\n",
    "    x=\"category\",\n",
    "    y=\"mean\",\n",
    "    color=\"mean\",\n",
    "    labels=chart_labels,\n",
    "    title=\"Average Article Length by Category\",\n",
    "    width=800,\n",
    "    height=500,\n",
    ")\n",
    "\n",
    "fig.update_layout(\n",
    "    {\n",
    "        \"plot_bgcolor\": \"rgba(0, 0, 0, 0)\",\n",
    "        \"paper_bgcolor\": \"rgba(0, 0, 0, 0)\",\n",
    "        \"font\": {\n",
    "            \"family\": \"Courier New, monospace\",\n",
    "            \"size\": 14,\n",
    "            # 'color': \"#eaeaea\"\n",
    "        },\n",
    "    }\n",
    ")\n",
    "\n",
    "# rotate x-axis ticks\n",
    "fig.update_xaxes(tickangle=-45)\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "This chart shows how lengthy the *polictics* articles, compared to *computer* articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "save_fig_as_div(fig, file_name=\"charts/average-article-length-bar-chart.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "categories_text_df = df.groupby(by=\"category\").agg({\"text\": \" \".join}).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "categories_text_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def plot_word_cloud(category_name, category_text):\n",
    "    plt.subplots(figsize=(8, 8))\n",
    "    wc = WordCloud(\n",
    "        background_color=\"white\", stopwords=en_stop_words, width=1000, height=600\n",
    "    )\n",
    "    wc.generate(category_text)\n",
    "    plt.title(label=category_name)\n",
    "    plt.axis(\"off\")\n",
    "    plt.imshow(wc, interpolation=\"bilinear\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "The following word-clouds will help us to take a glimpse on the data, and its content.\n",
    "\n",
    "Each word cloud shows the relative frequency of words in a category, words with higher *frequencies* have bigger *size*.\n",
    "\n",
    "This will help us to see what are the **dominant** words in each category."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "for idx, row in categories_text_df.iterrows():\n",
    "    plot_word_cloud(row[\"category\"], row[\"text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Split the data into 75% training and 25% testing, with stratified sampling, to make sure that the class labels percentages in both training and testing data is (nearly) equal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "X = df[\"text\"]\n",
    "y = df[\"category\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Vectorization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "In this step, we'll build a **text vectorization** transformer which will be used to convert the raw text documents into appropriate features, prepared to be input for machine learning algorithms."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "I will define a *custom vectorizer* called `NLTKVectorizer`. This vectorizer inherits the `TfidfVectorizer` and overrides the `build_analyzer` method. The resulting vectorizer will have the same parameters as the `TfidfVectorizer` but it will *analyze* the documents in a different manner, mainly it will use `NLTK` tokenzier, lemmatizer."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Use a very minimalistic Pipeline consisting of only two stpes:\n",
    "- Text Vectorizer (Transformer): in this step the vectorizer takes the raw text input, perform some data cleaning, text representation (using TF-IDF), and returns an array of features for each sample in the dataset.\n",
    "- Classifier (estimator): the classifier then takes the output produced by the previous step (which is features matrix), and use it as input to traing machine learning algorithm to learn from the data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Why pipelines?\n",
    "\n",
    "Pieplines are the recommended approach of combining data processing steps with machine learning stpes, they let use *seamlessly* apply many transformations on the input data, and finally train a model on the produced data.\n",
    "\n",
    "They are considered as one of the best-practises in `sklearn`.\n",
    "\n",
    "Here, we are creating a two-step pipeline, which may doesn't show how important it is, but it's very common to create a very complex pipline consisting of many data transformations steps followed by a machine learning model.\n",
    "\n",
    "Another bonus of pieplines, is that they can help us perform cross validation on the data all at once, and tune the hyper-parameters.\n",
    "\n",
    "There are many great resources about the importance of using pipelines, and how to use them:\n",
    "\n",
    "- [Deploying Machine Learning using sklearn pipelines](https://www.youtube.com/watch?v=URdnFlZnlaE)\n",
    "- [A Simple Example of Pipeline in Machine Learning with Scikit-learn](https://towardsdatascience.com/a-simple-example-of-pipeline-in-machine-learning-with-scikit-learn-e726ffbb6976)\n",
    "- [A Deep Dive Into Sklearn Pipelines](https://www.kaggle.com/baghern/a-deep-dive-into-sklearn-pipelines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# text vectorizer\n",
    "vectorizer = NLTKVectorizer(\n",
    "    stop_words=en_stop_words, max_df=0.5, min_df=10, max_features=10000\n",
    ")\n",
    "\n",
    "# Logistic Regression classifier\n",
    "lr_clf = LogisticRegression(C=1.0, solver=\"newton-cg\", multi_class=\"multinomial\")\n",
    "\n",
    "# Naive Bayes classifier\n",
    "nb_clf = MultinomialNB(alpha=0.01)\n",
    "\n",
    "# SVM classifier\n",
    "svm_clf = LinearSVC(C=1.0)\n",
    "\n",
    "# Random Forest classifier\n",
    "random_forest_clf = RandomForestClassifier(\n",
    "    n_estimators=100, criterion=\"gini\", max_depth=50, random_state=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([(\"vect\", vectorizer), (\"clf\", lr_clf)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyper-parameters tunning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "**NOTE**: only perform the `GridSearch` when you want to find the best parameters, once the best parameters are found, use them to create the best piepline."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Choosing the right parameters of our text vectorizer and classifier is a troublesome, there are many parameters, we need to assign a value for each parameter, train the model, and see the accuracy of our model.\n",
    "\n",
    "This trail-and-error approach could be summarized as the following:\n",
    "- select one parameter of the hyper-parameters at a time.\n",
    "- assign this parameter an appropriate value (depending on this parameter represent).\n",
    "- train the model.\n",
    "- compare the accuracy\n",
    "- repeat for another parameter.\n",
    "\n",
    "Luckily, `sklearn` supports tunning the hyper-parameters in a much nicer way using its `GridSearchCV` class, which performs **Exhaustive** search over the specified parameters settings, and beside searching for the best parameters, it optimize the results by cross validating the model, and reporting the average model across the splits.\n",
    "\n",
    "`sklearn` supports another approach for tuning the hyper-parameters, they are explaind in details in the [documentation](https://scikit-learn.org/stable/modules/grid_search.html) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "The parameter settings to tune:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# parameters = {\n",
    "#     # vectorizer hyper-parameters\n",
    "#     'vect__ngram_range': [(1, 1), (1, 2), (1, 3)],\n",
    "#     'vect__max_df': [0.4, 0.5, 0.6],\n",
    "#     'vect__min_df': [10, 50, 100],\n",
    "#     'vect__max_features': [5000, 10000],\n",
    "#     # classifiers\n",
    "#     'clf': [lr_clf, nb_clf, svm_clf, random_forest_clf]\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, refit=True)\n",
    "\n",
    "# print(\"Performing grid search...\")\n",
    "# print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "# print(\"parameters:\")\n",
    "# pprint(parameters)\n",
    "# t0 = time()\n",
    "# grid_search.fit(X_train, y_train)\n",
    "# print(\"done in %0.3fs\\n\" % (time() - t0))\n",
    "\n",
    "# print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "# print(\"Best parameters set:\")\n",
    "# best_parameters = grid_search.best_estimator_.get_params()\n",
    "# for param_name in sorted(parameters.keys()):\n",
    "#     print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## What are the best vectorizer and classifier params:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# grid_search.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Pipeline best params:\n",
    "\n",
    "{'clf': LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
    "       intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
    "       multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
    "       verbose=0),\n",
    "'vect__max_df': 0.5,\n",
    "'vect__max_features': 10000,\n",
    "'vect__min_df': 10,\n",
    "'vect__ngram_range': (1, 1)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "the best parameters reported after performing grid search:\n",
    "- `clf`: the classifier which achieved the highest score is the `LinearSVC`.\n",
    "- `max_df`: terms that have a document frequency higher than 50% are ignored.\n",
    "- `min_df`: terms that have document frequency strictly lower than 10 are ignored.\n",
    "- `ngram_range`: using only unigrams.\n",
    "- `max_features`: selecting only 10,000 features."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## What is the best estimator:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Best classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Create a pipeline with the best parameters found by the grid search:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "vectorizer = NLTKVectorizer(\n",
    "    max_df=0.5,\n",
    "    min_df=10,\n",
    "    ngram_range=(1, 1),\n",
    "    max_features=10000,\n",
    "    stop_words=en_stop_words,\n",
    ")\n",
    "\n",
    "svm_clf = LinearSVC(C=1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "The `LinearSVC` class, has only `predict` method, while other classifiers has the `predict_proba` method. Later on we'll need the `predict_proba` mthod for predicting probabilities for samples, to use it in explaining the model predictions.\n",
    "\n",
    "One way to obtain the probabilities of the classifier is to wrap it in `CalibratedClassifierCV` class, after using this class we'll be able to use the `predict_proba` method.\n",
    "\n",
    "Resources about how the `CalibratedClassifierCV` works, and how to use it:\n",
    "\n",
    "- [Sklearn Documentation](https://scikit-learn.org/stable/modules/generated/sklearn.calibration.CalibratedClassifierCV.html)\n",
    "- [Converting LinearSVC's decision function to probabilities (Scikit learn python)](https://stackoverflow.com/questions/26478000/converting-linearsvcs-decision-function-to-probabilities-scikit-learn-python/39712590#39712590)\n",
    "- [Scikit correct way to calibrate classifiers with CalibratedClassifierCV](https://stats.stackexchange.com/questions/263393/scikit-correct-way-to-calibrate-classifiers-with-calibratedclassifiercv/263411#263411)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "clf = CalibratedClassifierCV(base_estimator=svm_clf, cv=5, method=\"isotonic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([(\"vect\", vectorizer), (\"clf\", clf)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Fit the pipeline on the training data, and then evaluate it on the test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "pipeline.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "y_pred = pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Classification report:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fig = get_classification_report(\n",
    "    y_true=y_test, y_pred=y_pred, target_names=pipeline.classes_\n",
    ")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "save_fig_as_div(fig_obj=fig, file_name=\"charts/classification-report.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Confusion matrix:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fig = get_confusion_matrix(y_true=y_test, y_pred=y_pred, labels=pipeline.classes_)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "save_fig_as_div(fig_obj=fig, file_name=\"charts/confusion-matrix.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Explainability"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "At this point we should be done, right? after training the model and evaluating it, and achieving good results we should stop.\n",
    "\n",
    "There are many unanswered questions so far, for example:\n",
    "- How the model is working and making predictions (what are the features).\n",
    "- When the model predicts coorectly and when it's not.\n",
    "- Does the model generalize on the data.\n",
    "- Debugging the model on certain predictions.\n",
    "\n",
    "And most importantly, is this model reliable? can we use it in production with confidence?\n",
    "\n",
    "We can think of our model more or less like a *Black Box*, it takes some input, and produce some output.\n",
    "\n",
    "![alt text](assets/balck-box-ml.png \"Black Box Machine Learning\")\n",
    "\n",
    "The field of *Explainable artificial intelligence* which is concerned with the tools and methods for explaining and interpreting machine learning algorithms, catched a large interest in the past few years, and there has been many libraries that can be used out of the box for interpreting machine learning and deep learning models:\n",
    "- [eli5 (short for: Explain like I'm 5)](https://github.com/TeamHG-Memex/eli5)\n",
    "- [Lime: Explaining the predictions of any machine learning classifier](https://github.com/marcotcr/lime)\n",
    "- [SHAP: A game theoretic approach to explain the output of any machine learning model](https://github.com/slundberg/shap)\n",
    "\n",
    "These libraries differ in the way they work, and the type of models they can *interpret*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Model complexity vs interpretability:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "*Simple* linear models are easy to explain (since they consists of only linear equations), but their accuracy is low compared to *Complex* non-linear models, which achieve higher accuracy, but are harder to explain.\n",
    "\n",
    "The following figure illustrates the relation between model accuracy and interpretability (Source: [The balance: Accuracy vs. Interpretability](https://towardsdatascience.com/the-balance-accuracy-vs-interpretability-1b3861408062)):\n",
    "\n",
    "![alt text](assets/accuracy-vs-interpretability.png \"Accuracy vs Interpretability\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Since we're using a *linear SVM* classifier, we'll try to visualize the weights assigned to the features, and see for each class (category) in our data, what are the features that affect the model's prediction positivly and negatively.\n",
    "\n",
    "Resources about interpreting SVM classifier in `sklearn`:\n",
    "- [How does one interpret SVM feature weights?](https://stats.stackexchange.com/questions/39243/how-does-one-interpret-svm-feature-weights/39311#39311)\n",
    "- [Visualising Top Features in Linear SVM with Scikit Learn and Matplotlib](https://medium.com/@aneesha/visualising-top-features-in-linear-svm-with-scikit-learn-and-matplotlib-3454ab18a14d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Visualizing classifier weights:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "feature_names = pipeline[\"vect\"].get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "clf = pipeline.named_steps[\"clf\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "coefs_values = sum(\n",
    "    [classifier.base_estimator.coef_ for classifier in clf.calibrated_classifiers_]\n",
    ")\n",
    "\n",
    "coefs_values = coefs_values / len(clf.calibrated_classifiers_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "classes = clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "n_feature = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "for i, class_label in enumerate(classes):\n",
    "\n",
    "    # get indices of top positive/negative coefficient\n",
    "    negative_coefs_indices = np.argsort(coefs_values[i])[:n_feature]\n",
    "    positive_coefs_indices = np.argsort(coefs_values[i])[-n_feature:]\n",
    "\n",
    "    # get the coefficient values\n",
    "    negative_coefs = [coefs_values[i][coef_idx] for coef_idx in negative_coefs_indices]\n",
    "    positive_coefs = [coefs_values[i][coef_idx] for coef_idx in positive_coefs_indices]\n",
    "\n",
    "    # get the corresponding features names of the top coefficient\n",
    "    negative_features = [feature_names[coef_idx] for coef_idx in negative_coefs_indices]\n",
    "    positive_features = [feature_names[coef_idx] for coef_idx in positive_coefs_indices]\n",
    "\n",
    "    # stack arrays into one array\n",
    "    weights = np.concatenate([negative_coefs, positive_coefs])\n",
    "    features = np.concatenate([negative_features, positive_features])\n",
    "\n",
    "    # plot feature names agains their weight, using bar plot\n",
    "    chart_title = f\"{class_label} articles\"\n",
    "\n",
    "    chart_labels = {\"x\": \"Feature name\", \"y\": \"Feature weight\"}\n",
    "\n",
    "    fig = px.bar(\n",
    "        x=features,\n",
    "        y=weights,\n",
    "        color=weights,\n",
    "        title=chart_title,\n",
    "        orientation=\"v\",\n",
    "        labels=chart_labels,\n",
    "        width=800,\n",
    "        height=500,\n",
    "        color_continuous_scale=[\"red\", \"blue\"],\n",
    "    )\n",
    "\n",
    "    # rotate x-axis ticks\n",
    "    fig.update_xaxes(tickangle=45)\n",
    "\n",
    "    fig.update_layout(\n",
    "        {\n",
    "            \"plot_bgcolor\": \"rgba(0, 0, 0, 0)\",\n",
    "            \"paper_bgcolor\": \"rgba(0, 0, 0, 0)\",\n",
    "            \"font\": {\n",
    "                \"family\": \"Courier New, monospace\",\n",
    "                \"size\": 14,\n",
    "                # 'color': \"#eaeaea\"\n",
    "            },\n",
    "        }\n",
    "    )\n",
    "\n",
    "    fig.show()\n",
    "\n",
    "    save_fig_as_div(\n",
    "        fig, file_name=f\"model-coefficients/{class_label}-class-bar-chart.html\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Explaining individual predictions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "clf_predictions_df = pd.DataFrame(\n",
    "    data={\n",
    "        \"text\": X_test.values,\n",
    "        \"real category\": y_test.values,\n",
    "        \"predicted category\": y_pred,\n",
    "    }\n",
    ")\n",
    "\n",
    "clf_predictions_df[\"text length\"] = clf_predictions_df[\"text\"].str.len()\n",
    "\n",
    "clf_predictions_df.sort_values(by=\"text length\", ascending=False, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "clf_predictions_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# samples which has been correctly classified\n",
    "correct_classified_df = clf_predictions_df[\n",
    "    clf_predictions_df[\"real category\"] == clf_predictions_df[\"predicted category\"]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# samples which has been incorrectly classified\n",
    "incorrect_classified_df = clf_predictions_df[\n",
    "    clf_predictions_df[\"real category\"] != clf_predictions_df[\"predicted category\"]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# preprocessing function used in the pipeline to analyze input documents\n",
    "tokenize_fn = pipeline.named_steps[\"vect\"].build_analyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "explainer = LimeTextExplainer(verbose=True, class_names=pipeline.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "clf_fn = pipeline.predict_proba"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "### When the model is performing well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "correct_sample = correct_classified_df.loc[[2573]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "correct_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "text_to_explain = correct_sample[\"text\"].values[0]\n",
    "cleaned_text_to_explain = \" \".join(tokenize_fn(text_to_explain))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object = explainer.explain_instance(\n",
    "    text_instance=cleaned_text_to_explain,\n",
    "    classifier_fn=clf_fn,\n",
    "    top_labels=2,\n",
    "    num_features=10,\n",
    "    num_samples=10000,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "labels = exp_object.available_labels()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object.show_in_notebook(labels=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object.save_to_file(\n",
    "    file_path=\"assets/model-explanations/correct-classification-explanation.html\",\n",
    "    labels=labels,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Here the classifier predicted the correct class of the sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# politics_mideast class\n",
    "exp_object.as_list(label=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "We can see that words like `omar` and `hand` are negative fearues for this class, and if we remove them from the document, the the prediction percentage of class `politics_mideast` would increase."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "cleaned_text_to_explain = re.sub(\"omar|hand\", \"\", cleaned_text_to_explain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object = explainer.explain_instance(\n",
    "    text_instance=cleaned_text_to_explain,\n",
    "    classifier_fn=clf_fn,\n",
    "    top_labels=2,\n",
    "    num_features=10,\n",
    "    num_samples=10000,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "labels = exp_object.available_labels()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object.show_in_notebook(labels=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "### When the model is performing badly:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "incorrect_sample = incorrect_classified_df.loc[[3145]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "incorrect_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "text_to_explain = incorrect_sample[\"text\"].values[0]\n",
    "cleaned_text_to_explain = \" \".join(tokenize_fn(text_to_explain))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object = explainer.explain_instance(\n",
    "    text_instance=cleaned_text_to_explain,\n",
    "    classifier_fn=clf_fn,\n",
    "    top_labels=2,\n",
    "    num_features=10,\n",
    "    num_samples=10000,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "labels = exp_object.available_labels()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object.show_in_notebook(labels=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Here the classifier predicted an incorrect class, it predicted the class `autos` while the correct class is `forsale`, the reason is that the document very short, it contains two words about selling, and two words about cars, and the classification probabilities for the two classes were almost equal, let's try another example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "incorrect_sample = incorrect_classified_df.loc[[4251]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "incorrect_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Here, the classifier had done the opposite of the previous case, it predicted the class `forsale` while the real class is `autos`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "text_to_explain = incorrect_sample[\"text\"].values[0]\n",
    "cleaned_text_to_explain = \" \".join(tokenize_fn(text_to_explain))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object = explainer.explain_instance(\n",
    "    text_instance=cleaned_text_to_explain,\n",
    "    classifier_fn=clf_fn,\n",
    "    top_labels=2,\n",
    "    num_features=10,\n",
    "    num_samples=10000,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "labels = exp_object.available_labels()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object.show_in_notebook(labels=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "What happened here is that the classifier had correlated words like `seat` and `owner` with class `autos`, and words like `sale`, `original` and `sell` with class `forsale`, we can inspect the sample more to understand why it's so confusing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "print(text_to_explain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "Reading through the article, I *myself* actually got confused! so the article goes about someone criticizing the idea of posting car-seats ads, and he mentions two car models, the [Toyota MR2](https://en.wikipedia.org/wiki/Toyota_MR2) and the [Toyota Celica](https://en.wikipedia.org/wiki/Toyota_Celica), these two words are very informative to recognize the `autos` class, but it's hard for the classifier to catch them (since it's based on frequency statistics)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "exp_object.save_to_file(\n",
    "    file_path=\"assets/model-explanations/incorrect-classification-explanation.html\",\n",
    "    labels=labels,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Improving accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Edit normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "After inspecting how the model is predicting classes, we saw that word `MR2` is actually relevant to the classification, but it's being removed by the analyzer, I'll change the value of `min_token_length` to 2, this way only tokens of two letters or less will be removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "vectorizer = NLTKVectorizer(\n",
    "    max_df=0.5,\n",
    "    min_df=10,\n",
    "    ngram_range=(1, 1),\n",
    "    max_features=10000,\n",
    "    stop_words=en_stop_words,\n",
    "    min_token_length=2,\n",
    ")\n",
    "\n",
    "svm_clf = LinearSVC(C=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "clf = CalibratedClassifierCV(base_estimator=svm_clf, cv=5, method=\"isotonic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([(\"vect\", vectorizer), (\"clf\", clf)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "pipeline.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "y_pred = pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fig = get_classification_report(\n",
    "    y_true=y_test, y_pred=y_pred, target_names=pipeline.classes_\n",
    ")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "save_fig_as_div(fig_obj=fig, file_name=\"charts/classification-report-improved.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "false"
   },
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "fig = get_confusion_matrix(y_true=y_test, y_pred=y_pred, labels=pipeline.classes_)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "heading_collapsed": "false"
   },
   "outputs": [],
   "source": [
    "save_fig_as_div(fig_obj=fig, file_name=\"charts/confusion-matrix-improved.html\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc-autonumbering": true,
  "toc-showcode": false,
  "toc-showmarkdowntxt": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}