visualization_utilities.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-20T23:26:57.115880Z",
     "start_time": "2019-11-20T23:26:55.873937Z"
    }
   },
   "outputs": [],
   "source": [
    "from IPython.display import display, HTML, Image\n",
    "from ipywidgets.widgets import FloatSlider, IntSlider, Dropdown, SelectMultiple, Button\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, precision_score\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interface to select classification target and feature set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:17.868939Z",
     "start_time": "2019-11-17T02:17:17.862355Z"
    }
   },
   "outputs": [],
   "source": [
    "def interface(df, target, label):\n",
    "    all_columns = df.columns.values.tolist()\n",
    "    target_widget = Dropdown(\n",
    "        margin=10,\n",
    "        options=all_columns,\n",
    "        value=target,\n",
    "        description='Target Variable:',\n",
    "    )\n",
    "    features_widget = SelectMultiple(\n",
    "        margin=10,\n",
    "        description=\"Input Features:\",\n",
    "        options=all_columns\n",
    "    )\n",
    "    button = Button(description=label, width=100)\n",
    "    return target_widget, features_widget, button\n",
    "\n",
    "def explore_matrix(dependent_variable, input_features):\n",
    "    if dependent_variable in input_features:\n",
    "        raise Exception('Dependent Variable should not be one of the input features.')\n",
    "    pair_columns = [dependent_variable] + list(input_features)\n",
    "    sns.pairplot(df[pair_columns], hue=dependent_variable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Display Change in size of the dataset after dropping rows with missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:28.068716Z",
     "start_time": "2019-11-17T02:17:28.065552Z"
    }
   },
   "outputs": [],
   "source": [
    "def size_after_droppping_rows_with_missing_values(count_before_drop, count_after_drop):\n",
    "    print(\"\"\"\n",
    "    Total data points before dropna: \\t{0}\n",
    "    Total data points after dropna: \\t{1}\n",
    "    Data points with missing values: \\t{2}\n",
    "    \"\"\".format(count_before_drop, count_after_drop, count_before_drop - count_after_drop))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Display characteristics of train, validation, and test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:30.817248Z",
     "start_time": "2019-11-17T02:17:30.810757Z"
    }
   },
   "outputs": [],
   "source": [
    "def balance_as_text(label, label_ratio, data_size, numerator, denominator):\n",
    "    template = \"\"\"\n",
    "{0}\n",
    "   Size: {2}\n",
    "   {1} {3:.3f}\n",
    "\"\"\".format(label, label_ratio, data_size, numerator / (1.0 * denominator))\n",
    "    return template\n",
    "\n",
    "def display_partitioning(y_train, y_validation,  y_test, positive_class, negative_class):\n",
    "    print(balance_as_text(\"Training dataset\", \"positive/negative ratio:\", \n",
    "                          len(y_train), \n",
    "                          y_train.values.tolist().count(positive_class),\n",
    "                          y_train.values.tolist().count(negative_class)))\n",
    "    print(balance_as_text(\"Validation dataset\", \"positive/negative ratio:\",\n",
    "                          len(y_validation), \n",
    "                          y_validation.values.tolist().count(positive_class),\n",
    "                          y_validation.values.tolist().count(negative_class)))\n",
    "    print(balance_as_text(\"Test dataset\", \"positive/negative ratio:\",\n",
    "                          len(y_test), \n",
    "                          y_test.values.tolist().count(positive_class),\n",
    "                          y_test.values.tolist().count(negative_class)))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute and show evaluation metrics for logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:32.274784Z",
     "start_time": "2019-11-17T02:17:32.268870Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_class_separation(y_negative, y_positive, num_bins, threshold_value):\n",
    "\n",
    "    n0, bins0, patches0 = plt.hist(y_negative, num_bins,  facecolor='red', alpha=0.3, label=negative_class)\n",
    "    n1, bins1, patches1 = plt.hist(y_positive, num_bins,  facecolor='blue', alpha=0.3, label=positive_class)\n",
    "    \n",
    "    plt.axvline(x=threshold_value, ymin=0, ymax=1, linewidth=2, color='#aaaaaa')\n",
    "\n",
    "    plt.xlabel('probability')\n",
    "    plt.ylabel('total')\n",
    "    plt.title('Class separation')\n",
    "    plt.grid(True)\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:33.278044Z",
     "start_time": "2019-11-17T02:17:33.268279Z"
    }
   },
   "outputs": [],
   "source": [
    "def show_evaluation_metrics_logistics(model, threshold_value, x, y): \n",
    "          \n",
    "    y_pred = apply_threshold(model, threshold_value, x)\n",
    "    report = classification_report(y, y_pred, target_names=class_labels)\n",
    "    cm = confusion_matrix(y, y_pred)\n",
    "    accuracy = accuracy_score(y, y_pred)\n",
    "  \n",
    "    tpr_value = cm[1][1]/(1.0*y.values.tolist().count(1))\n",
    "    fpr_value = cm[0][1]/(1.0*y.values.tolist().count(0))\n",
    "\n",
    "    predicted_positive_count = y_pred.count(1)\n",
    "    predicted_negative_count = y_pred.count(0)\n",
    "        \n",
    "    display(HTML(\"\"\"\n",
    "    <div>\n",
    "      <table>\n",
    "        <tr> <th>Accuracy (Evaluation):</th> <td colspan=\"6\" style=\"background-color:#cceecc\"> {ACCURACY:.3f} </td></tr>\n",
    "        <tr> <th> {CTRL_LABEL} </th> <td> {Threshold:.4f}</td> <th>&nbsp;</th> <th>&nbsp;</th> <th>Predicted {POSITIVE_CLASS}</th> <th>Predicted {NEGATIVE_CLASS}</th> <th> Total </th></tr>\n",
    "        <tr> <th> True Positive Rate </th> <td style=\"background-color:#eeffcc\">{TPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {POSITIVE_CLASS}</th> <td style=\"background-color:#ccffcc\">{TP}</td> <td style=\"background-color:#eedddd\">{FN}</td> \n",
    "             <td style=\"background-color:#ccffff\"> {Y_TEST_POSITIVE} </td>\n",
    "        </tr>\n",
    "\n",
    "        <tr> <th> False positive Rate </th> <td style=\"background-color:#ddeebb\">{FPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {NEGATIVE_CLASS}</th> <td style=\"background-color:#ffdddd\">{FP}</td> <td style=\"background-color:#bbeebb\">{TN}</td> \n",
    "             <td style=\"background-color:#aadddd\"> {Y_TEST_NEGATIVE} </td>\n",
    "        </tr>\n",
    "        <tr>  <th colspan=\"3\">&nbsp;</th> <th>Total</th> <td style=\"background-color:#ffffcc\">{PREDICTED_POSITIVE}</td> <td style=\"background-color:#ddddaa\">{PREDICTED_NEGATIVE}</td> <td style=\"background-color:#dddddd\">{TOTAL}</td> </tr>\n",
    "\n",
    "        <tr> <td colspan=\"6\"><pre>{REPORT}</pre></td>   </tr>\n",
    "        \n",
    "      </table>\n",
    "    </div>\n",
    "\n",
    "    \"\"\".format(Threshold=threshold_value, \n",
    "                     CTRL_LABEL='Threshold', \n",
    "                     TPR=tpr_value, \n",
    "                     FPR=fpr_value, \n",
    "                     REPORT=report,  \n",
    "                     ACCURACY=accuracy,\n",
    "                     POSITIVE_CLASS=positive_class,\n",
    "                     NEGATIVE_CLASS=negative_class,\n",
    "                     Y_TEST_POSITIVE=y.values.tolist().count(1),\n",
    "                     Y_TEST_NEGATIVE=y.values.tolist().count(0),\n",
    "                     PREDICTED_POSITIVE=predicted_positive_count,\n",
    "                     PREDICTED_NEGATIVE=predicted_negative_count,\n",
    "                     TOTAL=predicted_positive_count + predicted_negative_count,\n",
    "                     TN=cm[0][0],\n",
    "                     FN=cm[1][0],\n",
    "                     FP=cm[0][1],\n",
    "                     TP=cm[1][1])))\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute and show evaluation metrics for other models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:37.180430Z",
     "start_time": "2019-11-17T02:17:37.170913Z"
    }
   },
   "outputs": [],
   "source": [
    "def show_evaluation_metrics(clf, control_label, k, x, y): \n",
    "    \n",
    "    y_pred = clf.predict(x)\n",
    "    report = classification_report(y, y_pred, target_names=class_labels)\n",
    "    cm = confusion_matrix(y, y_pred)\n",
    "    accuracy = accuracy_score(y, y_pred)\n",
    "    \n",
    "    tpr_value = cm[0][0]/(1.0*y.values.tolist().count(positive_class))\n",
    "    fpr_value = cm[1][0]/(1.0*y.values.tolist().count(negative_class))\n",
    "    \n",
    "    predicted_positive_count = y_pred.tolist().count(positive_class)\n",
    "    predicted_negative_count = y_pred.tolist().count(negative_class)\n",
    "    \n",
    "    display(HTML(\"\"\"\n",
    "    <div>\n",
    "      <table>\n",
    "        <tr> <th>Accuracy (Evaluation):</th> <td colspan=\"5\" style=\"background-color:#cceecc\"> {ACCURACY:.3f} </td></tr>\n",
    "        <tr> <th> {CTRL_LABEL} </th> <td> {Complexity:.1f} </td> <th>&nbsp;</th> <th>&nbsp;</th> <th>Predicted {POSITIVE_CLASS}</th> <th>Predicted {NEGATIVE_CLASS}</th> <th> Total </th></tr>\n",
    "        <tr> <th> True Positive Rate </th> <td style=\"background-color:#eeffcc\">{TPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {POSITIVE_CLASS}</th> <td style=\"background-color:#ccffcc\">{TP}</td> <td style=\"background-color:#eedddd\">{FN}</td> \n",
    "             <td style=\"background-color:#ccffff\"> {Y_TEST_POSITIVE} </td>\n",
    "        </tr>\n",
    "\n",
    "        <tr> <th> False positive Rate </th> <td style=\"background-color:#ddeebb\">{FPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {NEGATIVE_CLASS}</th> <td style=\"background-color:#ffdddd\">{FP}</td> <td style=\"background-color:#bbeebb\">{TN}</td> \n",
    "             <td style=\"background-color:#aadddd\"> {Y_TEST_NEGATIVE} </td>\n",
    "        </tr>\n",
    "        <tr>  <th colspan=\"3\">&nbsp;</th> <th>Total</th> <td style=\"background-color:#ffffcc\">{PREDICTED_POSITIVE}</td> <td style=\"background-color:#ddddaa\">{PREDICTED_NEGATIVE}</td> <td style=\"background-color:#dddddd\">{TOTAL}</td> </tr>\n",
    "\n",
    "        <tr> <td colspan=\"6\"><pre>{REPORT}</pre></td>   </tr>\n",
    "        \n",
    "      </table>\n",
    "    </div>\n",
    "\n",
    "    \"\"\".format(Complexity=k,\n",
    "              CTRL_LABEL= control_label, \n",
    "              TPR=tpr_value, \n",
    "              FPR=fpr_value, \n",
    "              REPORT=report,  \n",
    "              ACCURACY=accuracy,\n",
    "              POSITIVE_CLASS=positive_class,\n",
    "              NEGATIVE_CLASS=negative_class,\n",
    "              Y_TEST_POSITIVE=y.values.tolist().count(positive_class),\n",
    "              Y_TEST_NEGATIVE=y.values.tolist().count(negative_class),\n",
    "              PREDICTED_POSITIVE=predicted_positive_count,\n",
    "              PREDICTED_NEGATIVE=predicted_negative_count,\n",
    "              TOTAL=predicted_positive_count + predicted_negative_count,\n",
    "              TN=cm[1][1],\n",
    "              FN=cm[0][1],\n",
    "              FP=cm[1][0],\n",
    "              TP=cm[0][0])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Interactive ROC/Error Curve with computed metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:39.368261Z",
     "start_time": "2019-11-17T02:17:39.338974Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_roc(fpr, tpr, fpr_t=None, tpr_t=None, figsize=(10,9)):\n",
    "    plt.style.use('ggplot')\n",
    "    plt.figure(figsize=figsize)\n",
    "    plt.title('Receiver operating characteristic curve (ROC curve)')\n",
    "    plt.plot(fpr, tpr, lw=1, label ='Logistic Regression')\n",
    "    plt.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), label='Luck')\n",
    "    if fpr_t is not None:\n",
    "        plt.axvline(x=fpr_t, ymin=0, ymax=1, linewidth=1, color='#aaaaaa')\n",
    "    if tpr_t is not None:\n",
    "        plt.axhline(y=tpr_t, xmin=0, xmax=1, linewidth=1, color='#aaaaaa')\n",
    "    plt.legend(loc=\"lower right\")\n",
    "    plt.xlim([-0.05, 1.05])\n",
    "    plt.ylim([-0.05, 1.05])\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.show()\n",
    "\n",
    "def apply_threshold(model, threshold, x):\n",
    "    y_predicted_proba = model.predict_proba(x)\n",
    "    y_1 = [int(c1 > threshold) for _, c1 in y_predicted_proba]\n",
    "    return y_1\n",
    "\n",
    "\n",
    "def plot_error(depths, error_validation, error_train, model_complexity):\n",
    "    \n",
    "    plt.style.use('ggplot')\n",
    "    plt.figure(figsize=(10,9))\n",
    "    plt.title('Validation')\n",
    "    plt.plot(depths, error_validation, lw=1, label ='Validation dataset')\n",
    "    plt.plot(depths, error_train, lw=1, label ='Train dataset')\n",
    "    plt.axvline(x=model_complexity, ymin=0, ymax=1, linewidth=1, color='#aaaaaa')\n",
    "    plt.legend(loc=\"lower right\")\n",
    "    plt.xlabel('Model Complexity')\n",
    "    plt.ylabel('Error Rate')\n",
    "    plt.ylim([-0.05, 1.05])    \n",
    "    plt.show()    \n",
    "       \n",
    "    \n",
    "def evaluate_error(model_map, control_label, display_tree, model_complexity):\n",
    "    \n",
    "    y_pred = predicted_map[model_complexity]\n",
    "    report = classification_report(y_validation, y_pred, target_names=class_labels)\n",
    "    cm = confusion_matrix(y_validation, y_pred)\n",
    "    accuracy = accuracy_score(y_validation, y_pred)\n",
    "\n",
    "    tpr_value = cm[0][0]/(1.0*y_validation.values.tolist().count(positive_class))\n",
    "    fpr_value = cm[1][0]/(1.0*y_validation.values.tolist().count(negative_class))\n",
    "    \n",
    "    plot_error(complexities, error_validation, error_train, model_complexity)    \n",
    "      \n",
    "    display(HTML(\"\"\"\n",
    "    <div>\n",
    "      <table>\n",
    "        <tr> <th>Accuracy (Evaluation):</th> <td colspan=\"5\" style=\"background-color:#cceecc\"> {ACCURACY:.3f} </td></tr>\n",
    "\n",
    "        <tr> <th> {CTRL_LABEL} </th> <td>{Complexity:.1f} </td> <th>&nbsp;</th> <th>&nbsp;</th> <th>Predicted {POSITIVE_CLASS}</th> <th>Predicted {NEGATIVE_CLASS}</th> <th> Total </th></tr>\n",
    "        <tr> <th> True Positive Rate </th> <td style=\"background-color:#eeffcc\">{TPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {POSITIVE_CLASS}</th> <td style=\"background-color:#ccffcc\">{TP}</td> <td style=\"background-color:#eedddd\">{FN}</td> \n",
    "             <td style=\"background-color:#ccffff\"> {Y_TEST_POSITIVE} </td>\n",
    "        </tr>\n",
    "\n",
    "        <tr> <th> False Positive Rate </th> <td style=\"background-color:#ddeebb\">{FPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {NEGATIVE_CLASS}</th> <td style=\"background-color:#ffdddd\">{FP}</td> <td style=\"background-color:#bbeebb\">{TN}</td> \n",
    "             <td style=\"background-color:#aadddd\"> {Y_TEST_NEGATIVE} </td>\n",
    "        </tr>\n",
    "        <tr>  <th colspan=\"3\">&nbsp;</th> <th>Total</th> <td style=\"background-color:#ffffcc\">{PREDICTED_POSITIVE}</td> <td style=\"background-color:#ddddaa\">{PREDICTED_NEGATIVE}</td> </tr>\n",
    "\n",
    "        <tr> <td colspan=\"6\"><pre>{REPORT}</pre></td>   </tr>\n",
    "        \n",
    "      </table>\n",
    "    </div>\n",
    "\n",
    "    \"\"\".format(      Complexity=model_complexity,\n",
    "                     CTRL_LABEL= control_label, \n",
    "                     TPR=tpr_value, \n",
    "                     FPR=fpr_value, \n",
    "                     REPORT=report,  \n",
    "                     ACCURACY=accuracy,\n",
    "                     POSITIVE_CLASS=positive_class,\n",
    "                     NEGATIVE_CLASS=negative_class,\n",
    "                     Y_TEST_POSITIVE=y_validation.values.tolist().count(positive_class),\n",
    "                     Y_TEST_NEGATIVE=y_validation.values.tolist().count(negative_class),\n",
    "                     PREDICTED_POSITIVE=y_pred.tolist().count(positive_class),\n",
    "                     PREDICTED_NEGATIVE=y_pred.tolist().count(negative_class),\n",
    "                     TN=cm[1][1],\n",
    "                     FN=cm[0][1],\n",
    "                     FP=cm[1][0],\n",
    "                     TP=cm[0][0])))\n",
    "    if display_tree:\n",
    "        show_tree(model_map[model_complexity], class_names)\n",
    "\n",
    "    \n",
    "def interactive_roc(model, control_label, cursor):\n",
    "    threshold_value = cursor\n",
    "\n",
    "    fpr_float, tpr_float, thresholds = roc_curve(y_validation, probs_[:, 1])\n",
    "\n",
    "    y_predicted = apply_threshold(model, threshold_value, x_validation)\n",
    "    report = classification_report(y_validation, y_predicted, target_names=class_labels)\n",
    "    cm = confusion_matrix(y_validation, y_predicted)\n",
    "    accuracy = accuracy_score(y_validation, y_predicted)\n",
    "    fpr_threshold, tpr_threshold, _ = roc_curve(y_validation, y_predicted)    \n",
    "    auc_score = auc(fpr_threshold, tpr_threshold)  \n",
    "    \n",
    "    tpr_value = cm[1][1]/(1.0*y_validation.values.tolist().count(1))\n",
    "    fpr_value = cm[0][1]/(1.0*y_validation.values.tolist().count(0))\n",
    "   \n",
    "    plot_roc(fpr_float, tpr_float, fpr_value, tpr_value, figsize=(7,6.3))\n",
    "\n",
    "    predicted_positive_count = y_predicted.count(1)\n",
    "    predicted_negative_count = y_predicted.count(0)\n",
    "\n",
    "    \n",
    "    display(HTML(\"\"\"\n",
    "    <div>\n",
    "      <table>\n",
    "        <tr> <th>Accuracy (Evaluation):</th> <td  style=\"background-color:#cceecc\"> {ACCURACY:.3f} </td>  \n",
    "             <th>Area Under Curve (AUC):</th> <td  style=\"background-color:#cceecc\" colspan=\"4\"> {AUC:.3f} </td> \n",
    "        </tr>\n",
    "        <tr> <th> {CTRL_LABEL} </th> <td> {Threshold:.4f}</td> <th>&nbsp;</th> <th>&nbsp;</th> <th>Predicted {POSITIVE_CLASS}</th> <th>Predicted {NEGATIVE_CLASS}</th> <th> Total </th></tr>\n",
    "        <tr> <th> True Positive Rate </th> <td style=\"background-color:#eeffcc\">{TPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {POSITIVE_CLASS}</th> <td style=\"background-color:#ccffcc\">{TP}</td> <td style=\"background-color:#eedddd\">{FN}</td> \n",
    "             <td style=\"background-color:#ccffff\"> {Y_TEST_POSITIVE} </td>\n",
    "        </tr>\n",
    "\n",
    "        <tr> <th> False Positive Rate </th> <td style=\"background-color:#ddeebb\">{FPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {NEGATIVE_CLASS}</th> <td style=\"background-color:#ffdddd\">{FP}</td> <td style=\"background-color:#bbeebb\">{TN}</td> \n",
    "             <td style=\"background-color:#aadddd\"> {Y_TEST_NEGATIVE} </td>\n",
    "        </tr>\n",
    "        <tr>  <th colspan=\"3\">&nbsp;</th> <th>Total</th> <td style=\"background-color:#ffffcc\">{PREDICTED_POSITIVE}</td> <td style=\"background-color:#ddddaa\">{PREDICTED_NEGATIVE}</td>  <td style=\"background-color:#dddddd\">{TOTAL}</td> </tr>\n",
    "\n",
    "        <tr> <td colspan=\"6\"><pre>{REPORT}</pre></td>   </tr>\n",
    "        \n",
    "      </table>\n",
    "    </div>\n",
    "\n",
    "    \"\"\".format(Threshold=threshold_value, \n",
    "                 CTRL_LABEL=control_label, \n",
    "                 TPR=tpr_value, \n",
    "                 FPR=fpr_value, \n",
    "                 REPORT=report,  \n",
    "                 ACCURACY=accuracy,\n",
    "                 AUC=auc_score,\n",
    "                 POSITIVE_CLASS=positive_class,\n",
    "                 NEGATIVE_CLASS=negative_class,\n",
    "                 Y_TEST_POSITIVE=y_validation.values.tolist().count(1),\n",
    "                 Y_TEST_NEGATIVE=y_validation.values.tolist().count(0),\n",
    "                 PREDICTED_POSITIVE=predicted_positive_count,\n",
    "                 PREDICTED_NEGATIVE=predicted_negative_count,\n",
    "                 TOTAL=predicted_positive_count + predicted_negative_count,\n",
    "                 TN=cm[0][0],\n",
    "                 FN=cm[1][0],\n",
    "                 FP=cm[0][1],\n",
    "                 TP=cm[1][1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:40.355708Z",
     "start_time": "2019-11-17T02:17:40.345850Z"
    }
   },
   "outputs": [],
   "source": [
    "def interactive_class_separation(model, control_label, num_bins, cursor):\n",
    "\n",
    "    threshold_value = cursor\n",
    "    \n",
    "    y_pred = apply_threshold(model, threshold_value, x_validation)\n",
    "    report = classification_report(y_validation, y_pred, target_names=class_labels)\n",
    "    cm = confusion_matrix(y_validation, y_pred)\n",
    "    accuracy = accuracy_score(y_validation, y_pred)\n",
    "    fpr, tpr, _ = roc_curve(y_validation, y_pred)\n",
    "    auc_score = auc(fpr, tpr)\n",
    "    \n",
    "    tpr_value = cm[1][1]/(1.0*y_validation.values.tolist().count(1))\n",
    "    fpr_value = cm[0][1]/(1.0*y_validation.values.tolist().count(0))\n",
    "    \n",
    "    plot_class_separation(y_0, y_1, num_bins, threshold_value)\n",
    "    \n",
    "    predicted_positive_count = y_pred.count(1)\n",
    "    predicted_negative_count = y_pred.count(0)\n",
    "    \n",
    "    display(HTML(\"\"\"\n",
    "    <div>\n",
    "      <table>\n",
    "        <tr> <th>Accuracy (Evaluation):</th> <td  style=\"background-color:#cceecc\"> {ACCURACY:.3f} </td>  \n",
    "             <th>Area Under Curve (AUC):</th> <td  style=\"background-color:#cceecc\" colspan=\"4\"> {AUC:.3f} </td> \n",
    "        </tr>\n",
    "        <tr> <th> {CTRL_LABEL} </th> <td> {Threshold:.4f}</td> <th>&nbsp;</th> <th>&nbsp;</th> <th>Predicted {POSITIVE_CLASS}</th> <th>Predicted {NEGATIVE_CLASS}</th> <th> Total </th></tr>\n",
    "        <tr> <th> True Positive Rate </th> <td style=\"background-color:#eeffcc\">{TPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {POSITIVE_CLASS}</th> <td style=\"background-color:#ccffcc\">{TP}</td> <td style=\"background-color:#eedddd\">{FN}</td> \n",
    "             <td style=\"background-color:#ccffff\"> {Y_TEST_POSITIVE} </td>\n",
    "        </tr>\n",
    "\n",
    "        <tr> <th> False positive Rate </th> <td style=\"background-color:#ddeebb\">{FPR:.3f}</td>  <th>&nbsp;</th> \n",
    "             <th>Real {NEGATIVE_CLASS}</th> <td style=\"background-color:#ffdddd\">{FP}</td> <td style=\"background-color:#bbeebb\">{TN}</td> \n",
    "             <td style=\"background-color:#aadddd\"> {Y_TEST_NEGATIVE} </td>\n",
    "        </tr>\n",
    "        <tr>  <th colspan=\"3\">&nbsp;</th> <th>Total</th> <td style=\"background-color:#ffffcc\">{PREDICTED_POSITIVE}</td> <td style=\"background-color:#ddddaa\">{PREDICTED_NEGATIVE}</td> <td style=\"background-color:#dddddd\">{TOTAL}</td> </tr>\n",
    "\n",
    "        <tr> <td colspan=\"6\"><pre>{REPORT}</pre></td>   </tr>\n",
    "        \n",
    "      </table>\n",
    "    </div>\n",
    "\n",
    "    \"\"\".format(Threshold=threshold_value, \n",
    "                     CTRL_LABEL='Threshold', \n",
    "                     TPR=tpr_value, \n",
    "                     FPR=fpr_value, \n",
    "                     REPORT=report,  \n",
    "                     ACCURACY=accuracy,\n",
    "                     AUC=auc_score,\n",
    "                     POSITIVE_CLASS=positive_class,\n",
    "                     NEGATIVE_CLASS=negative_class,\n",
    "                     TOTAL=predicted_positive_count + predicted_negative_count,\n",
    "                     Y_TEST_POSITIVE=y_validation.values.tolist().count(1),\n",
    "                     Y_TEST_NEGATIVE=y_validation.values.tolist().count(0),\n",
    "                     PREDICTED_POSITIVE=predicted_positive_count,\n",
    "                     PREDICTED_NEGATIVE=predicted_negative_count,\n",
    "                     TN=cm[0][0],\n",
    "                     FN=cm[1][0],\n",
    "                     FP=cm[0][1],\n",
    "                     TP=cm[1][1])))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:17:42.437214Z",
     "start_time": "2019-11-17T02:17:42.433202Z"
    }
   },
   "outputs": [],
   "source": [
    "def find_closest(A, target):\n",
    "    idx = A.searchsorted(target)\n",
    "    idx = np.clip(idx, 1, len(A)-1)\n",
    "    left = A[idx-1]\n",
    "    right = A[idx]\n",
    "    idx -= target - left < right - target\n",
    "    return idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:18:00.392682Z",
     "start_time": "2019-11-17T02:18:00.386191Z"
    }
   },
   "outputs": [],
   "source": [
    "def check_confusion_matrix(model, x, y):\n",
    "    y_predicted = model.predict(x)\n",
    "    d = zip(y_validation, y_predicted) \n",
    "    tp = [(a,b) for (a,b) in d if (b==1 and a==1)]\n",
    "    fp = [(a,b) for (a,b) in d if (b==1 and a==0) ]\n",
    "    tn = [(a,b) for (a,b) in d if (b==0 and a==0) ]\n",
    "    fn = [(a,b) for (a,b) in d if (b==0 and a==1) ]\n",
    "    print(\"tp={tp} fp={fp} tn={tn} fn={fn}\".format(tp=len(tp), fp=len(fp), tn=len(tn), fn=len(fn)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T02:18:01.617462Z",
     "start_time": "2019-11-17T02:18:01.609318Z"
    }
   },
   "outputs": [],
   "source": [
    "def display_oob_error_rate(X, y, ensemble_clfs, min_estimators, max_estimators, target_n_estimators=None):\n",
    "    # Author: Kian Ho <hui.kian.ho@gmail.com>\n",
    "    #         Gilles Louppe <g.louppe@gmail.com>\n",
    "    #         Andreas Mueller <amueller@ais.uni-bonn.de>\n",
    "    #\n",
    "    # License: BSD 3 Clause\n",
    "\n",
    "    import matplotlib.pyplot as plt\n",
    "    from collections import OrderedDict\n",
    "    from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "\n",
    "    # NOTE: Setting the `warm_start` construction parameter to `True` disables\n",
    "    # support for paralellised ensembles but is necessary for tracking the OOB\n",
    "    # error trajectory during training.\n",
    "\n",
    "    # Map a classifier name to a list of (<n_estimators>, <error rate>) pairs.\n",
    "    error_rate = OrderedDict((label, []) for label, _ in ensemble_clfs)\n",
    "\n",
    "    # Range of `n_estimators` values to explore.\n",
    "    for label, clf in ensemble_clfs:\n",
    "        for i in range(min_estimators, max_estimators + 1):\n",
    "            clf.set_params(n_estimators=i)\n",
    "            clf.fit(X, y)\n",
    "\n",
    "            # Record the OOB error for each `n_estimators=i` setting.\n",
    "            oob_error = 1 - clf.oob_score_\n",
    "            error_rate[label].append((i, oob_error))\n",
    "\n",
    "    # Generate the \"OOB error rate\" vs. \"n_estimators\" plot.\n",
    "    for label, clf_err in error_rate.items():\n",
    "        xs, ys = zip(*clf_err)\n",
    "        plt.plot(xs, ys, label=label)\n",
    "\n",
    "    if target_n_estimators is not None:\n",
    "        plt.axvline(x=target_n_estimators, ymin=0, ymax=1, linewidth=1, color='#aaaaaa')\n",
    "\n",
    "    plt.xlim(min_estimators, max_estimators)\n",
    "    plt.xlabel(\"n_estimators\")\n",
    "    plt.ylabel(\"OOB error rate\")\n",
    "    plt.legend(loc=\"upper right\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}