results_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Losses, NAB scores and classification metrics analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import precision_recall_curve, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = pd.read_csv(\"results/scores.csv\")\n",
    "losses_log_total = pd.read_csv(\"results/losses_log_total.csv\")\n",
    "losses_square_total = pd.read_csv(\"results/losses_square_total.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "experts = [\n",
    "    \"knncad\",\n",
    "    \"numentaTM\",\n",
    "    \"twitterADVec\",\n",
    "    \"skyline\",\n",
    "    \"earthgeckoSkyline\",\n",
    "    \"numenta\",\n",
    "    \"bayesChangePt\",\n",
    "    \"null\",\n",
    "    \"expose\",\n",
    "    \"relativeEntropy\",\n",
    "    \"htmjava\",\n",
    "    \"randomCutForest\",\n",
    "    \"random\",\n",
    "    \"contextOSE\",\n",
    "    \"windowedGaussian\",\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Total logarithmic and square losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses_log = losses_log_total.filter(regex=\"^loss\", axis=1).sum()\n",
    "losses_square = losses_square_total.filter(regex=\"^loss\", axis=1).sum()\n",
    "losses_log.index = [i.replace(\"loss_\", \"\") for i in losses_log.index]\n",
    "losses_square.index = [i.replace(\"loss_\", \"\") for i in losses_square.index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc_score = pd.DataFrame(\n",
    "    index=[\n",
    "        \"realAWSCloudwatch\",\n",
    "        \"realTraffic\",\n",
    "        \"realAdExchange\",\n",
    "        \"realTweets\",\n",
    "        \"artificialWithAnomaly\",\n",
    "        \"realKnownCause\",\n",
    "        \"Total\",\n",
    "    ]\n",
    ")\n",
    "alg_list = scores.filter(regex=\"^score\", axis=1).columns.tolist()\n",
    "alg_list = [i.replace(\"score_\", \"\") for i in alg_list]\n",
    "f1_score = auc_score.copy()\n",
    "class_score = auc_score.copy()\n",
    "thresholds_mat = auc_score.copy()\n",
    "EPSILON = 1e-7\n",
    "for folder_name in auc_score.index:\n",
    "    for alg_ind in alg_list:\n",
    "        if folder_name == \"Total\":\n",
    "            auc_score.loc[\"Total\", alg_ind] = roc_auc_score(\n",
    "                scores[\"label\"], scores[f\"score_{alg_ind}\"]\n",
    "            )\n",
    "            precision, recall, thresholds = precision_recall_curve(\n",
    "                scores[\"label\"], scores[f\"score_{alg_ind}\"]\n",
    "            )\n",
    "            precision[(precision == 0) & (recall == 0)] = EPSILON\n",
    "            f1 = 2 * precision * recall / (precision + recall)\n",
    "            f1_score.loc[\"Total\", alg_ind] = np.max(f1)\n",
    "            thresholds_mat.loc[\"Total\", alg_ind] = (\n",
    "                thresholds[f1[1:] == np.max(f1[1:])]\n",
    "            ).item()\n",
    "            preds = np.where(\n",
    "                scores[f\"score_{alg_ind}\"]\n",
    "                >= (thresholds[f1[1:] == np.max(f1[1:])]).item(),\n",
    "                1,\n",
    "                0,\n",
    "            )\n",
    "            class_score.loc[\"Total\", alg_ind] = (\n",
    "                sum(preds == scores[\"label\"]) / scores.shape[0]\n",
    "            )\n",
    "        else:\n",
    "            scores_folder = scores[scores[\"folder_name\"] == folder_name]\n",
    "            auc_score.loc[folder_name, alg_ind] = roc_auc_score(\n",
    "                scores_folder[\"label\"], scores_folder[f\"score_{alg_ind}\"]\n",
    "            )\n",
    "            precision, recall, thresholds = precision_recall_curve(\n",
    "                scores_folder[\"label\"], scores_folder[f\"score_{alg_ind}\"]\n",
    "            )\n",
    "            precision[(precision == 0) & (recall == 0)] = EPSILON\n",
    "            f1 = 2 * precision * recall / (precision + recall)\n",
    "            f1_score.loc[folder_name, alg_ind] = np.max(f1)\n",
    "            thresholds_mat.loc[folder_name, alg_ind] = (\n",
    "                thresholds[f1[1:] == np.max(f1[1:])]\n",
    "            ).item()\n",
    "            preds = np.where(\n",
    "                scores_folder[f\"score_{alg_ind}\"]\n",
    "                >= (thresholds[f1[1:] == np.max(f1[1:])]).item(),\n",
    "                1,\n",
    "                0,\n",
    "            )\n",
    "            class_score.loc[folder_name, alg_ind] = (\n",
    "                sum(preds == scores_folder[\"label\"]) / scores_folder.shape[0]\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_total = pd.concat(\n",
    "    [\n",
    "        auc_score.T[\"Total\"],\n",
    "        f1_score.T[\"Total\"],\n",
    "        class_score.T[\"Total\"],\n",
    "        losses_log,\n",
    "        losses_square,\n",
    "    ],\n",
    "    axis=1,\n",
    ")\n",
    "metrics_total.columns = [\n",
    "    \"auc\",\n",
    "    \"f1_score\",\n",
    "    \"class_accuracy\",\n",
    "    \"log_loss\",\n",
    "    \"square_loss\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "share_names = auc_score.filter(\n",
    "    regex=\"^Fixed|^Variable\", axis=1\n",
    ").columns.tolist()\n",
    "metrics_delays = metrics_total.loc[share_names]\n",
    "metrics_delays[\"alpha_delay\"] = metrics_delays.index.to_series().replace(\n",
    "    r\"^(Fixed|Variable)\", \"\", regex=True\n",
    ")\n",
    "metrics_delays[[\"alpha\", \"delay\"]] = metrics_delays[\"alpha_delay\"].str.split(\n",
    "    \"d\", expand=True\n",
    ")\n",
    "metrics_delays[\"type_alpha\"] = (\n",
    "    metrics_delays.index.str.split(r\"\\d+\", n=1, expand=True).get_level_values(\n",
    "        0\n",
    "    )\n",
    "    + metrics_delays[\"alpha\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pivots = {\n",
    "    key: pd.pivot_table(\n",
    "        metrics_delays.sort_values(\"alpha\"),\n",
    "        values=key,\n",
    "        index=\"type_alpha\",\n",
    "        columns=\"delay\",\n",
    "    )\n",
    "    for key in (\"auc\", \"f1_score\", \"class_accuracy\", \"log_loss\", \"square_loss\")\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_sorter(index):\n",
    "    fixed = index.str.replace(r\"\\d+\", \"\", regex=True)\n",
    "    alpha = index.str.replace(r\"\\D+\", \"\", regex=True).astype(int)\n",
    "    return pd.MultiIndex.from_arrays((fixed, alpha))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Area under curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>delay</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type_alpha</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Fixed0</th>\n",
       "      <td>0.997</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.809</td>\n",
       "      <td>0.696</td>\n",
       "      <td>0.771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed1</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.922</td>\n",
       "      <td>0.833</td>\n",
       "      <td>0.894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed5</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.869</td>\n",
       "      <td>0.912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed10</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed30</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.893</td>\n",
       "      <td>0.920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable0</th>\n",
       "      <td>0.991</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.657</td>\n",
       "      <td>0.581</td>\n",
       "      <td>0.639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable1</th>\n",
       "      <td>0.997</td>\n",
       "      <td>0.966</td>\n",
       "      <td>0.926</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable5</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.906</td>\n",
       "      <td>0.925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable10</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable30</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.955</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.929</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "delay           0      1      2      3      4\n",
       "type_alpha                                   \n",
       "Fixed0      0.997  0.938  0.809  0.696  0.771\n",
       "Fixed1      0.998  0.970  0.922  0.833  0.894\n",
       "Fixed5      0.998  0.974  0.936  0.869  0.912\n",
       "Fixed10     0.998  0.975  0.940  0.881  0.918\n",
       "Fixed30     0.998  0.975  0.942  0.893  0.920\n",
       "Variable0   0.991  0.826  0.657  0.581  0.639\n",
       "Variable1   0.997  0.966  0.926  0.874  0.904\n",
       "Variable5   0.998  0.976  0.949  0.906  0.925\n",
       "Variable10  0.998  0.978  0.954  0.909  0.929\n",
       "Variable30  0.998  0.979  0.955  0.908  0.929"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pivots[\"auc\"].sort_index(key=my_sorter).round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "randomCutForest    0.616\n",
       "knncad             0.600\n",
       "skyline            0.566\n",
       "Name: auc, dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_total.loc[experts][\"auc\"].sort_values(ascending=False).head(3).round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## F-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>delay</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type_alpha</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Fixed0</th>\n",
       "      <td>0.978</td>\n",
       "      <td>0.684</td>\n",
       "      <td>0.418</td>\n",
       "      <td>0.296</td>\n",
       "      <td>0.372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed1</th>\n",
       "      <td>0.988</td>\n",
       "      <td>0.832</td>\n",
       "      <td>0.651</td>\n",
       "      <td>0.455</td>\n",
       "      <td>0.596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed5</th>\n",
       "      <td>0.990</td>\n",
       "      <td>0.863</td>\n",
       "      <td>0.714</td>\n",
       "      <td>0.540</td>\n",
       "      <td>0.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed10</th>\n",
       "      <td>0.990</td>\n",
       "      <td>0.877</td>\n",
       "      <td>0.744</td>\n",
       "      <td>0.584</td>\n",
       "      <td>0.701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed30</th>\n",
       "      <td>0.984</td>\n",
       "      <td>0.897</td>\n",
       "      <td>0.788</td>\n",
       "      <td>0.650</td>\n",
       "      <td>0.748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable0</th>\n",
       "      <td>0.961</td>\n",
       "      <td>0.521</td>\n",
       "      <td>0.319</td>\n",
       "      <td>0.237</td>\n",
       "      <td>0.300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable1</th>\n",
       "      <td>0.979</td>\n",
       "      <td>0.809</td>\n",
       "      <td>0.677</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable5</th>\n",
       "      <td>0.983</td>\n",
       "      <td>0.870</td>\n",
       "      <td>0.775</td>\n",
       "      <td>0.672</td>\n",
       "      <td>0.740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable10</th>\n",
       "      <td>0.984</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.802</td>\n",
       "      <td>0.690</td>\n",
       "      <td>0.763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable30</th>\n",
       "      <td>0.987</td>\n",
       "      <td>0.896</td>\n",
       "      <td>0.791</td>\n",
       "      <td>0.677</td>\n",
       "      <td>0.751</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "delay           0      1      2      3      4\n",
       "type_alpha                                   \n",
       "Fixed0      0.978  0.684  0.418  0.296  0.372\n",
       "Fixed1      0.988  0.832  0.651  0.455  0.596\n",
       "Fixed5      0.990  0.863  0.714  0.540  0.667\n",
       "Fixed10     0.990  0.877  0.744  0.584  0.701\n",
       "Fixed30     0.984  0.897  0.788  0.650  0.748\n",
       "Variable0   0.961  0.521  0.319  0.237  0.300\n",
       "Variable1   0.979  0.809  0.677  0.553  0.642\n",
       "Variable5   0.983  0.870  0.775  0.672  0.740\n",
       "Variable10  0.984  0.888  0.802  0.690  0.763\n",
       "Variable30  0.987  0.896  0.791  0.677  0.751"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pivots[\"f1_score\"].sort_index(key=my_sorter).round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "randomCutForest    0.233\n",
       "htmjava            0.218\n",
       "knncad             0.216\n",
       "Name: f1_score, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_total.loc[experts][\"f1_score\"].sort_values(ascending=False).head(\n",
    "    3\n",
    ").round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>delay</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type_alpha</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Fixed0</th>\n",
       "      <td>0.996</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.886</td>\n",
       "      <td>0.824</td>\n",
       "      <td>0.878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed1</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.968</td>\n",
       "      <td>0.934</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed5</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.901</td>\n",
       "      <td>0.933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed10</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.977</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.914</td>\n",
       "      <td>0.941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed30</th>\n",
       "      <td>0.997</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable0</th>\n",
       "      <td>0.993</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.873</td>\n",
       "      <td>0.855</td>\n",
       "      <td>0.869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable1</th>\n",
       "      <td>0.996</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.918</td>\n",
       "      <td>0.933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable5</th>\n",
       "      <td>0.997</td>\n",
       "      <td>0.976</td>\n",
       "      <td>0.957</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable10</th>\n",
       "      <td>0.997</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.962</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable30</th>\n",
       "      <td>0.998</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "delay           0      1      2      3      4\n",
       "type_alpha                                   \n",
       "Fixed0      0.996  0.939  0.886  0.824  0.878\n",
       "Fixed1      0.998  0.968  0.934  0.875  0.923\n",
       "Fixed5      0.998  0.975  0.943  0.901  0.933\n",
       "Fixed10     0.998  0.977  0.950  0.914  0.941\n",
       "Fixed30     0.997  0.981  0.960  0.932  0.952\n",
       "Variable0   0.993  0.909  0.873  0.855  0.869\n",
       "Variable1   0.996  0.965  0.938  0.918  0.933\n",
       "Variable5   0.997  0.976  0.957  0.936  0.951\n",
       "Variable10  0.997  0.979  0.962  0.939  0.955\n",
       "Variable30  0.998  0.980  0.959  0.935  0.952"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pivots[\"class_accuracy\"].sort_index(key=my_sorter).round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numenta             0.866\n",
       "numentaTM           0.864\n",
       "windowedGaussian    0.862\n",
       "Name: class_accuracy, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_total.loc[experts][\"class_accuracy\"].sort_values(ascending=False).head(\n",
    "    3\n",
    ").round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logarithmic loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>delay</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type_alpha</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Fixed0</th>\n",
       "      <td>12.1</td>\n",
       "      <td>61.1</td>\n",
       "      <td>95.3</td>\n",
       "      <td>106.2</td>\n",
       "      <td>101.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed1</th>\n",
       "      <td>11.8</td>\n",
       "      <td>36.3</td>\n",
       "      <td>61.5</td>\n",
       "      <td>87.5</td>\n",
       "      <td>69.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed5</th>\n",
       "      <td>19.9</td>\n",
       "      <td>37.6</td>\n",
       "      <td>56.6</td>\n",
       "      <td>78.0</td>\n",
       "      <td>63.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed10</th>\n",
       "      <td>28.4</td>\n",
       "      <td>42.6</td>\n",
       "      <td>58.3</td>\n",
       "      <td>76.4</td>\n",
       "      <td>63.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed30</th>\n",
       "      <td>56.8</td>\n",
       "      <td>64.2</td>\n",
       "      <td>73.7</td>\n",
       "      <td>84.6</td>\n",
       "      <td>77.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable0</th>\n",
       "      <td>17.4</td>\n",
       "      <td>90.0</td>\n",
       "      <td>109.6</td>\n",
       "      <td>115.4</td>\n",
       "      <td>112.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable1</th>\n",
       "      <td>12.4</td>\n",
       "      <td>40.6</td>\n",
       "      <td>59.2</td>\n",
       "      <td>75.6</td>\n",
       "      <td>64.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable5</th>\n",
       "      <td>11.6</td>\n",
       "      <td>34.1</td>\n",
       "      <td>49.7</td>\n",
       "      <td>66.0</td>\n",
       "      <td>56.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable10</th>\n",
       "      <td>11.3</td>\n",
       "      <td>32.9</td>\n",
       "      <td>48.8</td>\n",
       "      <td>66.2</td>\n",
       "      <td>55.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable30</th>\n",
       "      <td>11.3</td>\n",
       "      <td>35.6</td>\n",
       "      <td>52.5</td>\n",
       "      <td>70.0</td>\n",
       "      <td>60.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "delay          0     1      2      3      4\n",
       "type_alpha                                 \n",
       "Fixed0      12.1  61.1   95.3  106.2  101.4\n",
       "Fixed1      11.8  36.3   61.5   87.5   69.9\n",
       "Fixed5      19.9  37.6   56.6   78.0   63.2\n",
       "Fixed10     28.4  42.6   58.3   76.4   63.8\n",
       "Fixed30     56.8  64.2   73.7   84.6   77.0\n",
       "Variable0   17.4  90.0  109.6  115.4  112.3\n",
       "Variable1   12.4  40.6   59.2   75.6   64.8\n",
       "Variable5   11.6  34.1   49.7   66.0   56.1\n",
       "Variable10  11.3  32.9   48.8   66.2   55.9\n",
       "Variable30  11.3  35.6   52.5   70.0   60.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(pivots[\"log_loss\"].sort_index(key=my_sorter) / 10 ** 3).round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "randomCutForest    107.5\n",
       "htmjava            153.9\n",
       "numentaTM          161.4\n",
       "Name: log_loss, dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    metrics_total.loc[experts][\"log_loss\"].sort_values(ascending=True).head(3)\n",
    "    / 10 ** 3\n",
    ").round(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Square loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>delay</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type_alpha</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Fixed0</th>\n",
       "      <td>2.9</td>\n",
       "      <td>16.5</td>\n",
       "      <td>26.4</td>\n",
       "      <td>29.0</td>\n",
       "      <td>27.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed1</th>\n",
       "      <td>2.5</td>\n",
       "      <td>9.7</td>\n",
       "      <td>17.4</td>\n",
       "      <td>24.9</td>\n",
       "      <td>19.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed5</th>\n",
       "      <td>3.0</td>\n",
       "      <td>8.9</td>\n",
       "      <td>15.1</td>\n",
       "      <td>21.8</td>\n",
       "      <td>17.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed10</th>\n",
       "      <td>4.1</td>\n",
       "      <td>9.2</td>\n",
       "      <td>14.6</td>\n",
       "      <td>20.7</td>\n",
       "      <td>16.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fixed30</th>\n",
       "      <td>10.3</td>\n",
       "      <td>13.4</td>\n",
       "      <td>17.1</td>\n",
       "      <td>21.2</td>\n",
       "      <td>18.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable0</th>\n",
       "      <td>3.8</td>\n",
       "      <td>23.5</td>\n",
       "      <td>28.9</td>\n",
       "      <td>30.0</td>\n",
       "      <td>29.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable1</th>\n",
       "      <td>2.9</td>\n",
       "      <td>10.7</td>\n",
       "      <td>16.4</td>\n",
       "      <td>21.3</td>\n",
       "      <td>17.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable5</th>\n",
       "      <td>2.8</td>\n",
       "      <td>8.8</td>\n",
       "      <td>13.6</td>\n",
       "      <td>18.6</td>\n",
       "      <td>15.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable10</th>\n",
       "      <td>2.7</td>\n",
       "      <td>8.6</td>\n",
       "      <td>13.6</td>\n",
       "      <td>18.8</td>\n",
       "      <td>15.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variable30</th>\n",
       "      <td>2.7</td>\n",
       "      <td>10.2</td>\n",
       "      <td>15.3</td>\n",
       "      <td>19.9</td>\n",
       "      <td>16.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "delay          0     1     2     3     4\n",
       "type_alpha                              \n",
       "Fixed0       2.9  16.5  26.4  29.0  27.6\n",
       "Fixed1       2.5   9.7  17.4  24.9  19.6\n",
       "Fixed5       3.0   8.9  15.1  21.8  17.1\n",
       "Fixed10      4.1   9.2  14.6  20.7  16.4\n",
       "Fixed30     10.3  13.4  17.1  21.2  18.3\n",
       "Variable0    3.8  23.5  28.9  30.0  29.5\n",
       "Variable1    2.9  10.7  16.4  21.3  17.9\n",
       "Variable5    2.8   8.8  13.6  18.6  15.2\n",
       "Variable10   2.7   8.6  13.6  18.8  15.3\n",
       "Variable30   2.7  10.2  15.3  19.9  16.9"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(pivots[\"square_loss\"].sort_index(key=my_sorter) / 10 ** 3).round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "randomCutForest    29.4\n",
       "htmjava            31.5\n",
       "numentaTM          31.8\n",
       "Name: square_loss, dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(\n",
    "    metrics_total.loc[experts][\"square_loss\"]\n",
    "    .sort_values(ascending=True)\n",
    "    .head(3)\n",
    "    / 10 ** 3\n",
    ").round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "anomaly_delays_github",
   "language": "python",
   "name": "anomaly_delays_github"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}