calculate_predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from anomaly_delays.helper_functions import generate_random_delays, read_nab\n",
    "from anomaly_delays.main_functions import calculate_loss, get_scores\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "experts = [\n",
    "    \"knncad\",\n",
    "    \"numentaTM\",\n",
    "    \"twitterADVec\",\n",
    "    \"skyline\",\n",
    "    \"earthgeckoSkyline\",\n",
    "    \"numenta\",\n",
    "    \"bayesChangePt\",\n",
    "    \"null\",\n",
    "    \"expose\",\n",
    "    \"relativeEntropy\",\n",
    "    \"htmjava\",\n",
    "    \"randomCutForest\",\n",
    "    \"random\",\n",
    "    \"contextOSE\",\n",
    "    \"windowedGaussian\",\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read predictions from NAB and calculate predictions of Fixed-share and Variable-share"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [34:50<00:00, 298.67s/it]\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "folders = [\n",
    "    s\n",
    "    for s in os.listdir(\"NAB/results/numenta\")\n",
    "    if s.startswith(\"real\") or s.startswith(\"artificial\")\n",
    "]\n",
    "alpha_range = (0, 0.01, 0.05, 0.1, 0.3)\n",
    "share_range = (\"Fixed\", \"Variable\")\n",
    "for m, folder_name in enumerate(tqdm(folders)):\n",
    "    files = [\n",
    "        i.replace(\"numenta\", \"\")\n",
    "        for i in os.listdir(\n",
    "            os.path.join(\"NAB/results/numenta\", f\"{folder_name}\")\n",
    "        )\n",
    "    ]\n",
    "    for n, file_name in enumerate(files):\n",
    "        dt = read_nab(experts, folder_name, file_name)\n",
    "        score_experts = np.array(dt.filter(regex=\"^score\", axis=1))\n",
    "        assert score_experts.shape[1] == len(experts)\n",
    "        target = dt[\"label\"].values\n",
    "        delays_random = generate_random_delays(\n",
    "            max_length=dt.shape[0], min_delay=20, max_delay=100\n",
    "        )\n",
    "        delays_range = (1, 20, 50, 100, delays_random)\n",
    "        scores_share = get_scores(\n",
    "            target, score_experts, share_range, alpha_range, delays_range\n",
    "        )\n",
    "        dt = pd.merge(\n",
    "            dt, scores_share, left_index=True, right_index=True, validate=\"1:1\"\n",
    "        )\n",
    "        dt[\"file_name\"] = file_name\n",
    "        dt[\"folder_name\"] = folder_name\n",
    "        if (m == 0) & (n == 0):\n",
    "            results = dt.copy()\n",
    "        else:\n",
    "            results = pd.concat([results, dt], axis=0, sort=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.39 s, sys: 520 ms, total: 1.91 s\n",
      "Wall time: 1.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "alg_list = results.filter(regex=\"^score\", axis=1).columns.tolist()\n",
    "alg_list = [i.replace(\"score_\", \"\") for i in alg_list]\n",
    "losses_log = results[\n",
    "    [\"timestamp\", \"value\", \"label\", \"file_name\", \"folder_name\"]\n",
    "].copy()\n",
    "losses_square = results[\n",
    "    [\"timestamp\", \"value\", \"label\", \"file_name\", \"folder_name\"]\n",
    "].copy()\n",
    "for alg_ind in alg_list:\n",
    "    losses_log[f\"loss_{alg_ind}\"] = calculate_loss(\n",
    "        results[\"label\"].values,\n",
    "        results[f\"score_{alg_ind}\"].values,\n",
    "        share_type=\"Fixed\",\n",
    "    )\n",
    "    losses_square[f\"loss_{alg_ind}\"] = calculate_loss(\n",
    "        results[\"label\"].values,\n",
    "        results[f\"score_{alg_ind}\"].values,\n",
    "        share_type=\"Variable\",\n",
    "    )\n",
    "losses_log_total = losses_log.groupby([\"folder_name\", \"file_name\"])[\n",
    "    losses_log.filter(regex=\"^loss\", axis=1).columns.tolist()\n",
    "].sum()\n",
    "losses_square_total = losses_square.groupby([\"folder_name\", \"file_name\"])[\n",
    "    losses_square.filter(regex=\"^loss\", axis=1).columns.tolist()\n",
    "].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 15s, sys: 1.25 s, total: 1min 16s\n",
      "Wall time: 1min 16s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "if not os.path.exists(\"results\"):\n",
    "    os.makedirs(\"results\")\n",
    "results.to_csv(\"results/scores.csv\", index=False)\n",
    "losses_log.to_csv(\"results/losses_log.csv\", index=False)\n",
    "losses_square.to_csv(\"results/losses_square.csv\", index=False)\n",
    "losses_log_total.to_csv(\"results/losses_log_total.csv\")\n",
    "losses_square_total.to_csv(\"results/losses_square_total.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "anomaly_delays_github",
   "language": "python",
   "name": "anomaly_delays_github"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}