Paper Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "\n",
    "import pandas as pd\n",
    "import logpred_method as experiment\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "# Use \"FRACTION = None\" for full dataset\n",
    "FRACTION: float = None\n",
    "\n",
    "\n",
    "# lr: Linear Regression\n",
    "# ab: Ada Boost\n",
    "# rf: Random Forest\n",
    "# dt: Decision Tree\n",
    "# et: Extra Trees\n",
    "MODELS = [\"lr\", \"ab\", \"rf\", \"dt\", \"et\"]\n",
    "\n",
    "# You can ignore features on the experiment\n",
    "IGNORED_FEATURES = [\"tryCatchQty_class\", \"tryCatchQty_method\"]\n",
    "\n",
    "\n",
    "# Hyperparameter tuning\n",
    "TUNING_ENABLED = False\n",
    "\n",
    "\n",
    "# Stores estimators and feature importances across experiments\n",
    "ESTIMATORS = {}\n",
    "FEATURE_IMPORTANCES = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_scores(scores):\n",
    "    \"\"\"\n",
    "    Returns a merged score from a sequence of scores.\n",
    "    This is useful to see scores as Pandas DataFrames.\n",
    "    \n",
    "    Example:\n",
    "        in  - [{\"a\": 1, \"b\": 2}, {\"a\": 10, \"b\": 20}]\n",
    "        out - {\"a\": [1, 10], \"b\": [2, 20]}\n",
    "    \"\"\"\n",
    "    merged = {k:[] for k in scores[0].keys()}\n",
    "    for score in scores:\n",
    "        for k, v in score.items():\n",
    "            merged[k].append(v)\n",
    "\n",
    "    return merged"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment CSV and Output directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_path = os.path.abspath(os.path.join(\"out\", \"dataset\", \"adyen-main\", \"dataset_full.csv\"))\n",
    "\n",
    "X, y = experiment.load_dataset(csv_path, drops=IGNORED_FEATURES, fraction=FRACTION)\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2, stratify=y, random_state=experiment.RANDOM_SEED\n",
    ")\n",
    "\n",
    "output_dir = os.path.abspath(os.path.join(\"out\", \"ml\", f\"evaluation-tuning-{TUNING_ENABLED}\"))\n",
    "if os.path.exists(output_dir):\n",
    "    shutil.rmtree(output_dir)\n",
    "os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RQ 1. What  is  the  performance  of  machine  learning  models in  predicting  log  placement  in  a  large-scale  enterprise system?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.dummy import DummyClassifier\n",
    "\n",
    "\n",
    "def rq1():\n",
    "    scores = []\n",
    "    for model in MODELS:\n",
    "        out = experiment.run(\n",
    "            model,\n",
    "            X_train=X_train,\n",
    "            X_test=X_test,\n",
    "            y_train=y_train,\n",
    "            y_test=y_test,\n",
    "            output_to=os.path.join(output_dir, f\"rq1-{model}.log\"),\n",
    "            tuning_enabled=TUNING_ENABLED\n",
    "        )\n",
    "        estimator, score, fi = out\n",
    "        scores.append(score)\n",
    "        \n",
    "        # Save to the global state this run\n",
    "        ESTIMATORS[model] = estimator\n",
    "        FEATURE_IMPORTANCES[model] = fi\n",
    "    \n",
    "    \n",
    "    # Dummy baselines\n",
    "    biased_guess = DummyClassifier(\n",
    "        strategy=\"stratified\",\n",
    "        random_state=experiment.RANDOM_SEED\n",
    "    )\n",
    "    biased_guess.fit(X_train, y_train)\n",
    "    bg_score = experiment.make_score(y_test, biased_guess.predict(X_test))\n",
    "    scores.append(bg_score)\n",
    "\n",
    "    random_guess = DummyClassifier(\n",
    "        strategy=\"uniform\",\n",
    "        random_state=experiment.RANDOM_SEED\n",
    "    )\n",
    "    random_guess.fit(X_train, y_train)\n",
    "    rg_score = experiment.make_score(y_test, random_guess.predict(X_test))\n",
    "    scores.append(rg_score)\n",
    "\n",
    "    return scores\n",
    "\n",
    "rq1_scores = rq1()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data alignment for later merging\n",
    "dummy_models = [\"BG\", \"RG\"]\n",
    "for i, baseline_score in enumerate(rq1_scores[-2:]):\n",
    "    baseline_score[\"balancing\"] = \"-\"\n",
    "    baseline_score[\"model\"] = dummy_models[i]\n",
    "    if \"mean_fit_time\" in rq1_scores[0].keys():\n",
    "        baseline_score[\"mean_fit_time\"] = 0\n",
    "        baseline_score[\"std_fit_time\"] = 0\n",
    "        baseline_score[\"mean_test_score\"] = 0\n",
    "        baseline_score[\"std_test_score\"] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>prec</th>\n",
       "      <th>recall</th>\n",
       "      <th>tn</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tp</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>lr</th>\n",
       "      <td>0.594921</td>\n",
       "      <td>0.515567</td>\n",
       "      <td>0.205901</td>\n",
       "      <td>56243</td>\n",
       "      <td>918</td>\n",
       "      <td>3768</td>\n",
       "      <td>977</td>\n",
       "      <td>61906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ab</th>\n",
       "      <td>0.699748</td>\n",
       "      <td>0.638728</td>\n",
       "      <td>0.419178</td>\n",
       "      <td>56036</td>\n",
       "      <td>1125</td>\n",
       "      <td>2756</td>\n",
       "      <td>1989</td>\n",
       "      <td>61906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rf</th>\n",
       "      <td>0.781477</td>\n",
       "      <td>0.802295</td>\n",
       "      <td>0.574710</td>\n",
       "      <td>56489</td>\n",
       "      <td>672</td>\n",
       "      <td>2018</td>\n",
       "      <td>2727</td>\n",
       "      <td>61906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dt</th>\n",
       "      <td>0.783883</td>\n",
       "      <td>0.592739</td>\n",
       "      <td>0.602107</td>\n",
       "      <td>55198</td>\n",
       "      <td>1963</td>\n",
       "      <td>1888</td>\n",
       "      <td>2857</td>\n",
       "      <td>61906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>et</th>\n",
       "      <td>0.778879</td>\n",
       "      <td>0.790766</td>\n",
       "      <td>0.570285</td>\n",
       "      <td>56445</td>\n",
       "      <td>716</td>\n",
       "      <td>2039</td>\n",
       "      <td>2706</td>\n",
       "      <td>61906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BG</th>\n",
       "      <td>0.501791</td>\n",
       "      <td>0.079933</td>\n",
       "      <td>0.080506</td>\n",
       "      <td>52764</td>\n",
       "      <td>4397</td>\n",
       "      <td>4363</td>\n",
       "      <td>382</td>\n",
       "      <td>61906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RG</th>\n",
       "      <td>0.498704</td>\n",
       "      <td>0.076281</td>\n",
       "      <td>0.497155</td>\n",
       "      <td>28595</td>\n",
       "      <td>28566</td>\n",
       "      <td>2386</td>\n",
       "      <td>2359</td>\n",
       "      <td>61906</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            acc      prec    recall     tn     fp    fn    tp  total\n",
       "model                                                               \n",
       "lr     0.594921  0.515567  0.205901  56243    918  3768   977  61906\n",
       "ab     0.699748  0.638728  0.419178  56036   1125  2756  1989  61906\n",
       "rf     0.781477  0.802295  0.574710  56489    672  2018  2727  61906\n",
       "dt     0.783883  0.592739  0.602107  55198   1963  1888  2857  61906\n",
       "et     0.778879  0.790766  0.570285  56445    716  2039  2706  61906\n",
       "BG     0.501791  0.079933  0.080506  52764   4397  4363   382  61906\n",
       "RG     0.498704  0.076281  0.497155  28595  28566  2386  2359  61906"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_rq1 = pd.DataFrame.from_dict(merge_scores(rq1_scores)).set_index([\"model\"])\n",
    "results_rq1.reset_index().to_csv(\n",
    "    os.path.join(output_dir, \"rq1-results.csv\"),\n",
    "    index=False,\n",
    ")\n",
    "results_rq1[\"acc prec recall tn fp fn tp total\".split(\" \")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RQ 2. What is the impact of different class balancing strategies on prediction?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Similar to rq1 but we include sampling in the experiment now.\n",
    "def rq2():\n",
    "    scores = []\n",
    "    for model in MODELS:\n",
    "        for balancing in [\"smote\", \"rus\"]:\n",
    "            out = experiment.run(\n",
    "                model,\n",
    "                X_train=X_train,\n",
    "                X_test=X_test,\n",
    "                y_train=y_train,\n",
    "                y_test=y_test,\n",
    "                balancing=balancing,\n",
    "                output_to=os.path.join(output_dir, f\"rq2-{model}-{balancing}.log\"),\n",
    "                tuning_enabled=TUNING_ENABLED\n",
    "            )\n",
    "            estimator, score, fi = out\n",
    "            scores.append(score)\n",
    "            \n",
    "            # Save to the global state this run\n",
    "            key = f\"{model}-{balancing}\"\n",
    "            ESTIMATORS[key] = estimator\n",
    "            FEATURE_IMPORTANCES[key] = fi\n",
    "\n",
    "    return scores\n",
    "\n",
    "rq2_scores = rq2()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>prec</th>\n",
       "      <th>recall</th>\n",
       "      <th>tn</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tp</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>balancing</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">lr</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.828068</td>\n",
       "      <td>0.249294</td>\n",
       "      <td>0.874816</td>\n",
       "      <td>44661</td>\n",
       "      <td>12500</td>\n",
       "      <td>594</td>\n",
       "      <td>4151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.811805</td>\n",
       "      <td>0.218075</td>\n",
       "      <td>0.887882</td>\n",
       "      <td>42055</td>\n",
       "      <td>15106</td>\n",
       "      <td>532</td>\n",
       "      <td>4213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ab</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.855593</td>\n",
       "      <td>0.413469</td>\n",
       "      <td>0.806112</td>\n",
       "      <td>51735</td>\n",
       "      <td>5426</td>\n",
       "      <td>920</td>\n",
       "      <td>3825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.887841</td>\n",
       "      <td>0.355694</td>\n",
       "      <td>0.912961</td>\n",
       "      <td>49314</td>\n",
       "      <td>7847</td>\n",
       "      <td>413</td>\n",
       "      <td>4332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">rf</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.843905</td>\n",
       "      <td>0.728216</td>\n",
       "      <td>0.709800</td>\n",
       "      <td>55904</td>\n",
       "      <td>1257</td>\n",
       "      <td>1377</td>\n",
       "      <td>3368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.915836</td>\n",
       "      <td>0.398340</td>\n",
       "      <td>0.950896</td>\n",
       "      <td>50346</td>\n",
       "      <td>6815</td>\n",
       "      <td>233</td>\n",
       "      <td>4512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">dt</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.794038</td>\n",
       "      <td>0.586350</td>\n",
       "      <td>0.624658</td>\n",
       "      <td>55070</td>\n",
       "      <td>2091</td>\n",
       "      <td>1781</td>\n",
       "      <td>2964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.877774</td>\n",
       "      <td>0.367381</td>\n",
       "      <td>0.881560</td>\n",
       "      <td>49958</td>\n",
       "      <td>7203</td>\n",
       "      <td>562</td>\n",
       "      <td>4183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">et</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.868622</td>\n",
       "      <td>0.677707</td>\n",
       "      <td>0.767545</td>\n",
       "      <td>55429</td>\n",
       "      <td>1732</td>\n",
       "      <td>1103</td>\n",
       "      <td>3642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.914761</td>\n",
       "      <td>0.384935</td>\n",
       "      <td>0.956375</td>\n",
       "      <td>49910</td>\n",
       "      <td>7251</td>\n",
       "      <td>207</td>\n",
       "      <td>4538</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      acc      prec    recall     tn     fp    fn    tp\n",
       "model balancing                                                        \n",
       "lr    smote      0.828068  0.249294  0.874816  44661  12500   594  4151\n",
       "      rus        0.811805  0.218075  0.887882  42055  15106   532  4213\n",
       "ab    smote      0.855593  0.413469  0.806112  51735   5426   920  3825\n",
       "      rus        0.887841  0.355694  0.912961  49314   7847   413  4332\n",
       "rf    smote      0.843905  0.728216  0.709800  55904   1257  1377  3368\n",
       "      rus        0.915836  0.398340  0.950896  50346   6815   233  4512\n",
       "dt    smote      0.794038  0.586350  0.624658  55070   2091  1781  2964\n",
       "      rus        0.877774  0.367381  0.881560  49958   7203   562  4183\n",
       "et    smote      0.868622  0.677707  0.767545  55429   1732  1103  3642\n",
       "      rus        0.914761  0.384935  0.956375  49910   7251   207  4538"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_rq2 = pd.DataFrame.from_dict(merge_scores(rq2_scores)).set_index([\"model\", \"balancing\"])\n",
    "results_rq2.reset_index().to_csv(\n",
    "    os.path.join(output_dir, \"rq2-results.csv\"),\n",
    "    index=False,\n",
    ")\n",
    "relevant_cols = \"acc prec recall tn fp fn tp\".split(\" \")\n",
    "results_rq2[relevant_cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparative result to the baseline (no balancing). Positive value indicates improvement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>prec</th>\n",
       "      <th>recall</th>\n",
       "      <th>tn</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tp</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>balancing</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">lr</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.233147</td>\n",
       "      <td>-0.266273</td>\n",
       "      <td>0.668915</td>\n",
       "      <td>-11582</td>\n",
       "      <td>11582</td>\n",
       "      <td>-3174</td>\n",
       "      <td>3174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.216885</td>\n",
       "      <td>-0.297492</td>\n",
       "      <td>0.681981</td>\n",
       "      <td>-14188</td>\n",
       "      <td>14188</td>\n",
       "      <td>-3236</td>\n",
       "      <td>3236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ab</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.155845</td>\n",
       "      <td>-0.225260</td>\n",
       "      <td>0.386934</td>\n",
       "      <td>-4301</td>\n",
       "      <td>4301</td>\n",
       "      <td>-1836</td>\n",
       "      <td>1836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.188093</td>\n",
       "      <td>-0.283034</td>\n",
       "      <td>0.493783</td>\n",
       "      <td>-6722</td>\n",
       "      <td>6722</td>\n",
       "      <td>-2343</td>\n",
       "      <td>2343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">rf</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.062428</td>\n",
       "      <td>-0.074079</td>\n",
       "      <td>0.135090</td>\n",
       "      <td>-585</td>\n",
       "      <td>585</td>\n",
       "      <td>-641</td>\n",
       "      <td>641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.134359</td>\n",
       "      <td>-0.403955</td>\n",
       "      <td>0.376185</td>\n",
       "      <td>-6143</td>\n",
       "      <td>6143</td>\n",
       "      <td>-1785</td>\n",
       "      <td>1785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">dt</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.010155</td>\n",
       "      <td>-0.006388</td>\n",
       "      <td>0.022550</td>\n",
       "      <td>-128</td>\n",
       "      <td>128</td>\n",
       "      <td>-107</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.093891</td>\n",
       "      <td>-0.225358</td>\n",
       "      <td>0.279452</td>\n",
       "      <td>-5240</td>\n",
       "      <td>5240</td>\n",
       "      <td>-1326</td>\n",
       "      <td>1326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">et</th>\n",
       "      <th>smote</th>\n",
       "      <td>0.089743</td>\n",
       "      <td>-0.113058</td>\n",
       "      <td>0.197260</td>\n",
       "      <td>-1016</td>\n",
       "      <td>1016</td>\n",
       "      <td>-936</td>\n",
       "      <td>936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rus</th>\n",
       "      <td>0.135882</td>\n",
       "      <td>-0.405831</td>\n",
       "      <td>0.386091</td>\n",
       "      <td>-6535</td>\n",
       "      <td>6535</td>\n",
       "      <td>-1832</td>\n",
       "      <td>1832</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      acc      prec    recall     tn     fp    fn    tp\n",
       "model balancing                                                        \n",
       "lr    smote      0.233147 -0.266273  0.668915 -11582  11582 -3174  3174\n",
       "      rus        0.216885 -0.297492  0.681981 -14188  14188 -3236  3236\n",
       "ab    smote      0.155845 -0.225260  0.386934  -4301   4301 -1836  1836\n",
       "      rus        0.188093 -0.283034  0.493783  -6722   6722 -2343  2343\n",
       "rf    smote      0.062428 -0.074079  0.135090   -585    585  -641   641\n",
       "      rus        0.134359 -0.403955  0.376185  -6143   6143 -1785  1785\n",
       "dt    smote      0.010155 -0.006388  0.022550   -128    128  -107   107\n",
       "      rus        0.093891 -0.225358  0.279452  -5240   5240 -1326  1326\n",
       "et    smote      0.089743 -0.113058  0.197260  -1016   1016  -936   936\n",
       "      rus        0.135882 -0.405831  0.386091  -6535   6535 -1832  1832"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_rq2_rel = results_rq2.loc[MODELS, relevant_cols] - results_rq1.loc[MODELS, relevant_cols]\n",
    "results_rq2_rel.reset_index().to_csv(\n",
    "    os.path.join(output_dir, \"rq2-results-relative.csv\"),\n",
    "    index=False\n",
    ")\n",
    "results_rq2_rel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RQ 3. What are the most recurring relevant features across models?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rank_to_df(rank, top=3):\n",
    "    cols = [\"total\"] + [i+1 for i in range(top)]\n",
    "    data = pd.DataFrame.from_records(\n",
    "        [(name, sum(count[:top]), *count[:top]) for name, count in rank.items()],\n",
    "        columns=[\"feature\"] + cols\n",
    "    )\n",
    "    return data[data[\"total\"] > 0].sort_values(by=cols, ascending=False)\n",
    "\n",
    "\n",
    "def feature_importance_rank(selected_models):\n",
    "    rank = {}\n",
    "    for model in selected_models:\n",
    "        ordered_features = sorted(\n",
    "            FEATURE_IMPORTANCES[model],\n",
    "            key=lambda pair: abs(pair[1]),\n",
    "            reverse=True\n",
    "        )\n",
    "        for pos, feature_pair, in enumerate(ordered_features):\n",
    "            feature = feature_pair[0]\n",
    "            if feature not in rank.keys():\n",
    "                rank[feature] = [0 for i in range(len(ordered_features))]\n",
    "            rank[feature][pos] += 1\n",
    "    return rank"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>total</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>maxNestedBlocks</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>loc_method</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>uniqueWordsQty_method</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>methodsInvokedQty</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>cbo_method</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dit</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>wmc_method</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>publicMethodsQty</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>returnsQty</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>variablesQty_method</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>rfc_method</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>publicFieldsQty</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>totalMethodsQty</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>cbo_class</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>parametersQty</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>abstractMethodsQty</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>methodsInvokedIndirectLocalQty</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>maxNestedBlocksQty</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>numbersQty_class</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>constructor_False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>assignmentsQty_method</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>stringLiteralsQty_method</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>loc_class</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           feature  total  1  2  3  4  5\n",
       "20                 maxNestedBlocks     12  8  2  2  0  0\n",
       "14                      loc_method     11  4  2  2  2  1\n",
       "15           uniqueWordsQty_method      7  0  4  1  2  0\n",
       "10               methodsInvokedQty      7  0  0  4  0  3\n",
       "23                      cbo_method      4  0  2  1  1  0\n",
       "3                              dit      3  1  1  0  1  0\n",
       "39                      wmc_method      3  0  2  0  1  0\n",
       "2                 publicMethodsQty      3  0  1  1  1  0\n",
       "41                      returnsQty      3  0  0  2  1  0\n",
       "13             variablesQty_method      3  0  0  1  0  2\n",
       "9                       rfc_method      3  0  0  0  2  1\n",
       "27                 publicFieldsQty      3  0  0  0  1  2\n",
       "0                  totalMethodsQty      2  1  0  1  0  0\n",
       "56                       cbo_class      2  0  0  0  1  1\n",
       "16                   parametersQty      1  1  0  0  0  0\n",
       "1               abstractMethodsQty      1  0  1  0  0  0\n",
       "31  methodsInvokedIndirectLocalQty      1  0  0  0  1  0\n",
       "44              maxNestedBlocksQty      1  0  0  0  1  0\n",
       "4                 numbersQty_class      1  0  0  0  0  1\n",
       "5                constructor_False      1  0  0  0  0  1\n",
       "24           assignmentsQty_method      1  0  0  0  0  1\n",
       "37        stringLiteralsQty_method      1  0  0  0  0  1\n",
       "49                       loc_class      1  0  0  0  0  1"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi = rank_to_df(\n",
    "    feature_importance_rank(\n",
    "        FEATURE_IMPORTANCES.keys()\n",
    "    ),\n",
    "    top=5\n",
    ")\n",
    "fi.to_csv(\n",
    "    os.path.join(output_dir, \"rq3-results.csv\"),\n",
    "    index=False\n",
    ")\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RQ 4. How  well  a  model  trained  with  open-source  data  can generalize  to  the  context  of  a  large-scale  enterprise system?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selected_apache_projects():\n",
    "    \"\"\"\n",
    "    Returns the name of the selected Apache projects as listed in the \"out/selection\" directory.\n",
    "    \"\"\"\n",
    "    selection_dir = os.path.abspath(os.path.join(\"out\", \"selection\"))\n",
    "    return sorted([\n",
    "        selected.replace(\".sh\", \"\")\n",
    "        for selected in os.listdir(selection_dir)\n",
    "        if selected.endswith(\".sh\")\n",
    "    ])\n",
    "\n",
    "\n",
    "def load_X_y(project: str):\n",
    "    dataset_path = os.path.abspath(\n",
    "        os.path.join(\"out\", \"dataset\", project, \"dataset_full.csv\")\n",
    "    )\n",
    "    X_apache, y_apache = experiment.load_dataset(\n",
    "        dataset_path, drops=IGNORED_FEATURES\n",
    "    )\n",
    "    assert X.shape[1] == X.shape[1]\n",
    "\n",
    "    return X_apache, y_apache\n",
    "\n",
    "\n",
    "APACHE_PROJECTS = {\n",
    "    project: load_X_y(project)\n",
    "    for project in selected_apache_projects()\n",
    "}\n",
    "\n",
    "assert len(APACHE_PROJECTS) == 29"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accumulo                 (25458, 63)\n",
      "ambari                   (21997, 63)\n",
      "archiva                   (5995, 63)\n",
      "bookkeeper               (12711, 63)\n",
      "cloudstack               (52390, 63)\n",
      "commons-beanutils         (1176, 63)\n",
      "cxf                      (33589, 63)\n",
      "fluo                      (2094, 63)\n",
      "giraph                    (8039, 63)\n",
      "helix                     (6787, 63)\n",
      "ignite                   (65181, 63)\n",
      "jmeter                    (8597, 63)\n",
      "knox                      (6821, 63)\n",
      "lens                      (6231, 63)\n",
      "metamodel                 (4122, 63)\n",
      "myfaces-tobago            (3866, 63)\n",
      "nutch                     (3321, 63)\n",
      "oodt                      (6933, 63)\n",
      "oozie                     (8821, 63)\n",
      "openmeetings              (4839, 63)\n",
      "reef                      (6150, 63)\n",
      "sqoop                     (3080, 63)\n",
      "storm                    (24208, 63)\n",
      "syncope                  (14915, 63)\n",
      "tez                       (8947, 63)\n",
      "thrift                    (1797, 63)\n",
      "tomcat                   (23789, 63)\n",
      "zeppelin                 (10953, 63)\n",
      "zookeeper                 (5279, 63)\n"
     ]
    }
   ],
   "source": [
    "for k, v in APACHE_PROJECTS.items():\n",
    "    print(f\"{k:20} {str(v[0].shape):>15}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning from all Apache projects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(388086, 63)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_apache_all = pd.concat(\n",
    "    [X_apache for X_apache, _ in APACHE_PROJECTS.values()],\n",
    "    ignore_index=True,\n",
    ")\n",
    "y_apache_all = pd.concat(\n",
    "    [y_apache for _, y_apache in APACHE_PROJECTS.values()],\n",
    "    ignore_index=True,\n",
    ")\n",
    "\n",
    "# Sum of entries must be equals to the number of final entries\n",
    "assert sum([X.shape[0] for X, _ in APACHE_PROJECTS.values()]) == X_apache_all.shape[0]\n",
    "\n",
    "# apache dataset size, all together\n",
    "X_apache_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rq4():\n",
    "    scores = []\n",
    "    model = \"rf\"\n",
    "    out = experiment.run(\n",
    "        model,\n",
    "        X_train=X_apache_all,\n",
    "        X_test=X_test,\n",
    "        y_train=y_apache_all,\n",
    "        y_test=y_test,\n",
    "        output_to=os.path.join(output_dir, f\"rq4-{model}-apache-all.log\"),\n",
    "        tuning_enabled=TUNING_ENABLED\n",
    "    )\n",
    "    estimator, score, fi = out\n",
    "    score[\"project\"] = \"apache-all\"\n",
    "    score[\"training_size\"] = X_apache_all.shape[0]\n",
    "    scores.append(score)\n",
    "\n",
    "    # Save to the global state this run\n",
    "    key = f\"{model}-apache-all\"\n",
    "    ESTIMATORS[key] = estimator\n",
    "    FEATURE_IMPORTANCES[key] = fi\n",
    "\n",
    "    return scores\n",
    "\n",
    "\n",
    "rq4_scores_all = rq4()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning from Projects Individually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rq4_individual():\n",
    "    scores = []\n",
    "    model = \"rf\"\n",
    "    for project, Xy in APACHE_PROJECTS.items():\n",
    "        out = experiment.run(\n",
    "            model,\n",
    "            X_train=Xy[0].drop(columns=[\"type\"]),\n",
    "            X_test=X_test.drop(columns=[\"type\"]),\n",
    "            y_train=Xy[1].drop(columns=[\"type\"]),\n",
    "            y_test=y_test.drop(columns=[\"type\"]),\n",
    "            output_to=os.path.join(output_dir, f\"rq4-{model}-{project}.log\"),\n",
    "            tuning_enabled=TUNING_ENABLED\n",
    "        )\n",
    "        estimator, score, fi = out\n",
    "        score[\"project\"] = project\n",
    "        score[\"training_size\"] = Xy[0].shape[0]\n",
    "        scores.append(score)\n",
    "\n",
    "        # Save to the global state this run\n",
    "        key = f\"{model}-{project}\"\n",
    "        ESTIMATORS[key] = estimator\n",
    "        FEATURE_IMPORTANCES[key] = fi\n",
    "\n",
    "    return scores\n",
    "\n",
    "\n",
    "rq4_scores_individual = rq4_individual()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prec</th>\n",
       "      <th>recall</th>\n",
       "      <th>acc</th>\n",
       "      <th>tn</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tp</th>\n",
       "      <th>total</th>\n",
       "      <th>project</th>\n",
       "      <th>training_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.645066</td>\n",
       "      <td>0.413277</td>\n",
       "      <td>0.697200</td>\n",
       "      <td>56082</td>\n",
       "      <td>1079</td>\n",
       "      <td>2784</td>\n",
       "      <td>1961</td>\n",
       "      <td>61906</td>\n",
       "      <td>cloudstack</td>\n",
       "      <td>52390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.613984</td>\n",
       "      <td>0.397893</td>\n",
       "      <td>0.688563</td>\n",
       "      <td>55974</td>\n",
       "      <td>1187</td>\n",
       "      <td>2857</td>\n",
       "      <td>1888</td>\n",
       "      <td>61906</td>\n",
       "      <td>zeppelin</td>\n",
       "      <td>10953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.637495</td>\n",
       "      <td>0.356164</td>\n",
       "      <td>0.669676</td>\n",
       "      <td>56200</td>\n",
       "      <td>961</td>\n",
       "      <td>3055</td>\n",
       "      <td>1690</td>\n",
       "      <td>61906</td>\n",
       "      <td>oodt</td>\n",
       "      <td>6933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.567834</td>\n",
       "      <td>0.351949</td>\n",
       "      <td>0.664857</td>\n",
       "      <td>55890</td>\n",
       "      <td>1271</td>\n",
       "      <td>3075</td>\n",
       "      <td>1670</td>\n",
       "      <td>61906</td>\n",
       "      <td>sqoop</td>\n",
       "      <td>3080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.636476</td>\n",
       "      <td>0.324341</td>\n",
       "      <td>0.654482</td>\n",
       "      <td>56282</td>\n",
       "      <td>879</td>\n",
       "      <td>3206</td>\n",
       "      <td>1539</td>\n",
       "      <td>61906</td>\n",
       "      <td>nutch</td>\n",
       "      <td>3321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.646845</td>\n",
       "      <td>0.300316</td>\n",
       "      <td>0.643353</td>\n",
       "      <td>56383</td>\n",
       "      <td>778</td>\n",
       "      <td>3320</td>\n",
       "      <td>1425</td>\n",
       "      <td>61906</td>\n",
       "      <td>helix</td>\n",
       "      <td>6787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.639578</td>\n",
       "      <td>0.293572</td>\n",
       "      <td>0.639920</td>\n",
       "      <td>56376</td>\n",
       "      <td>785</td>\n",
       "      <td>3352</td>\n",
       "      <td>1393</td>\n",
       "      <td>61906</td>\n",
       "      <td>bookkeeper</td>\n",
       "      <td>12711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.646917</td>\n",
       "      <td>0.283035</td>\n",
       "      <td>0.635106</td>\n",
       "      <td>56428</td>\n",
       "      <td>733</td>\n",
       "      <td>3402</td>\n",
       "      <td>1343</td>\n",
       "      <td>61906</td>\n",
       "      <td>jmeter</td>\n",
       "      <td>8597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.675096</td>\n",
       "      <td>0.258799</td>\n",
       "      <td>0.624230</td>\n",
       "      <td>56570</td>\n",
       "      <td>591</td>\n",
       "      <td>3517</td>\n",
       "      <td>1228</td>\n",
       "      <td>61906</td>\n",
       "      <td>archiva</td>\n",
       "      <td>5995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.644840</td>\n",
       "      <td>0.259431</td>\n",
       "      <td>0.623785</td>\n",
       "      <td>56483</td>\n",
       "      <td>678</td>\n",
       "      <td>3514</td>\n",
       "      <td>1231</td>\n",
       "      <td>61906</td>\n",
       "      <td>openmeetings</td>\n",
       "      <td>4839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.636946</td>\n",
       "      <td>0.251423</td>\n",
       "      <td>0.619763</td>\n",
       "      <td>56481</td>\n",
       "      <td>680</td>\n",
       "      <td>3552</td>\n",
       "      <td>1193</td>\n",
       "      <td>61906</td>\n",
       "      <td>storm</td>\n",
       "      <td>24208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.675942</td>\n",
       "      <td>0.245732</td>\n",
       "      <td>0.617976</td>\n",
       "      <td>56602</td>\n",
       "      <td>559</td>\n",
       "      <td>3579</td>\n",
       "      <td>1166</td>\n",
       "      <td>61906</td>\n",
       "      <td>zookeeper</td>\n",
       "      <td>5279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.647092</td>\n",
       "      <td>0.243836</td>\n",
       "      <td>0.616398</td>\n",
       "      <td>56530</td>\n",
       "      <td>631</td>\n",
       "      <td>3588</td>\n",
       "      <td>1157</td>\n",
       "      <td>61906</td>\n",
       "      <td>tez</td>\n",
       "      <td>8947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.613834</td>\n",
       "      <td>0.237513</td>\n",
       "      <td>0.612555</td>\n",
       "      <td>56452</td>\n",
       "      <td>709</td>\n",
       "      <td>3618</td>\n",
       "      <td>1127</td>\n",
       "      <td>61906</td>\n",
       "      <td>syncope</td>\n",
       "      <td>14915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.692065</td>\n",
       "      <td>0.231612</td>\n",
       "      <td>0.611529</td>\n",
       "      <td>56672</td>\n",
       "      <td>489</td>\n",
       "      <td>3646</td>\n",
       "      <td>1099</td>\n",
       "      <td>61906</td>\n",
       "      <td>ambari</td>\n",
       "      <td>21997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.527020</td>\n",
       "      <td>0.215806</td>\n",
       "      <td>0.599864</td>\n",
       "      <td>56242</td>\n",
       "      <td>919</td>\n",
       "      <td>3721</td>\n",
       "      <td>1024</td>\n",
       "      <td>61906</td>\n",
       "      <td>giraph</td>\n",
       "      <td>8039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.587156</td>\n",
       "      <td>0.202318</td>\n",
       "      <td>0.595255</td>\n",
       "      <td>56486</td>\n",
       "      <td>675</td>\n",
       "      <td>3785</td>\n",
       "      <td>960</td>\n",
       "      <td>61906</td>\n",
       "      <td>reef</td>\n",
       "      <td>6150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.725649</td>\n",
       "      <td>0.188409</td>\n",
       "      <td>0.591248</td>\n",
       "      <td>56823</td>\n",
       "      <td>338</td>\n",
       "      <td>3851</td>\n",
       "      <td>894</td>\n",
       "      <td>61906</td>\n",
       "      <td>accumulo</td>\n",
       "      <td>25458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.772853</td>\n",
       "      <td>0.176396</td>\n",
       "      <td>0.586046</td>\n",
       "      <td>56915</td>\n",
       "      <td>246</td>\n",
       "      <td>3908</td>\n",
       "      <td>837</td>\n",
       "      <td>61906</td>\n",
       "      <td>apache-all</td>\n",
       "      <td>388086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.688634</td>\n",
       "      <td>0.151949</td>\n",
       "      <td>0.573123</td>\n",
       "      <td>56835</td>\n",
       "      <td>326</td>\n",
       "      <td>4024</td>\n",
       "      <td>721</td>\n",
       "      <td>61906</td>\n",
       "      <td>commons-beanutils</td>\n",
       "      <td>1176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.717573</td>\n",
       "      <td>0.144573</td>\n",
       "      <td>0.569925</td>\n",
       "      <td>56891</td>\n",
       "      <td>270</td>\n",
       "      <td>4059</td>\n",
       "      <td>686</td>\n",
       "      <td>61906</td>\n",
       "      <td>myfaces-tobago</td>\n",
       "      <td>3866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.684275</td>\n",
       "      <td>0.117387</td>\n",
       "      <td>0.556445</td>\n",
       "      <td>56904</td>\n",
       "      <td>257</td>\n",
       "      <td>4188</td>\n",
       "      <td>557</td>\n",
       "      <td>61906</td>\n",
       "      <td>lens</td>\n",
       "      <td>6231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.766091</td>\n",
       "      <td>0.102845</td>\n",
       "      <td>0.550119</td>\n",
       "      <td>57012</td>\n",
       "      <td>149</td>\n",
       "      <td>4257</td>\n",
       "      <td>488</td>\n",
       "      <td>61906</td>\n",
       "      <td>oozie</td>\n",
       "      <td>8821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.722222</td>\n",
       "      <td>0.087671</td>\n",
       "      <td>0.542436</td>\n",
       "      <td>57001</td>\n",
       "      <td>160</td>\n",
       "      <td>4329</td>\n",
       "      <td>416</td>\n",
       "      <td>61906</td>\n",
       "      <td>knox</td>\n",
       "      <td>6821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.732075</td>\n",
       "      <td>0.081770</td>\n",
       "      <td>0.539643</td>\n",
       "      <td>57019</td>\n",
       "      <td>142</td>\n",
       "      <td>4357</td>\n",
       "      <td>388</td>\n",
       "      <td>61906</td>\n",
       "      <td>thrift</td>\n",
       "      <td>1797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.746959</td>\n",
       "      <td>0.064700</td>\n",
       "      <td>0.531440</td>\n",
       "      <td>57057</td>\n",
       "      <td>104</td>\n",
       "      <td>4438</td>\n",
       "      <td>307</td>\n",
       "      <td>61906</td>\n",
       "      <td>metamodel</td>\n",
       "      <td>4122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.803468</td>\n",
       "      <td>0.029294</td>\n",
       "      <td>0.514350</td>\n",
       "      <td>57127</td>\n",
       "      <td>34</td>\n",
       "      <td>4606</td>\n",
       "      <td>139</td>\n",
       "      <td>61906</td>\n",
       "      <td>ignite</td>\n",
       "      <td>65181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.875817</td>\n",
       "      <td>0.028240</td>\n",
       "      <td>0.513954</td>\n",
       "      <td>57142</td>\n",
       "      <td>19</td>\n",
       "      <td>4611</td>\n",
       "      <td>134</td>\n",
       "      <td>61906</td>\n",
       "      <td>cxf</td>\n",
       "      <td>33589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.609756</td>\n",
       "      <td>0.010537</td>\n",
       "      <td>0.504989</td>\n",
       "      <td>57129</td>\n",
       "      <td>32</td>\n",
       "      <td>4695</td>\n",
       "      <td>50</td>\n",
       "      <td>61906</td>\n",
       "      <td>fluo</td>\n",
       "      <td>2094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.735849</td>\n",
       "      <td>0.008219</td>\n",
       "      <td>0.503987</td>\n",
       "      <td>57147</td>\n",
       "      <td>14</td>\n",
       "      <td>4706</td>\n",
       "      <td>39</td>\n",
       "      <td>61906</td>\n",
       "      <td>tomcat</td>\n",
       "      <td>23789</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        prec    recall       acc     tn    fp    fn    tp  total  \\\n",
       "5   0.645066  0.413277  0.697200  56082  1079  2784  1961  61906   \n",
       "28  0.613984  0.397893  0.688563  55974  1187  2857  1888  61906   \n",
       "18  0.637495  0.356164  0.669676  56200   961  3055  1690  61906   \n",
       "22  0.567834  0.351949  0.664857  55890  1271  3075  1670  61906   \n",
       "17  0.636476  0.324341  0.654482  56282   879  3206  1539  61906   \n",
       "10  0.646845  0.300316  0.643353  56383   778  3320  1425  61906   \n",
       "4   0.639578  0.293572  0.639920  56376   785  3352  1393  61906   \n",
       "12  0.646917  0.283035  0.635106  56428   733  3402  1343  61906   \n",
       "3   0.675096  0.258799  0.624230  56570   591  3517  1228  61906   \n",
       "20  0.644840  0.259431  0.623785  56483   678  3514  1231  61906   \n",
       "23  0.636946  0.251423  0.619763  56481   680  3552  1193  61906   \n",
       "29  0.675942  0.245732  0.617976  56602   559  3579  1166  61906   \n",
       "25  0.647092  0.243836  0.616398  56530   631  3588  1157  61906   \n",
       "24  0.613834  0.237513  0.612555  56452   709  3618  1127  61906   \n",
       "2   0.692065  0.231612  0.611529  56672   489  3646  1099  61906   \n",
       "9   0.527020  0.215806  0.599864  56242   919  3721  1024  61906   \n",
       "21  0.587156  0.202318  0.595255  56486   675  3785   960  61906   \n",
       "1   0.725649  0.188409  0.591248  56823   338  3851   894  61906   \n",
       "0   0.772853  0.176396  0.586046  56915   246  3908   837  61906   \n",
       "6   0.688634  0.151949  0.573123  56835   326  4024   721  61906   \n",
       "16  0.717573  0.144573  0.569925  56891   270  4059   686  61906   \n",
       "14  0.684275  0.117387  0.556445  56904   257  4188   557  61906   \n",
       "19  0.766091  0.102845  0.550119  57012   149  4257   488  61906   \n",
       "13  0.722222  0.087671  0.542436  57001   160  4329   416  61906   \n",
       "26  0.732075  0.081770  0.539643  57019   142  4357   388  61906   \n",
       "15  0.746959  0.064700  0.531440  57057   104  4438   307  61906   \n",
       "11  0.803468  0.029294  0.514350  57127    34  4606   139  61906   \n",
       "7   0.875817  0.028240  0.513954  57142    19  4611   134  61906   \n",
       "8   0.609756  0.010537  0.504989  57129    32  4695    50  61906   \n",
       "27  0.735849  0.008219  0.503987  57147    14  4706    39  61906   \n",
       "\n",
       "              project  training_size  \n",
       "5          cloudstack          52390  \n",
       "28           zeppelin          10953  \n",
       "18               oodt           6933  \n",
       "22              sqoop           3080  \n",
       "17              nutch           3321  \n",
       "10              helix           6787  \n",
       "4          bookkeeper          12711  \n",
       "12             jmeter           8597  \n",
       "3             archiva           5995  \n",
       "20       openmeetings           4839  \n",
       "23              storm          24208  \n",
       "29          zookeeper           5279  \n",
       "25                tez           8947  \n",
       "24            syncope          14915  \n",
       "2              ambari          21997  \n",
       "9              giraph           8039  \n",
       "21               reef           6150  \n",
       "1            accumulo          25458  \n",
       "0          apache-all         388086  \n",
       "6   commons-beanutils           1176  \n",
       "16     myfaces-tobago           3866  \n",
       "14               lens           6231  \n",
       "19              oozie           8821  \n",
       "13               knox           6821  \n",
       "26             thrift           1797  \n",
       "15          metamodel           4122  \n",
       "11             ignite          65181  \n",
       "7                 cxf          33589  \n",
       "8                fluo           2094  \n",
       "27             tomcat          23789  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_rq4 = pd.DataFrame.from_dict(\n",
    "    merge_scores(\n",
    "        rq4_scores_all + rq4_scores_individual\n",
    "    )\n",
    ")\n",
    "results_rq4.to_csv(\n",
    "    os.path.join(output_dir, \"rq4-results.csv\"),\n",
    "    index=False\n",
    ")\n",
    "results_rq4.drop(columns=[\"model\", \"balancing\"]).sort_values(by=\"acc prec recall\".split(\" \"), ascending=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}