experiments_ml.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MovieLens Experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can load the saved models and reproduce the results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>METRIC</th>\n",
       "      <th>MODEL</th>\n",
       "      <th>CLOSED</th>\n",
       "      <th>IPS</th>\n",
       "      <th>UNBIASED</th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>520</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>GA</td>\n",
       "      <td>0.259</td>\n",
       "      <td>0.144</td>\n",
       "      <td>0.255</td>\n",
       "      <td>0.257</td>\n",
       "      <td>0.102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>521</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>MPOP</td>\n",
       "      <td>0.373</td>\n",
       "      <td>0.214</td>\n",
       "      <td>0.344</td>\n",
       "      <td>0.340</td>\n",
       "      <td>0.696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>BaselineOnly</td>\n",
       "      <td>0.297</td>\n",
       "      <td>0.158</td>\n",
       "      <td>0.291</td>\n",
       "      <td>0.293</td>\n",
       "      <td>0.156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>MLP</td>\n",
       "      <td>0.414</td>\n",
       "      <td>0.237</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.401</td>\n",
       "      <td>0.341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>MF10</td>\n",
       "      <td>0.323</td>\n",
       "      <td>0.171</td>\n",
       "      <td>0.316</td>\n",
       "      <td>0.318</td>\n",
       "      <td>0.183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF60</td>\n",
       "      <td>0.385</td>\n",
       "      <td>0.220</td>\n",
       "      <td>0.377</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>620</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF70</td>\n",
       "      <td>0.382</td>\n",
       "      <td>0.218</td>\n",
       "      <td>0.374</td>\n",
       "      <td>0.376</td>\n",
       "      <td>0.205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>621</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF80</td>\n",
       "      <td>0.376</td>\n",
       "      <td>0.215</td>\n",
       "      <td>0.369</td>\n",
       "      <td>0.371</td>\n",
       "      <td>0.193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>622</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF90</td>\n",
       "      <td>0.369</td>\n",
       "      <td>0.209</td>\n",
       "      <td>0.362</td>\n",
       "      <td>0.364</td>\n",
       "      <td>0.183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623</th>\n",
       "      <td>NDCG@-1</td>\n",
       "      <td>NeuMF100</td>\n",
       "      <td>0.368</td>\n",
       "      <td>0.209</td>\n",
       "      <td>0.361</td>\n",
       "      <td>0.363</td>\n",
       "      <td>0.184</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>104 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      METRIC         MODEL CLOSED    IPS UNBIASED     Q1     Q2\n",
       "520  NDCG@-1            GA  0.259  0.144    0.255  0.257  0.102\n",
       "521  NDCG@-1          MPOP  0.373  0.214    0.344  0.340  0.696\n",
       "522  NDCG@-1  BaselineOnly  0.297  0.158    0.291  0.293  0.156\n",
       "523  NDCG@-1           MLP  0.414  0.237    0.400  0.401  0.341\n",
       "524  NDCG@-1          MF10  0.323  0.171    0.316  0.318  0.183\n",
       "..       ...           ...    ...    ...      ...    ...    ...\n",
       "619  NDCG@-1       NeuMF60  0.385  0.220    0.377  0.379  0.203\n",
       "620  NDCG@-1       NeuMF70  0.382  0.218    0.374  0.376  0.205\n",
       "621  NDCG@-1       NeuMF80  0.376  0.215    0.369  0.371  0.193\n",
       "622  NDCG@-1       NeuMF90  0.369  0.209    0.362  0.364  0.183\n",
       "623  NDCG@-1      NeuMF100  0.368  0.209    0.361  0.363  0.184\n",
       "\n",
       "[104 rows x 7 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "import qgrid\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from scipy.stats import kendalltau, ttest_rel\n",
    "from utils import natural_keys\n",
    "\n",
    "\n",
    "# load results\n",
    "with open('./data/exp_stra_ml.pkl', 'rb') as exp_file:\n",
    "    res_stra = pickle.load(exp_file)\n",
    "\n",
    "    \n",
    "def get_metric_value(results, model_name, metric, user_base=False):\n",
    "\n",
    "    for result in results:\n",
    "        if result.model_name == model_name:\n",
    "            if user_base:\n",
    "                return result.metric_user_results[metric]\n",
    "            else:\n",
    "                return result.metric_avg_results[metric]\n",
    "\n",
    "    \n",
    "METRICS = sorted([m for m in res_stra[0][0].metric_avg_results.keys() if 'SIZE' not in m], \n",
    "                 key=natural_keys)\n",
    "\n",
    "MODELS  = [res[0].model_name for res in res_stra]\n",
    "\n",
    "\n",
    "idx = 0\n",
    "\n",
    "df = pd.DataFrame(columns=['METRIC', 'MODEL', 'CLOSED', 'IPS', 'UNBIASED', 'Q1', 'Q2'])\n",
    "\n",
    "for metric in METRICS:\n",
    "        \n",
    "    for model in MODELS:\n",
    "        \n",
    "        closed = get_metric_value([res_array[0] for res_array in res_stra], model, metric)\n",
    "        ips = get_metric_value([res_array[1] for res_array in res_stra], model, metric)\n",
    "        q1 = get_metric_value([res_array[3] for res_array in res_stra], model, metric)\n",
    "        q2 = get_metric_value([res_array[4] for res_array in res_stra], model, metric)\n",
    "        unbiased = get_metric_value([res_array[5] for res_array in res_stra], model, metric)\n",
    "        \n",
    "        df.loc[idx] = [metric, model, \n",
    "                       '%.3f' %(closed), \n",
    "                       '%.3f' %(ips),\n",
    "                       '%.3f' %(unbiased), \n",
    "                       '%.3f' %(q1), \n",
    "                       '%.3f' %(q2)]\n",
    "        idx += 1\n",
    "    \n",
    "\n",
    "\n",
    "df[df.METRIC == 'NDCG@-1']\n",
    "\n",
    "\n",
    "# use the following to interact with the table\n",
    "# qgrid.show_grid(df[df.METRIC == 'NDCG@-1'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you want to train all the models from scratch, use the following block (it takes a long time to compete the training/evaluation of all available models):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "from cornac.datasets import movielens\n",
    "from eval_methods.stratified_evaluation import StratifiedEvaluation\n",
    "from experiment.experiment import STExperiment\n",
    "from utils import get_models, get_metrics\n",
    "\n",
    "\n",
    "# load the movielens dataset\n",
    "ml = movielens.load_feedback(variant=\"1M\")\n",
    "\n",
    "\n",
    "# propensity-based stratified evaluation\n",
    "stra_eval_method = StratifiedEvaluation(data=ml,\n",
    "                                        n_strata=2,\n",
    "                                        rating_threshold=4.0,\n",
    "                                        verbose=True)\n",
    "\n",
    "# run the experiment\n",
    "exp_stra = STExperiment(eval_method=stra_eval_method,\n",
    "                        models=get_models(variant='large'),\n",
    "                        metrics=get_metrics(variant='large'),\n",
    "                        verbose=True)\n",
    "\n",
    "exp_stra.run()\n",
    "\n",
    "with open('./data/exp_stra_ml.pkl', 'wb') as exp_file:\n",
    "    pickle.dump(exp_stra.result, exp_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}