Train and Evaluate Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "from MetaMF import *\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime as dt\n",
    "import os.path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize random seeds and select GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(1)\n",
    "np.random.seed(1)\n",
    "torch.manual_seed(1) # set random seed for cpu\n",
    "torch.cuda.manual_seed(1) # set random seed for current gpu\n",
    "torch.cuda.manual_seed_all(1) # set random seed for all gpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    use_cuda = True\n",
    "    torch.cuda.set_device(0)\n",
    "else:\n",
    "    use_cuda = False\n",
    "print(\"CUDA available? \" + str(use_cuda))\n",
    "if use_cuda:\n",
    "    print(\"Current device: %d\" % torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utility Functions\n",
    "## Read Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_dataset(path):\n",
    "    trainset = pd.read_csv(path + \".train.rating\", sep=\"\\t\", header=None).to_records(index=False).tolist()\n",
    "    valset = pd.read_csv(path + \".valid.rating\", sep=\"\\t\", header=None).to_records(index=False).tolist()\n",
    "    testset = pd.read_csv(path + \".test.rating\", sep=\"\\t\", header=None).to_records(index=False).tolist()\n",
    "    \n",
    "    return trainset, valset, testset\n",
    "\n",
    "def read_usergroups(path):\n",
    "    low_users = pd.read_csv(path + \"_low.userlist\", header=None, squeeze=True).values.tolist()\n",
    "    med_users = pd.read_csv(path + \"_med.userlist\", header=None, squeeze=True).values.tolist()\n",
    "    high_users = pd.read_csv(path + \"_high.userlist\", header=None, squeeze=True).values.tolist()\n",
    "    \n",
    "    return low_users, med_users, high_users\n",
    "\n",
    "def read_useranditemlist(path):\n",
    "    userlist = pd.read_csv(path + \".userlist\", header=None, squeeze=True).values.tolist()\n",
    "    itemlist = pd.read_csv(path + \".itemlist\", header=None, squeeze=True).values.tolist()\n",
    "    \n",
    "    return userlist, itemlist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helpers for Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batchtoinput(batch, use_cuda):\n",
    "    users = []\n",
    "    items = []\n",
    "    ratings = []\n",
    "    for example in batch:\n",
    "        users.append(example[0])\n",
    "        items.append(example[1])\n",
    "        ratings.append(example[2])\n",
    "    users = torch.tensor(users, dtype=torch.int64)\n",
    "    items = torch.tensor(items, dtype=torch.int64)\n",
    "    ratings = torch.tensor(ratings, dtype=torch.float32)\n",
    "    if use_cuda:\n",
    "        users = users.cuda()\n",
    "        items = items.cuda()\n",
    "        ratings = ratings.cuda()\n",
    "    return users, items, ratings\n",
    "\n",
    "def getbatches(traindata, batch_size, use_cuda, shuffle):\n",
    "    dataset = traindata.copy()\n",
    "    if shuffle:\n",
    "        random.shuffle(dataset)\n",
    "    for batch_i in range(0,int(np.ceil(len(dataset)/batch_size))):\n",
    "        start_i = batch_i*batch_size\n",
    "        batch = dataset[start_i:start_i+batch_size]\n",
    "        yield batchtoinput(batch, use_cuda)\n",
    "        \n",
    "def weights_init(m):\n",
    "    classname = m.__class__.__name__\n",
    "    if classname.find('Linear') != -1:\n",
    "        nn.init.xavier_normal_(m.weight.data)\n",
    "        nn.init.constant_(m.bias.data, 0)\n",
    "        \n",
    "def get_eval(ratlist, predlist):\n",
    "    mae = np.mean(np.abs(ratlist-predlist))\n",
    "    mse = np.mean(np.square(ratlist-predlist))       \n",
    "    return  mae, mse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Other Functions\n",
    "## Sampling Procedure\n",
    "Here, we implement a sampling procedure to simulate a privacy budget $\\beta$. In detail, we randomly select a fraction of $\\beta$ of each user's rating data to be shared with the model. Thus, a user holds back a fraction of $1-\\beta$ of her data and provides only a fraction of $\\beta$ of her data for model training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sampling_procedure(dataset, beta):\n",
    "    dataframe = pd.DataFrame(dataset, columns=[\"user_id\", \"item_id\", \"rating\"])\n",
    "    n_samples = np.ceil(dataframe.groupby(\"user_id\").size() * (beta)).astype(int)\n",
    "    new_dataset = []\n",
    "    for uid, group in dataframe.groupby(\"user_id\"):\n",
    "        new_dataset.extend(group.sample(n=n_samples.loc[uid]).to_records(index=False).tolist())\n",
    "    return new_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Experiments\n",
    "This method trains and tests MetaMF and NoMetaMF under ten different privacy budgets (i.e., $\\beta \\in \\{1.0, 0.9, \\dots, 0.1\\}$). For NoMetaMF, meta learning the parameters of the rating prediction model can be disabled. Furthermore, we evaluate the model's accuracy in terms of the mean squared error and the mean absolute error on both, all users in the dataset and on our three user groups (i.e., $Low, Med, High$)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(path, traindata, valdata, testdata, userlist, itemlist, low, med, high, hyperparameters, betas=None, disable_meta_learning=False, save=False):\n",
    "    # default choice of privacy budget beta\n",
    "    if betas is None:\n",
    "        betas = [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1]\n",
    "        \n",
    "    if os.path.exists(path + \"/results.csv\"):\n",
    "        results_df = pd.read_csv(path + \"/results.csv\")\n",
    "    else:\n",
    "        results_df = pd.DataFrame()\n",
    "        \n",
    "    for beta in betas:\n",
    "        results_dict = {\"beta\": beta}\n",
    "        model_name = \"beta_\" + str(int(beta*100)) + \"p\"  \n",
    "        print(\"==========================\")\n",
    "        print(model_name)\n",
    "        print(\"==========================\")\n",
    "        starttime = dt.now()\n",
    "        \n",
    "        # sample a fraction of beta of each user's data to simulate privacy budget beta\n",
    "        R_train_beta = sampling_procedure(traindata, beta)\n",
    "        \n",
    "        train_loss, validation_loss = [], []\n",
    "        net = MetaMF(len(userlist), len(itemlist))\n",
    "        \n",
    "        # disable meta learning for NoMetaMF\n",
    "        if disable_meta_learning:\n",
    "            net.disable_meta_learning()\n",
    "        \n",
    "        # initialize parameters of neural network\n",
    "        net.apply(weights_init)\n",
    "        if use_cuda:\n",
    "            net.cuda()\n",
    "        \n",
    "        # model training\n",
    "        optimizer = optim.Adam(net.parameters(), lr=hyperparameters[\"lr\"], weight_decay=hyperparameters[\"lambda\"])\n",
    "        batch_size = hyperparameters[\"batch_size\"]\n",
    "        n_epochs = hyperparameters[\"n_epochs\"]\n",
    "        \n",
    "        for epoch in range(n_epochs):\n",
    "            net.train()\n",
    "            error = 0\n",
    "            num = 0\n",
    "            for k, (users, items, ratings) in enumerate(getbatches(R_train_beta, batch_size, use_cuda, True)):\n",
    "                optimizer.zero_grad()\n",
    "                pred = net(users, items)\n",
    "\n",
    "                loss = net.loss(pred, ratings)\n",
    "                loss.backward()\n",
    "                nn.utils.clip_grad_norm_(net.parameters(), 5)\n",
    "                optimizer.step()\n",
    "                error += loss.detach().cpu().numpy()*len(users)\n",
    "                num += len(users)\n",
    "            train_loss.append(error/num)\n",
    "            \n",
    "            # evaluate training error\n",
    "            net.eval()\n",
    "            groundtruth, estimation = [], []\n",
    "            for users, items, ratings in getbatches(valdata, batch_size, use_cuda, False):\n",
    "                predictions = net(users, items)\n",
    "                estimation.extend(predictions.tolist())\n",
    "                groundtruth.extend(ratings.tolist())\n",
    "            mae, mse = get_eval(np.array(groundtruth), np.array(estimation))\n",
    "            validation_loss.append(mse)\n",
    "            \n",
    "            print('Epoch {}/{} - Training Loss: {:.3f}, Validation Loss: {:.3f}, Time Elapsed: {}'.format(epoch+1, n_epochs, error/num, mse, dt.now()-starttime))\n",
    "            \n",
    "            if epoch+1 == n_epochs:\n",
    "                if save:\n",
    "                    torch.save(net, path + \"/\" + model_name + '.model')\n",
    "                    print(\"Saved Model to \" + path)\n",
    "                \n",
    "                results_dict[\"train_mse_all\"] = error / num\n",
    "                results_dict[\"val_mse_all\"] = mse\n",
    "        \n",
    "        # plot training and validation error to observe convergence\n",
    "        net.eval()\n",
    "        plt.figure()\n",
    "        plt.plot(range(n_epochs), train_loss, label=\"Train\")\n",
    "        plt.plot(range(n_epochs), validation_loss, label=\"Val\")\n",
    "        plt.legend()\n",
    "        plt.ylabel(\"MSE\")\n",
    "        plt.xlabel(\"Epoch\")\n",
    "        plt.tight_layout()\n",
    "        \n",
    "        # evaluate test error on both, all users in the dataset and on our three user groups\n",
    "        groundtruth, estimation = [], []\n",
    "        group_groundtruth = defaultdict(list)\n",
    "        group_estimation = defaultdict(list)\n",
    "        for users, items, ratings in getbatches(testdata, batch_size, use_cuda, False):\n",
    "            predictions = net(users, items)\n",
    "            estimation.extend(predictions.tolist())\n",
    "            groundtruth.extend(ratings.tolist())\n",
    "            \n",
    "            for uid, iid, r, p in zip(users.cpu().numpy(), items.cpu().numpy(), ratings.cpu().numpy(), predictions.detach().cpu().numpy()):\n",
    "                if uid in low:\n",
    "                    group_groundtruth[\"low\"].append(r)\n",
    "                    group_estimation[\"low\"].append(p)\n",
    "                elif uid in med:\n",
    "                    group_groundtruth[\"med\"].append(r)\n",
    "                    group_estimation[\"med\"].append(p)\n",
    "                elif uid in high:\n",
    "                    group_groundtruth[\"high\"].append(r)\n",
    "                    group_estimation[\"high\"].append(p)\n",
    "        \n",
    "        test_mae, test_mse = get_eval(np.array(groundtruth), np.array(estimation))\n",
    "        low_mae, low_mse = get_eval(np.array(group_groundtruth[\"low\"]), np.array(group_estimation[\"low\"]))\n",
    "        med_mae, med_mse = get_eval(np.array(group_groundtruth[\"med\"]), np.array(group_estimation[\"med\"]))\n",
    "        high_mae, high_mse = get_eval(np.array(group_groundtruth[\"high\"]), np.array(group_estimation[\"high\"]))\n",
    "        \n",
    "        results_dict[\"test_mse_all\"] = test_mse\n",
    "        results_dict[\"test_mae_all\"] = test_mae\n",
    "        results_dict[\"test_mse_low\"] = low_mse\n",
    "        results_dict[\"test_mae_low\"] = low_mae\n",
    "        results_dict[\"test_mse_med\"] = med_mse\n",
    "        results_dict[\"test_mae_med\"] = med_mae\n",
    "        results_dict[\"test_mse_high\"] = high_mse\n",
    "        results_dict[\"test_mae_high\"] = high_mae\n",
    "        \n",
    "        print(results_dict)\n",
    "        if save:\n",
    "            plt.savefig(path + \"/\" + model_name + \".png\", dpi=300)\n",
    "            results_df = results_df.append(pd.DataFrame([results_dict]))\n",
    "            results_df.to_csv(path + \"/results.csv\", index=False)\n",
    "            print(\"Saved Results to \" + path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the following lines, we load the entire Hetrec-MovieLens dataset and obtain two lists containing the user ids and the item ids respectively. Additionally, we conduct experiments on both, MetaMF and NoMetaMF under ten privacy budgets. The chosen hyperparameters are equal to those of Lin et. al."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, val, test = read_dataset(\"data/ht-ml\")\n",
    "users, items = read_useranditemlist(\"data/ht-ml\")\n",
    "low, med, high = read_usergroups(\"data/User Groups/ht-ml\")\n",
    "run(\"experiments/meta/ht-ml\", train, val, test, users, items, low, med, high, save=True, disable_meta_learning=False, \n",
    "    betas=[1],\n",
    "    hyperparameters={\"lr\": 0.0001, \"lambda\": 0.001, \"batch_size\": 64, \"n_epochs\": 100})\n",
    "\n",
    "run(\"experiments/nometa/ht-ml\", train, val, test, users, items, low, med, high, save=True, disable_meta_learning=True, \n",
    "    hyperparameters={\"lr\": 0.0001, \"lambda\": 0.001, \"batch_size\": 64, \"n_epochs\": 100})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}