Identification of User Groups.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "db_train_df = pd.read_csv('data/db.train.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "db_val_df = pd.read_csv('data/db.valid.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "db_test_df = pd.read_csv('data/db.test.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "db_df = db_train_df.append(db_val_df).append(db_test_df)\n",
    "\n",
    "html_train_df = pd.read_csv('data/ht-ml.train.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "html_val_df = pd.read_csv('data/ht-ml.valid.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "html_test_df = pd.read_csv('data/ht-ml.test.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "html_df = html_train_df.append(html_val_df).append(html_test_df)\n",
    "\n",
    "ml_train_df = pd.read_csv('data/ml.train.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "ml_val_df = pd.read_csv('data/ml.valid.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "ml_test_df = pd.read_csv('data/ml.test.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "ml_df = ml_train_df.append(ml_val_df).append(ml_test_df)\n",
    "\n",
    "ciao_train_df = pd.read_csv('data/ciao.train.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "ciao_val_df = pd.read_csv('data/ciao.valid.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "ciao_test_df = pd.read_csv('data/ciao.test.rating', sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "ciao_df = ciao_train_df.append(ciao_val_df).append(ciao_test_df)\n",
    "\n",
    "jester_train_df = pd.read_csv(\"data/jester.train.rating\", sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "jester_val_df = pd.read_csv(\"data/jester.valid.rating\", sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "jester_test_df = pd.read_csv(\"data/jester.test.rating\", sep=\"\\t\", header=None, names=[\"user_id\", \"item_id\", \"rating\"])\n",
    "jester_df = jester_train_df.append(jester_val_df).append(jester_test_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Identify User Groups\n",
    "We identify three user groups: Users with the smallest amount of rating data ($Low$), users with most rating data ($High$) and users, whose amount of ratings is around the median ($Med$). These three user groups are constructed for five different datasets: Douban, Hetrec-MovieLens, MovieLens 1M, Ciao and Jester.\n",
    "## Douban"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Constructed low (125 users), med (125 users), high (125 users) usergroups.\n",
      "|U|: 375, |I|: 32191, |R|: 266517\n"
     ]
    }
   ],
   "source": [
    "user_popularity = db_df.groupby(\"user_id\").size().sort_values(ascending=True)\n",
    "n_5p_users = np.round(db_df[\"user_id\"].nunique() * 0.05).astype(int)\n",
    "low_users = user_popularity[:n_5p_users].index.tolist()\n",
    "med_users = np.abs(user_popularity - user_popularity.median()).sort_values(ascending=True)[:n_5p_users].index.tolist()\n",
    "high_users = user_popularity[-n_5p_users:].index.tolist()\n",
    "print(\"Constructed low (%d users), med (%d users), high (%d users) usergroups.\" % (len(low_users), len(med_users), len(high_users)))\n",
    "\n",
    "pd.DataFrame(low_users).to_csv(\"data/User Groups/db_low.userlist\", header=None, index=False)\n",
    "pd.DataFrame(med_users).to_csv(\"data/User Groups/db_med.userlist\", header=None, index=False)\n",
    "pd.DataFrame(high_users).to_csv(\"data/User Groups/db_high.userlist\", header=None, index=False)\n",
    "\n",
    "df = db_df[db_df[\"user_id\"].isin(low_users)].append(db_df[db_df[\"user_id\"].isin(med_users)]).append(db_df[db_df[\"user_id\"].isin(high_users)])\n",
    "df.to_csv(\"data/User Groups/db_ratings.txt\", index=False)\n",
    "\n",
    "n_users = df[\"user_id\"].nunique()\n",
    "n_items = df[\"item_id\"].nunique()\n",
    "n_ratings = len(df)\n",
    "print(\"|U|: %d, |I|: %d, |R|: %d\" % (n_users, n_items, n_ratings))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hetrec-MovieLens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Constructed low (106 users), med (106 users), high (106 users) usergroups.\n",
      "|U|: 318, |I|: 9553, |R|: 207943\n"
     ]
    }
   ],
   "source": [
    "user_popularity = html_df.groupby(\"user_id\").size().sort_values(ascending=True)\n",
    "n_5p_users = np.round(html_df[\"user_id\"].nunique() * 0.05).astype(int)\n",
    "low_users = user_popularity[:n_5p_users].index.tolist()\n",
    "med_users = np.abs(user_popularity - user_popularity.median()).sort_values(ascending=True)[:n_5p_users].index.tolist()\n",
    "high_users = user_popularity[-n_5p_users:].index.tolist()\n",
    "print(\"Constructed low (%d users), med (%d users), high (%d users) usergroups.\" % (len(low_users), len(med_users), len(high_users)))\n",
    "\n",
    "pd.DataFrame(low_users).to_csv(\"data/User Groups/ht-ml_low.userlist\", header=None, index=False)\n",
    "pd.DataFrame(med_users).to_csv(\"data/User Groups/ht-ml_med.userlist\", header=None, index=False)\n",
    "pd.DataFrame(high_users).to_csv(\"data/User Groups/ht-ml_high.userlist\", header=None, index=False)\n",
    "df = html_df[html_df[\"user_id\"].isin(low_users)].append(html_df[html_df[\"user_id\"].isin(med_users)]).append(html_df[html_df[\"user_id\"].isin(high_users)])\n",
    "df.to_csv(\"data/User Groups/ht-ml_ratings.txt\", index=False)\n",
    "\n",
    "n_users = df[\"user_id\"].nunique()\n",
    "n_items = df[\"item_id\"].nunique()\n",
    "n_ratings = len(df)\n",
    "print(\"|U|: %d, |I|: %d, |R|: %d\" % (n_users, n_items, n_ratings))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MovieLens 1M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Constructed low (302 users), med (302 users), high (302 users) usergroups.\n",
      "|U|: 906, |I|: 3613, |R|: 275119\n"
     ]
    }
   ],
   "source": [
    "user_popularity = ml_df.groupby(\"user_id\").size().sort_values(ascending=True)\n",
    "n_5p_users = np.round(ml_df[\"user_id\"].nunique() * 0.05).astype(int)\n",
    "low_users = user_popularity[:n_5p_users].index.tolist()\n",
    "med_users = np.abs(user_popularity - user_popularity.median()).sort_values(ascending=True)[:n_5p_users].index.tolist()\n",
    "high_users = user_popularity[-n_5p_users:].index.tolist()\n",
    "print(\"Constructed low (%d users), med (%d users), high (%d users) usergroups.\" % (len(low_users), len(med_users), len(high_users)))\n",
    "\n",
    "pd.DataFrame(low_users).to_csv(\"data/User Groups/ml_low.userlist\", header=None, index=False)\n",
    "pd.DataFrame(med_users).to_csv(\"data/User Groups/ml_med.userlist\", header=None, index=False)\n",
    "pd.DataFrame(high_users).to_csv(\"data/User Groups/ml_high.userlist\", header=None, index=False)\n",
    "df = ml_df[ml_df[\"user_id\"].isin(low_users)].append(ml_df[ml_df[\"user_id\"].isin(med_users)]).append(ml_df[ml_df[\"user_id\"].isin(high_users)])\n",
    "df.to_csv(\"data/User Groups/ml_ratings.txt\", index=False)\n",
    "\n",
    "n_users = df[\"user_id\"].nunique()\n",
    "n_items = df[\"item_id\"].nunique()\n",
    "n_ratings = len(df)\n",
    "print(\"|U|: %d, |I|: %d, |R|: %d\" % (n_users, n_items, n_ratings))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ciao"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Constructed low (369 users), med (369 users), high (369 users) usergroups.\n",
      "|U|: 1107, |I|: 60132, |R|: 107807\n"
     ]
    }
   ],
   "source": [
    "user_popularity = ciao_df.groupby(\"user_id\").size().sort_values(ascending=True)\n",
    "n_5p_users = np.round(ciao_df[\"user_id\"].nunique() * 0.05).astype(int)\n",
    "low_users = user_popularity[:n_5p_users].index.tolist()\n",
    "med_users = np.abs(user_popularity - user_popularity.median()).sort_values(ascending=True)[:n_5p_users].index.tolist()\n",
    "high_users = user_popularity[-n_5p_users:].index.tolist()\n",
    "print(\"Constructed low (%d users), med (%d users), high (%d users) usergroups.\" % (len(low_users), len(med_users), len(high_users)))\n",
    "\n",
    "pd.DataFrame(low_users).to_csv(\"data/User Groups/ciao_low.userlist\", header=None, index=False)\n",
    "pd.DataFrame(med_users).to_csv(\"data/User Groups/ciao_med.userlist\", header=None, index=False)\n",
    "pd.DataFrame(high_users).to_csv(\"data/User Groups/ciao_high.userlist\", header=None, index=False)\n",
    "df = ciao_df[ciao_df[\"user_id\"].isin(low_users)].append(ciao_df[ciao_df[\"user_id\"].isin(med_users)]).append(ciao_df[ciao_df[\"user_id\"].isin(high_users)])\n",
    "df.to_csv(\"data/User Groups/ciao_ratings.txt\", index=False)\n",
    "\n",
    "n_users = df[\"user_id\"].nunique()\n",
    "n_items = df[\"item_id\"].nunique()\n",
    "n_ratings = len(df)\n",
    "print(\"|U|: %d, |I|: %d, |R|: %d\" % (n_users, n_items, n_ratings))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Jester"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Constructed low (3671 users), med (3671 users), high (3671 users) usergroups.\n",
      "|U|: 11013, |I|: 100, |R|: 618768\n"
     ]
    }
   ],
   "source": [
    "user_popularity = jester_df.groupby(\"user_id\").size().sort_values(ascending=True)\n",
    "n_5p_users = np.round(jester_df[\"user_id\"].nunique() * 0.05).astype(int)\n",
    "low_users = user_popularity[:n_5p_users].index.tolist()\n",
    "med_users = np.abs(user_popularity - user_popularity.median()).sort_values(ascending=True)[:n_5p_users].index.tolist()\n",
    "high_users = user_popularity[-n_5p_users:].index.tolist()\n",
    "print(\"Constructed low (%d users), med (%d users), high (%d users) usergroups.\" % (len(low_users), len(med_users), len(high_users)))\n",
    "\n",
    "pd.DataFrame(low_users).to_csv(\"data/User Groups/jester_low.userlist\", header=None, index=False)\n",
    "pd.DataFrame(med_users).to_csv(\"data/User Groups/jester_med.userlist\", header=None, index=False)\n",
    "pd.DataFrame(high_users).to_csv(\"data/User Groups/jester_high.userlist\", header=None, index=False)\n",
    "df = jester_df[jester_df[\"user_id\"].isin(low_users)].append(jester_df[jester_df[\"user_id\"].isin(med_users)]).append(jester_df[jester_df[\"user_id\"].isin(high_users)])\n",
    "df.to_csv(\"data/User Groups/jester_ratings.txt\", index=False)\n",
    "\n",
    "n_users = df[\"user_id\"].nunique()\n",
    "n_items = df[\"item_id\"].nunique()\n",
    "n_ratings = len(df)\n",
    "print(\"|U|: %d, |I|: %d, |R|: %d\" % (n_users, n_items, n_ratings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}