prepare_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "varying-archives",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import pytorch_lightning as pl\n",
    "from tqdm import tqdm\n",
    "import torchmetrics\n",
    "import math\n",
    "from urllib.request import urlretrieve\n",
    "from zipfile import ZipFile\n",
    "import os\n",
    "import torch.nn as nn\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "subsequent-syndicate",
   "metadata": {},
   "source": [
    "## Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "looking-browse",
   "metadata": {},
   "outputs": [],
   "source": [
    "WINDOW_SIZE = 20"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "packed-prayer",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "blond-flight",
   "metadata": {},
   "outputs": [],
   "source": [
    "urlretrieve(\"http://files.grouplens.org/datasets/movielens/ml-1m.zip\", \"movielens.zip\")\n",
    "ZipFile(\"movielens.zip\", \"r\").extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "emerging-devon",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jiwidi/anaconda3/envs/cuda/lib/python3.7/site-packages/ipykernel_launcher.py:4: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  after removing the cwd from sys.path.\n",
      "/home/jiwidi/anaconda3/envs/cuda/lib/python3.7/site-packages/ipykernel_launcher.py:10: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/home/jiwidi/anaconda3/envs/cuda/lib/python3.7/site-packages/ipykernel_launcher.py:14: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "users = pd.read_csv(\n",
    "    \"ml-1m/users.dat\",\n",
    "    sep=\"::\",\n",
    "    names=[\"user_id\", \"sex\", \"age_group\", \"occupation\", \"zip_code\"],\n",
    ")\n",
    "\n",
    "ratings = pd.read_csv(\n",
    "    \"ml-1m/ratings.dat\",\n",
    "    sep=\"::\",\n",
    "    names=[\"user_id\", \"movie_id\", \"rating\", \"unix_timestamp\"],\n",
    ")\n",
    "\n",
    "movies = pd.read_csv(\n",
    "    \"ml-1m/movies.dat\", sep=\"::\", names=[\"movie_id\", \"title\", \"genres\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "three-picking",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Movies\n",
    "movies[\"year\"] = movies[\"title\"].apply(lambda x: x[-5:-1])\n",
    "movies.year = pd.Categorical(movies.year)\n",
    "movies[\"year\"] = movies.year.cat.codes\n",
    "## Users\n",
    "users.sex = pd.Categorical(users.sex)\n",
    "users[\"sex\"] = users.sex.cat.codes\n",
    "\n",
    "\n",
    "users.age_group = pd.Categorical(users.age_group)\n",
    "users[\"age_group\"] = users.age_group.cat.codes\n",
    "\n",
    "\n",
    "users.occupation = pd.Categorical(users.occupation)\n",
    "users[\"occupation\"] = users.occupation.cat.codes\n",
    "\n",
    "\n",
    "users.zip_code = pd.Categorical(users.zip_code)\n",
    "users[\"zip_code\"] = users.zip_code.cat.codes\n",
    "\n",
    "#Ratings\n",
    "ratings['unix_timestamp'] = pd.to_datetime(ratings['unix_timestamp'],unit='s')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "preliminary-raising",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save primary csv's\n",
    "if not os.path.exists('data'):\n",
    "    os.makedirs('data')\n",
    "    \n",
    "    \n",
    "users.to_csv(\"data/users.csv\",index=False)\n",
    "movies.to_csv(\"data/movies.csv\",index=False)\n",
    "ratings.to_csv(\"data/ratings.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "assisted-third",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Movies\n",
    "movies[\"movie_id\"] = movies[\"movie_id\"].astype(str)\n",
    "## Users\n",
    "users[\"user_id\"] = users[\"user_id\"].astype(str)\n",
    "\n",
    "##Ratings \n",
    "ratings[\"movie_id\"] = ratings[\"movie_id\"].astype(str)\n",
    "ratings[\"user_id\"] = ratings[\"user_id\"].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "satisfied-buyer",
   "metadata": {
    "id": "6_CC3yYCLxVN"
   },
   "outputs": [],
   "source": [
    "genres = [\n",
    "    \"Action\",\n",
    "    \"Adventure\",\n",
    "    \"Animation\",\n",
    "    \"Children's\",\n",
    "    \"Comedy\",\n",
    "    \"Crime\",\n",
    "    \"Documentary\",\n",
    "    \"Drama\",\n",
    "    \"Fantasy\",\n",
    "    \"Film-Noir\",\n",
    "    \"Horror\",\n",
    "    \"Musical\",\n",
    "    \"Mystery\",\n",
    "    \"Romance\",\n",
    "    \"Sci-Fi\",\n",
    "    \"Thriller\",\n",
    "    \"War\",\n",
    "    \"Western\",\n",
    "]\n",
    "\n",
    "for genre in genres:\n",
    "    movies[genre] = movies[\"genres\"].apply(\n",
    "        lambda values: int(genre in values.split(\"|\"))\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liquid-delight",
   "metadata": {
    "id": "0KsqW_4rLxVN"
   },
   "source": [
    "### Transform the movie ratings data into sequences\n",
    "\n",
    "First, let's sort the the ratings data using the `unix_timestamp`, and then group the\n",
    "`movie_id` values and the `rating` values by `user_id`.\n",
    "\n",
    "The output DataFrame will have a record for each `user_id`, with two ordered lists\n",
    "(sorted by rating datetime): the movies they have rated, and their ratings of these movies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "plastic-outdoors",
   "metadata": {
    "id": "D5v700zTLxVN"
   },
   "outputs": [],
   "source": [
    "ratings_group = ratings.sort_values(by=[\"unix_timestamp\"]).groupby(\"user_id\")\n",
    "\n",
    "ratings_data = pd.DataFrame(\n",
    "    data={\n",
    "        \"user_id\": list(ratings_group.groups.keys()),\n",
    "        \"movie_ids\": list(ratings_group.movie_id.apply(list)),\n",
    "        \"ratings\": list(ratings_group.rating.apply(list)),\n",
    "        \"timestamps\": list(ratings_group.unix_timestamp.apply(list)),\n",
    "    }\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "accessory-disorder",
   "metadata": {
    "id": "USa6rk0eLxVN"
   },
   "source": [
    "Now, let's split the `movie_ids` list into a set of sequences of a fixed length.\n",
    "We do the same for the `ratings`. Set the `sequence_length` variable to change the length\n",
    "of the input sequence to the model. You can also change the `step_size` to control the\n",
    "number of sequences to generate for each user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "shaped-reflection",
   "metadata": {
    "id": "XdhRJlxULxVN"
   },
   "outputs": [],
   "source": [
    "sequence_length = 8\n",
    "step_size = 1\n",
    "\n",
    "\n",
    "def create_sequences(values, window_size, step_size):\n",
    "    sequences = []\n",
    "    start_index = 0\n",
    "    while True:\n",
    "        end_index = start_index + window_size\n",
    "        seq = values[start_index:end_index]\n",
    "        if len(seq) < window_size:\n",
    "            seq = values[-window_size:]\n",
    "            if len(seq) == window_size:\n",
    "                sequences.append(seq)\n",
    "            break\n",
    "        sequences.append(seq)\n",
    "        start_index += step_size\n",
    "    return sequences\n",
    "\n",
    "\n",
    "ratings_data.movie_ids = ratings_data.movie_ids.apply(\n",
    "    lambda ids: create_sequences(ids, sequence_length, step_size)\n",
    ")\n",
    "\n",
    "ratings_data.ratings = ratings_data.ratings.apply(\n",
    "    lambda ids: create_sequences(ids, sequence_length, step_size)\n",
    ")\n",
    "\n",
    "del ratings_data[\"timestamps\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distinguished-soundtrack",
   "metadata": {
    "id": "5dYEduaqLxVN"
   },
   "source": [
    "After that, we process the output to have each sequence in a separate records in\n",
    "the DataFrame. In addition, we join the user features with the ratings data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "upper-upgrade",
   "metadata": {
    "id": "gM5_RBACLxVO"
   },
   "outputs": [],
   "source": [
    "ratings_data_movies = ratings_data[[\"user_id\", \"movie_ids\"]].explode(\n",
    "    \"movie_ids\", ignore_index=True\n",
    ")\n",
    "ratings_data_rating = ratings_data[[\"ratings\"]].explode(\"ratings\", ignore_index=True)\n",
    "ratings_data_transformed = pd.concat([ratings_data_movies, ratings_data_rating], axis=1)\n",
    "ratings_data_transformed = ratings_data_transformed.join(\n",
    "    users.set_index(\"user_id\"), on=\"user_id\"\n",
    ")\n",
    "ratings_data_transformed.movie_ids = ratings_data_transformed.movie_ids.apply(\n",
    "    lambda x: \",\".join(x)\n",
    ")\n",
    "ratings_data_transformed.ratings = ratings_data_transformed.ratings.apply(\n",
    "    lambda x: \",\".join([str(v) for v in x])\n",
    ")\n",
    "\n",
    "del ratings_data_transformed[\"zip_code\"]\n",
    "\n",
    "ratings_data_transformed.rename(\n",
    "    columns={\"movie_ids\": \"sequence_movie_ids\", \"ratings\": \"sequence_ratings\"},\n",
    "    inplace=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "green-inquiry",
   "metadata": {
    "id": "YnxbQ-wQLxVO"
   },
   "source": [
    "With `sequence_length` of 4 and `step_size` of 2, we end up with 498,623 sequences.\n",
    "\n",
    "Finally, we split the data into training and testing splits, with 85% and 15% of\n",
    "the instances, respectively, and store them to CSV files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "verified-berry",
   "metadata": {
    "id": "0lPMjBoRLxVO"
   },
   "outputs": [],
   "source": [
    "random_selection = np.random.rand(len(ratings_data_transformed.index)) <= 0.85\n",
    "train_data = ratings_data_transformed[random_selection]\n",
    "test_data = ratings_data_transformed[~random_selection]\n",
    "\n",
    "train_data.to_csv(\"data/train_data.csv\", index=False, sep=\",\")\n",
    "test_data.to_csv(\"data/test_data.csv\", index=False, sep=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "expired-kansas",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>sequence_movie_ids</th>\n",
       "      <th>sequence_ratings</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_group</th>\n",
       "      <th>occupation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1270,1022,2340,1836,3408,1207,2804,260</td>\n",
       "      <td>5,5,3,5,4,4,5,4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>1035,1962,1028,2018,150,1097,914,1287</td>\n",
       "      <td>5,4,5,4,5,4,3,5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>914,1287,2797,1246,2762,661,2918,531</td>\n",
       "      <td>3,5,4,4,4,3,4,4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>1</td>\n",
       "      <td>1029,2321,1197,594,2398,1545,527,745</td>\n",
       "      <td>5,3,3,4,4,4,5,3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1</td>\n",
       "      <td>594,2398,1545,527,745,595,588,1</td>\n",
       "      <td>4,4,4,5,3,5,4,5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963938</th>\n",
       "      <td>999</td>\n",
       "      <td>382,1027,1442,2146,358,2541,2431,1727</td>\n",
       "      <td>3,4,4,3,4,3,4,3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963946</th>\n",
       "      <td>999</td>\n",
       "      <td>3100,2320,996,3513,3109,3156,282,1888</td>\n",
       "      <td>4,3,2,3,3,4,4,3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963957</th>\n",
       "      <td>999</td>\n",
       "      <td>2875,2316,2165,361,2688,24,2264,1959</td>\n",
       "      <td>4,3,1,3,3,3,2,1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963960</th>\n",
       "      <td>999</td>\n",
       "      <td>361,2688,24,2264,1959,2676,2540,1363</td>\n",
       "      <td>3,3,3,2,1,3,2,3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963961</th>\n",
       "      <td>999</td>\n",
       "      <td>2688,24,2264,1959,2676,2540,1363,765</td>\n",
       "      <td>3,3,2,1,3,2,3,3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>144013 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id                      sequence_movie_ids sequence_ratings  sex  \\\n",
       "2            1  1270,1022,2340,1836,3408,1207,2804,260  5,5,3,5,4,4,5,4    0   \n",
       "19           1   1035,1962,1028,2018,150,1097,914,1287  5,4,5,4,5,4,3,5    0   \n",
       "25           1    914,1287,2797,1246,2762,661,2918,531  3,5,4,4,4,3,4,4    0   \n",
       "35           1    1029,2321,1197,594,2398,1545,527,745  5,3,3,4,4,4,5,3    0   \n",
       "38           1         594,2398,1545,527,745,595,588,1  4,4,4,5,3,5,4,5    0   \n",
       "...        ...                                     ...              ...  ...   \n",
       "963938     999   382,1027,1442,2146,358,2541,2431,1727  3,4,4,3,4,3,4,3    1   \n",
       "963946     999   3100,2320,996,3513,3109,3156,282,1888  4,3,2,3,3,4,4,3    1   \n",
       "963957     999    2875,2316,2165,361,2688,24,2264,1959  4,3,1,3,3,3,2,1    1   \n",
       "963960     999    361,2688,24,2264,1959,2676,2540,1363  3,3,3,2,1,3,2,3    1   \n",
       "963961     999    2688,24,2264,1959,2676,2540,1363,765  3,3,2,1,3,2,3,3    1   \n",
       "\n",
       "        age_group  occupation  \n",
       "2               0          10  \n",
       "19              0          10  \n",
       "25              0          10  \n",
       "35              0          10  \n",
       "38              0          10  \n",
       "...           ...         ...  \n",
       "963938          2          15  \n",
       "963946          2          15  \n",
       "963957          2          15  \n",
       "963960          2          15  \n",
       "963961          2          15  \n",
       "\n",
       "[144013 rows x 6 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "CUDAtorch",
   "language": "python",
   "name": "cudatorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}