EDA_mini.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:32.455768Z",
     "start_time": "2018-12-17T11:54:31.266888Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['track_features', 'training_set']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "# import dask\n",
    "# import dask.dataframe as dd\n",
    "\n",
    "import keggler as kg\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "import gc\n",
    "gc.enable()\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=Warning)\n",
    "\n",
    "import os, psutil\n",
    "\n",
    "# Set up a logger to dump messages to both log file and notebook\n",
    "import logging as logging\n",
    "def ini_log(filename):\n",
    "    logger = logging.getLogger(__name__)\n",
    "    ## avoid multiple printouts due to same handlers added several times\n",
    "    if not logger.handlers:\n",
    "        logger.setLevel(logging.DEBUG)\n",
    "\n",
    "        handlers = [logging.StreamHandler(None), logging.FileHandler(filename, 'a')]\n",
    "\n",
    "        fmt=logging.Formatter('%(asctime)-15s: %(levelname)s  %(message)s')\n",
    "        for h in handlers:\n",
    "            h.setFormatter(fmt)\n",
    "            logger.addHandler(h)\n",
    "    return logger\n",
    "        \n",
    "log = ini_log('out.log')\n",
    "\n",
    "PATH='data_mini/'\n",
    "\n",
    "print(os.listdir(PATH))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:49:59.083792Z",
     "start_time": "2018-12-17T11:49:59.080943Z"
    }
   },
   "source": [
    "# Read in training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.202201Z",
     "start_time": "2018-12-17T11:54:32.457868Z"
    }
   },
   "outputs": [],
   "source": [
    "df_trn = pd.read_csv(PATH+'training_set/log_mini.csv.gz', usecols=[i for i in range(21) if i != 16])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.275197Z",
     "start_time": "2018-12-17T11:54:33.203946Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index                               0.000076\n",
       "session_id                         15.209770\n",
       "session_position                    1.280823\n",
       "session_length                      1.280823\n",
       "track_id_clean                     15.209770\n",
       "skip_1                              0.160103\n",
       "skip_2                              0.160103\n",
       "skip_3                              0.160103\n",
       "not_skipped                         0.160103\n",
       "context_switch                      1.280823\n",
       "no_pause_before_play                1.280823\n",
       "short_pause_before_play             1.280823\n",
       "long_pause_before_play              1.280823\n",
       "hist_user_behavior_n_seekfwd        1.280823\n",
       "hist_user_behavior_n_seekback       1.280823\n",
       "hist_user_behavior_is_shuffle       0.160103\n",
       "hour_of_day                         1.280823\n",
       "premium                             0.160103\n",
       "context_type                       11.132299\n",
       "hist_user_behavior_reason_start    10.292255\n",
       "hist_user_behavior_reason_end      10.278235\n",
       "dtype: float64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.memory_usage(deep=True) / 1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.283496Z",
     "start_time": "2018-12-17T11:54:33.278221Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "session_id                         object\n",
       "session_position                    int64\n",
       "session_length                      int64\n",
       "track_id_clean                     object\n",
       "skip_1                               bool\n",
       "skip_2                               bool\n",
       "skip_3                               bool\n",
       "not_skipped                          bool\n",
       "context_switch                      int64\n",
       "no_pause_before_play                int64\n",
       "short_pause_before_play             int64\n",
       "long_pause_before_play              int64\n",
       "hist_user_behavior_n_seekfwd        int64\n",
       "hist_user_behavior_n_seekback       int64\n",
       "hist_user_behavior_is_shuffle        bool\n",
       "hour_of_day                         int64\n",
       "premium                              bool\n",
       "context_type                       object\n",
       "hist_user_behavior_reason_start    object\n",
       "hist_user_behavior_reason_end      object\n",
       "dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.314536Z",
     "start_time": "2018-12-17T11:54:33.286278Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>session_position</th>\n",
       "      <th>session_length</th>\n",
       "      <th>track_id_clean</th>\n",
       "      <th>skip_1</th>\n",
       "      <th>skip_2</th>\n",
       "      <th>skip_3</th>\n",
       "      <th>not_skipped</th>\n",
       "      <th>context_switch</th>\n",
       "      <th>no_pause_before_play</th>\n",
       "      <th>short_pause_before_play</th>\n",
       "      <th>long_pause_before_play</th>\n",
       "      <th>hist_user_behavior_n_seekfwd</th>\n",
       "      <th>hist_user_behavior_n_seekback</th>\n",
       "      <th>hist_user_behavior_is_shuffle</th>\n",
       "      <th>hour_of_day</th>\n",
       "      <th>premium</th>\n",
       "      <th>context_type</th>\n",
       "      <th>hist_user_behavior_reason_start</th>\n",
       "      <th>hist_user_behavior_reason_end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>t_0479f24c-27d2-46d6-a00c-7ec928f2b539</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>t_9099cd7b-c238-47b7-9381-f23f2c1d1043</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>t_fc5df5ba-5396-49a7-8b29-35d0d28249e0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>t_23cff8d6-d874-4b20-83dc-94e450e8aa20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>t_64f3743c-f624-46bb-a579-0f3f9a07a123</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>t_c815228b-3212-4f9e-9d4f-9cb19b248184</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>fwdbtn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>7</td>\n",
       "      <td>20</td>\n",
       "      <td>t_e23c19f5-4c32-4557-aa44-81372c2e3705</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>fwdbtn</td>\n",
       "      <td>fwdbtn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>t_0be6eced-f56f-48bd-8086-f2e0b760fdee</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>fwdbtn</td>\n",
       "      <td>fwdbtn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>t_f3ecbd3b-9e8e-4557-b8e0-39cfcd7e65dd</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>fwdbtn</td>\n",
       "      <td>fwdbtn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>10</td>\n",
       "      <td>20</td>\n",
       "      <td>t_2af4dfa0-7df3-4b7e-b7ab-353ba48237f9</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>fwdbtn</td>\n",
       "      <td>fwdbtn</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               session_id  session_position  session_length  \\\n",
       "0  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 1              20   \n",
       "1  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 2              20   \n",
       "2  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 3              20   \n",
       "3  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 4              20   \n",
       "4  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 5              20   \n",
       "5  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 6              20   \n",
       "6  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 7              20   \n",
       "7  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 8              20   \n",
       "8  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 9              20   \n",
       "9  0_00006f66-33e5-4de7-a324-2d18e439fc1e                10              20   \n",
       "\n",
       "                           track_id_clean  skip_1  skip_2  skip_3  \\\n",
       "0  t_0479f24c-27d2-46d6-a00c-7ec928f2b539   False   False   False   \n",
       "1  t_9099cd7b-c238-47b7-9381-f23f2c1d1043   False   False   False   \n",
       "2  t_fc5df5ba-5396-49a7-8b29-35d0d28249e0   False   False   False   \n",
       "3  t_23cff8d6-d874-4b20-83dc-94e450e8aa20   False   False   False   \n",
       "4  t_64f3743c-f624-46bb-a579-0f3f9a07a123   False   False   False   \n",
       "5  t_c815228b-3212-4f9e-9d4f-9cb19b248184   False   False    True   \n",
       "6  t_e23c19f5-4c32-4557-aa44-81372c2e3705    True    True    True   \n",
       "7  t_0be6eced-f56f-48bd-8086-f2e0b760fdee    True    True    True   \n",
       "8  t_f3ecbd3b-9e8e-4557-b8e0-39cfcd7e65dd   False    True    True   \n",
       "9  t_2af4dfa0-7df3-4b7e-b7ab-353ba48237f9    True    True    True   \n",
       "\n",
       "   not_skipped  context_switch  no_pause_before_play  short_pause_before_play  \\\n",
       "0         True               0                     0                        0   \n",
       "1         True               0                     1                        0   \n",
       "2         True               0                     1                        0   \n",
       "3         True               0                     1                        0   \n",
       "4         True               0                     1                        0   \n",
       "5        False               0                     1                        0   \n",
       "6        False               0                     1                        0   \n",
       "7        False               0                     1                        0   \n",
       "8        False               0                     1                        0   \n",
       "9        False               0                     1                        0   \n",
       "\n",
       "   long_pause_before_play  hist_user_behavior_n_seekfwd  \\\n",
       "0                       0                             0   \n",
       "1                       0                             0   \n",
       "2                       0                             0   \n",
       "3                       0                             0   \n",
       "4                       0                             0   \n",
       "5                       0                             0   \n",
       "6                       0                             0   \n",
       "7                       0                             0   \n",
       "8                       0                             0   \n",
       "9                       0                             0   \n",
       "\n",
       "   hist_user_behavior_n_seekback  hist_user_behavior_is_shuffle  hour_of_day  \\\n",
       "0                              0                           True           16   \n",
       "1                              0                           True           16   \n",
       "2                              0                           True           16   \n",
       "3                              0                           True           16   \n",
       "4                              0                           True           16   \n",
       "5                              0                           True           16   \n",
       "6                              0                           True           16   \n",
       "7                              0                           True           16   \n",
       "8                              0                           True           16   \n",
       "9                              0                           True           16   \n",
       "\n",
       "   premium        context_type hist_user_behavior_reason_start  \\\n",
       "0     True  editorial_playlist                       trackdone   \n",
       "1     True  editorial_playlist                       trackdone   \n",
       "2     True  editorial_playlist                       trackdone   \n",
       "3     True  editorial_playlist                       trackdone   \n",
       "4     True  editorial_playlist                       trackdone   \n",
       "5     True  editorial_playlist                       trackdone   \n",
       "6     True  editorial_playlist                          fwdbtn   \n",
       "7     True  editorial_playlist                          fwdbtn   \n",
       "8     True  editorial_playlist                          fwdbtn   \n",
       "9     True  editorial_playlist                          fwdbtn   \n",
       "\n",
       "  hist_user_behavior_reason_end  \n",
       "0                     trackdone  \n",
       "1                     trackdone  \n",
       "2                     trackdone  \n",
       "3                     trackdone  \n",
       "4                     trackdone  \n",
       "5                        fwdbtn  \n",
       "6                        fwdbtn  \n",
       "7                        fwdbtn  \n",
       "8                        fwdbtn  \n",
       "9                        fwdbtn  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reduce memory footprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.402111Z",
     "start_time": "2018-12-17T11:54:33.316612Z"
    }
   },
   "outputs": [],
   "source": [
    "enc = {\n",
    "    'hist_user_behavior_reason_start': {'trackdone': 1, 'fwdbtn': 2, 'trackerror': 8, 'remote': 7, 'clickrow': 4, 'backbtn': 3, 'playbtn': 6, 'appload': 5, 'endplay': 9},\n",
    "    'context_type': {'radio': 3, 'personalized_playlist': 4, 'charts': 6, 'user_collection': 2, 'editorial_playlist': 1, 'catalog': 5},\n",
    "    'hist_user_behavior_reason_end': {'trackdone': 1, 'fwdbtn': 2, 'logout': 5, 'clickrow': 7, 'backbtn': 3, 'endplay': 4, 'remote': 6}\n",
    "}\n",
    "\n",
    "for c in enc:\n",
    "    df_trn[c] = df_trn[c].map(enc[c]).fillna(0).astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.435101Z",
     "start_time": "2018-12-17T11:54:33.404020Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43.38794708251953"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.memory_usage(deep=True).sum()/1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.472189Z",
     "start_time": "2018-12-17T11:54:33.436548Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index                               0.000076\n",
       "session_id                         15.209770\n",
       "session_position                    1.280823\n",
       "session_length                      1.280823\n",
       "track_id_clean                     15.209770\n",
       "skip_1                              0.160103\n",
       "skip_2                              0.160103\n",
       "skip_3                              0.160103\n",
       "not_skipped                         0.160103\n",
       "context_switch                      1.280823\n",
       "no_pause_before_play                1.280823\n",
       "short_pause_before_play             1.280823\n",
       "long_pause_before_play              1.280823\n",
       "hist_user_behavior_n_seekfwd        1.280823\n",
       "hist_user_behavior_n_seekback       1.280823\n",
       "hist_user_behavior_is_shuffle       0.160103\n",
       "hour_of_day                         1.280823\n",
       "premium                             0.160103\n",
       "context_type                        0.160103\n",
       "hist_user_behavior_reason_start     0.160103\n",
       "hist_user_behavior_reason_end       0.160103\n",
       "dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.memory_usage(deep=True)/1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.481653Z",
     "start_time": "2018-12-17T11:54:33.474593Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "session_id                         object\n",
       "session_position                    int64\n",
       "session_length                      int64\n",
       "track_id_clean                     object\n",
       "skip_1                               bool\n",
       "skip_2                               bool\n",
       "skip_3                               bool\n",
       "not_skipped                          bool\n",
       "context_switch                      int64\n",
       "no_pause_before_play                int64\n",
       "short_pause_before_play             int64\n",
       "long_pause_before_play              int64\n",
       "hist_user_behavior_n_seekfwd        int64\n",
       "hist_user_behavior_n_seekback       int64\n",
       "hist_user_behavior_is_shuffle        bool\n",
       "hour_of_day                         int64\n",
       "premium                              bool\n",
       "context_type                        uint8\n",
       "hist_user_behavior_reason_start     uint8\n",
       "hist_user_behavior_reason_end       uint8\n",
       "dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.486061Z",
     "start_time": "2018-12-17T11:54:33.483698Z"
    }
   },
   "outputs": [],
   "source": [
    "# the code to generate `enc` mapping dictionary above\n",
    "\n",
    "# enc = {}\n",
    "# for c in ['context_type', 'hist_user_behavior_reason_start', 'hist_user_behavior_reason_end']:\n",
    "#     enc[c] = {}\n",
    "#     for i,f in enumerate(df_trn[c].unique()):\n",
    "#         enc[c][f] = i+1\n",
    "# print(enc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.739035Z",
     "start_time": "2018-12-17T11:54:33.488212Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 43.39 MB\n",
      "Memory usage after optimization is: 12.31 MB\n",
      "Decreased by 71.6%\n"
     ]
    }
   ],
   "source": [
    "df_trn = kg.reduce_mem_usage(df_trn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.747788Z",
     "start_time": "2018-12-17T11:54:33.741248Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "session_id                         category\n",
       "session_position                       int8\n",
       "session_length                         int8\n",
       "track_id_clean                     category\n",
       "skip_1                                uint8\n",
       "skip_2                                uint8\n",
       "skip_3                                uint8\n",
       "not_skipped                           uint8\n",
       "context_switch                         int8\n",
       "no_pause_before_play                   int8\n",
       "short_pause_before_play                int8\n",
       "long_pause_before_play                 int8\n",
       "hist_user_behavior_n_seekfwd           int8\n",
       "hist_user_behavior_n_seekback         int16\n",
       "hist_user_behavior_is_shuffle         uint8\n",
       "hour_of_day                            int8\n",
       "premium                               uint8\n",
       "context_type                          uint8\n",
       "hist_user_behavior_reason_start       uint8\n",
       "hist_user_behavior_reason_end         uint8\n",
       "dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:03:14.866890Z",
     "start_time": "2018-12-17T11:03:14.861479Z"
    }
   },
   "source": [
    "# Read in track features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:54:33.872661Z",
     "start_time": "2018-12-17T11:54:33.750196Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_mini/track_features/tf_mini.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls data_mini/track_features/tf_mini.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:13:51.071739Z",
     "start_time": "2018-12-17T12:13:50.819940Z"
    }
   },
   "outputs": [],
   "source": [
    "df_trk = [pd.read_csv(PATH+'track_features/tf_mini.csv', usecols=range(4), dtype={'release_year': np.uint32})]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:13:55.795577Z",
     "start_time": "2018-12-17T12:13:55.784142Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index                     0.000076\n",
       "track_id                  4.593735\n",
       "duration                  0.386841\n",
       "release_year              0.193420\n",
       "us_popularity_estimate    0.386841\n",
       "dtype: float64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trk.memory_usage(deep=True)/1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:13:57.394464Z",
     "start_time": "2018-12-17T12:13:57.280330Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 5.56 MB\n",
      "Memory usage after optimization is: 7.58 MB\n",
      "Decreased by -36.3%\n"
     ]
    }
   ],
   "source": [
    "df_trk = kg.reduce_mem_usage(df_trk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:13:58.428755Z",
     "start_time": "2018-12-17T12:13:58.418577Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>track_id</th>\n",
       "      <th>duration</th>\n",
       "      <th>release_year</th>\n",
       "      <th>us_popularity_estimate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>t_a540e552-16d4-42f8-a185-232bd650ea7d</td>\n",
       "      <td>109.6875</td>\n",
       "      <td>1950</td>\n",
       "      <td>100.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>t_67965da0-132b-4b1e-8a69-0ef99b32287c</td>\n",
       "      <td>187.7500</td>\n",
       "      <td>1950</td>\n",
       "      <td>100.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t_0614ecd3-a7d5-40a1-816e-156d5872a467</td>\n",
       "      <td>160.8750</td>\n",
       "      <td>1951</td>\n",
       "      <td>99.6250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>t_070a63a0-744a-434e-9913-a97b02926a29</td>\n",
       "      <td>175.3750</td>\n",
       "      <td>1951</td>\n",
       "      <td>99.6875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>t_d6990e17-9c31-4b01-8559-47d9ce476df1</td>\n",
       "      <td>369.5000</td>\n",
       "      <td>1951</td>\n",
       "      <td>100.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 track_id  duration  release_year  \\\n",
       "0  t_a540e552-16d4-42f8-a185-232bd650ea7d  109.6875          1950   \n",
       "1  t_67965da0-132b-4b1e-8a69-0ef99b32287c  187.7500          1950   \n",
       "2  t_0614ecd3-a7d5-40a1-816e-156d5872a467  160.8750          1951   \n",
       "3  t_070a63a0-744a-434e-9913-a97b02926a29  175.3750          1951   \n",
       "4  t_d6990e17-9c31-4b01-8559-47d9ce476df1  369.5000          1951   \n",
       "\n",
       "   us_popularity_estimate  \n",
       "0                100.0000  \n",
       "1                100.0000  \n",
       "2                 99.6250  \n",
       "3                 99.6875  \n",
       "4                100.0000  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trk.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:14:03.311689Z",
     "start_time": "2018-12-17T12:14:03.294211Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index                     0.000076\n",
       "track_id                  7.287155\n",
       "duration                  0.096710\n",
       "release_year              0.096710\n",
       "us_popularity_estimate    0.096710\n",
       "dtype: float64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trk.memory_usage(deep=True)/1024**2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:53:39.360659Z",
     "start_time": "2018-12-17T11:53:39.355981Z"
    }
   },
   "source": [
    "# Merge DS with Track features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:38:15.790940Z",
     "start_time": "2018-12-17T12:38:15.598379Z"
    }
   },
   "outputs": [],
   "source": [
    "x = df_trn.merge(df_trk, how='left', right_on='track_id', left_on='track_id_clean').drop(['track_id_clean', 'track_id'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:38:17.863522Z",
     "start_time": "2018-12-17T12:38:17.858425Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "session_id                         category\n",
       "session_position                       int8\n",
       "session_length                         int8\n",
       "skip_1                                uint8\n",
       "skip_2                                uint8\n",
       "skip_3                                uint8\n",
       "not_skipped                           uint8\n",
       "context_switch                         int8\n",
       "no_pause_before_play                   int8\n",
       "short_pause_before_play                int8\n",
       "long_pause_before_play                 int8\n",
       "hist_user_behavior_n_seekfwd           int8\n",
       "hist_user_behavior_n_seekback         int16\n",
       "hist_user_behavior_is_shuffle         uint8\n",
       "hour_of_day                            int8\n",
       "premium                               uint8\n",
       "context_type                          uint8\n",
       "hist_user_behavior_reason_start       uint8\n",
       "hist_user_behavior_reason_end         uint8\n",
       "duration                            float16\n",
       "release_year                         uint16\n",
       "us_popularity_estimate              float16\n",
       "dtype: object"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:38:21.246525Z",
     "start_time": "2018-12-17T12:38:21.239230Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.822090148925781"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.memory_usage(deep=True).sum()/1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:55:54.541755Z",
     "start_time": "2018-12-17T12:55:54.536227Z"
    }
   },
   "outputs": [],
   "source": [
    "id_trn = df_trn['session_id'].cat.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:56:44.018546Z",
     "start_time": "2018-12-17T12:56:44.012284Z"
    }
   },
   "outputs": [],
   "source": [
    "id_trn_1 = id_trn[:5000]\n",
    "id_trn_2 = id_trn[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T12:57:41.366816Z",
     "start_time": "2018-12-17T12:57:41.323768Z"
    }
   },
   "outputs": [],
   "source": [
    "df_trn_1 = df_trn.query('session_id in @id_trn_1')\n",
    "df_trn_2 = df_trn.query('session_id in @id_trn_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T13:13:13.819207Z",
     "start_time": "2018-12-17T13:13:09.951870Z"
    }
   },
   "outputs": [],
   "source": [
    "df_trn_1.to_csv(PATH+'training_set/log_mini_1.csv.gz', index=False, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T13:13:17.572839Z",
     "start_time": "2018-12-17T13:13:13.821151Z"
    }
   },
   "outputs": [],
   "source": [
    "df_trn_2.to_csv(PATH+'training_set/log_mini_2.csv.gz', index=False, header=False, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}