Preprocess_Pandas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:28:40.407115Z",
     "start_time": "2018-12-17T20:28:39.448320Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['20181113_submissions.tar.gz', '20181113_test_set.tar.gz', 'submissions', 'track_features', 'training_set', 'training_set_0.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import keggler as kg\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "\n",
    "import gc\n",
    "gc.enable()\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=Warning)\n",
    "\n",
    "import os, psutil\n",
    "\n",
    "# Set up a logger to dump messages to both log file and notebook\n",
    "import logging as logging\n",
    "def ini_log(filename):\n",
    "    logger = logging.getLogger(__name__)\n",
    "    ## avoid multiple printouts due to same handlers added several times\n",
    "    if not logger.handlers:\n",
    "        logger.setLevel(logging.DEBUG)\n",
    "\n",
    "        handlers = [#logging.StreamHandler(None), \n",
    "            logging.FileHandler(filename, 'a')\n",
    "        ]\n",
    "\n",
    "        fmt=logging.Formatter('%(asctime)-15s: %(levelname)s  %(message)s')\n",
    "        for h in handlers:\n",
    "            h.setFormatter(fmt)\n",
    "            logger.addHandler(h)\n",
    "    return logger\n",
    "        \n",
    "log = ini_log('out.log')\n",
    "\n",
    "#PATH='data_mini/'\n",
    "#prefix='_mini'\n",
    "\n",
    "PATH='data/'\n",
    "prefix=''\n",
    "\n",
    "print(os.listdir(PATH))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:03:14.866890Z",
     "start_time": "2018-12-17T11:03:14.861479Z"
    }
   },
   "source": [
    "# Read in track features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:28:40.550564Z",
     "start_time": "2018-12-17T20:28:40.545228Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['._tf_000000000000.csv',\n",
       " '._tf_000000000001.csv',\n",
       " 'tf_000000000000.csv.gz',\n",
       " 'tf_000000000001.csv.gz']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from os import listdir\n",
    "listdir(PATH+'track_features/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:29:18.915300Z",
     "start_time": "2018-12-17T20:28:41.278560Z"
    }
   },
   "outputs": [],
   "source": [
    "df_trk = pd.concat([pd.read_csv(PATH+'track_features/'+f, \n",
    "                     usecols=range(4), \n",
    "                     dtype={'release_year': np.uint32},\n",
    "                     compression='gzip'\n",
    "                    ) for f in listdir(PATH+'track_features/') if f.startswith('tf_')\n",
    "         ], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:29:19.204915Z",
     "start_time": "2018-12-17T20:29:18.917902Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index                      28.277496\n",
       "track_id                  335.795269\n",
       "duration                   28.277496\n",
       "release_year               14.138748\n",
       "us_popularity_estimate     28.277496\n",
       "dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trk.memory_usage(deep=True)/1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:29:19.209526Z",
     "start_time": "2018-12-17T20:29:19.206550Z"
    }
   },
   "outputs": [],
   "source": [
    "# df_trk = kg.reduce_mem_usage(df_trk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:29:19.215906Z",
     "start_time": "2018-12-17T20:29:19.211800Z"
    }
   },
   "outputs": [],
   "source": [
    "# df_trk.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:53:39.360659Z",
     "start_time": "2018-12-17T11:53:39.355981Z"
    }
   },
   "source": [
    "# Merge DS with Track features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:41:36.334939Z",
     "start_time": "2018-12-17T20:41:36.327773Z"
    }
   },
   "outputs": [],
   "source": [
    "def preprocess_data(fin_, path_, df_trk_):\n",
    "    df_trn = pd.read_csv(path_ + '/' + fin_,\n",
    "                         usecols=[i for i in range(21) if i != 16]\n",
    "                        )\n",
    "    enc = {\n",
    "        'hist_user_behavior_reason_start': {'trackdone': 1, 'fwdbtn': 2, 'trackerror': 8, 'remote': 7, 'clickrow': 4, 'backbtn': 3, 'playbtn': 6, 'appload': 5, 'endplay': 9},\n",
    "        'context_type': {'radio': 3, 'personalized_playlist': 4, 'charts': 6, 'user_collection': 2, 'editorial_playlist': 1, 'catalog': 5},\n",
    "        'hist_user_behavior_reason_end': {'trackdone': 1, 'fwdbtn': 2, 'logout': 5, 'clickrow': 7, 'backbtn': 3, 'endplay': 4, 'remote': 6}\n",
    "    }\n",
    "\n",
    "    for c in enc:\n",
    "        df_trn[c] = df_trn[c].map(enc[c]).fillna(0).astype(np.uint8)\n",
    "        \n",
    "    df_out = df_trn.merge(df_trk_, \n",
    "                          how='left', \n",
    "                          right_on='track_id', \n",
    "                          left_on='track_id_clean'\n",
    "                         ).drop(['track_id_clean', 'track_id'], \n",
    "                                axis=1\n",
    "                               )\n",
    "    cols_bool = [c for c in df_out.columns if df_out[c].dtype==bool]\n",
    "    #print(cols_bool)\n",
    "    df_out.loc[:,cols_bool] = df_out.loc[:,cols_bool].astype(np.uint8)\n",
    "    \n",
    "    df_out.to_csv(path_ + '/out_' + fin_, index=False, float_format='%.5f')#, compression='gzip')\n",
    "    \n",
    "    del df_out, df_trn\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:32:27.297432Z",
     "start_time": "2018-12-17T20:32:27.293676Z"
    }
   },
   "outputs": [],
   "source": [
    "file_list = [c \n",
    "             for c in listdir(PATH+'training_set/') \n",
    "             if c.startswith('tmp_')\n",
    "             and not os.path.isfile(PATH+'training_set/'+'out_'+c)\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:32:27.642804Z",
     "start_time": "2018-12-17T20:32:27.638837Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tmp_0.csv']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:41:39.774829Z",
     "start_time": "2018-12-17T20:41:38.172437Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:01<00:00,  1.60s/it]\n"
     ]
    }
   ],
   "source": [
    "for f in tqdm(file_list):\n",
    "    #['log_0_20180715_000000000000.csv.gz', 'log_0_20180716_000000000000.csv.gz']\n",
    "    preprocess_data(f, PATH+'training_set/', df_trk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:16:54.402904Z",
     "start_time": "2018-12-17T20:16:54.383843Z"
    }
   },
   "outputs": [],
   "source": [
    "x = pd.read_csv('data/training_set/log_0_20180715_000000000000.csv.gz', nrows=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:25:57.074944Z",
     "start_time": "2018-12-17T20:25:57.068509Z"
    }
   },
   "outputs": [],
   "source": [
    "x.to_csv('data/training_set/tmp_0.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T20:24:47.113291Z",
     "start_time": "2018-12-17T20:24:47.094129Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>session_position</th>\n",
       "      <th>session_length</th>\n",
       "      <th>track_id_clean</th>\n",
       "      <th>skip_1</th>\n",
       "      <th>skip_2</th>\n",
       "      <th>skip_3</th>\n",
       "      <th>not_skipped</th>\n",
       "      <th>context_switch</th>\n",
       "      <th>no_pause_before_play</th>\n",
       "      <th>short_pause_before_play</th>\n",
       "      <th>long_pause_before_play</th>\n",
       "      <th>hist_user_behavior_n_seekfwd</th>\n",
       "      <th>hist_user_behavior_n_seekback</th>\n",
       "      <th>hist_user_behavior_is_shuffle</th>\n",
       "      <th>hour_of_day</th>\n",
       "      <th>date</th>\n",
       "      <th>premium</th>\n",
       "      <th>context_type</th>\n",
       "      <th>hist_user_behavior_reason_start</th>\n",
       "      <th>hist_user_behavior_reason_end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>t_0479f24c-27d2-46d6-a00c-7ec928f2b539</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>t_9099cd7b-c238-47b7-9381-f23f2c1d1043</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>t_fc5df5ba-5396-49a7-8b29-35d0d28249e0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>t_23cff8d6-d874-4b20-83dc-94e450e8aa20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>t_64f3743c-f624-46bb-a579-0f3f9a07a123</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>True</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               session_id  session_position  session_length  \\\n",
       "0  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 1              20   \n",
       "1  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 2              20   \n",
       "2  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 3              20   \n",
       "3  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 4              20   \n",
       "4  0_00006f66-33e5-4de7-a324-2d18e439fc1e                 5              20   \n",
       "\n",
       "                           track_id_clean  skip_1  skip_2  skip_3  \\\n",
       "0  t_0479f24c-27d2-46d6-a00c-7ec928f2b539   False   False   False   \n",
       "1  t_9099cd7b-c238-47b7-9381-f23f2c1d1043   False   False   False   \n",
       "2  t_fc5df5ba-5396-49a7-8b29-35d0d28249e0   False   False   False   \n",
       "3  t_23cff8d6-d874-4b20-83dc-94e450e8aa20   False   False   False   \n",
       "4  t_64f3743c-f624-46bb-a579-0f3f9a07a123   False   False   False   \n",
       "\n",
       "   not_skipped  context_switch  no_pause_before_play  short_pause_before_play  \\\n",
       "0         True               0                     0                        0   \n",
       "1         True               0                     1                        0   \n",
       "2         True               0                     1                        0   \n",
       "3         True               0                     1                        0   \n",
       "4         True               0                     1                        0   \n",
       "\n",
       "   long_pause_before_play  hist_user_behavior_n_seekfwd  \\\n",
       "0                       0                             0   \n",
       "1                       0                             0   \n",
       "2                       0                             0   \n",
       "3                       0                             0   \n",
       "4                       0                             0   \n",
       "\n",
       "   hist_user_behavior_n_seekback  hist_user_behavior_is_shuffle  hour_of_day  \\\n",
       "0                              0                           True           16   \n",
       "1                              0                           True           16   \n",
       "2                              0                           True           16   \n",
       "3                              0                           True           16   \n",
       "4                              0                           True           16   \n",
       "\n",
       "         date  premium        context_type hist_user_behavior_reason_start  \\\n",
       "0  2018-07-15     True  editorial_playlist                       trackdone   \n",
       "1  2018-07-15     True  editorial_playlist                       trackdone   \n",
       "2  2018-07-15     True  editorial_playlist                       trackdone   \n",
       "3  2018-07-15     True  editorial_playlist                       trackdone   \n",
       "4  2018-07-15     True  editorial_playlist                       trackdone   \n",
       "\n",
       "  hist_user_behavior_reason_end  \n",
       "0                     trackdone  \n",
       "1                     trackdone  \n",
       "2                     trackdone  \n",
       "3                     trackdone  \n",
       "4                     trackdone  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}