Preprocess_Dask_AcousticVector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:38:52.587129Z",
     "start_time": "2019-01-03T23:38:50.956113Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['submissions', 'test_set', 'track_features', 'training_set']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import dask\n",
    "import dask.dataframe as dd\n",
    "\n",
    "from helpers import *\n",
    "import keggler as kg\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "import gc\n",
    "gc.enable()\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=Warning)\n",
    "\n",
    "import os, psutil\n",
    "\n",
    "# Set up a logger to dump messages to both log file and notebook\n",
    "import logging as logging\n",
    "def ini_log(filename):\n",
    "    logger = logging.getLogger(__name__)\n",
    "    ## avoid multiple printouts due to same handlers added several times\n",
    "    if not logger.handlers:\n",
    "        logger.setLevel(logging.DEBUG)\n",
    "\n",
    "        handlers = [logging.StreamHandler(None), logging.FileHandler(filename, 'a')]\n",
    "\n",
    "        fmt=logging.Formatter('%(asctime)-15s: %(levelname)s  %(message)s')\n",
    "        for h in handlers:\n",
    "            h.setFormatter(fmt)\n",
    "            logger.addHandler(h)\n",
    "    return logger\n",
    "        \n",
    "log = ini_log('out.log')\n",
    "\n",
    "#PATH='data_mini/'\n",
    "#prefix='_mini'\n",
    "\n",
    "PATH='data/'\n",
    "SUB_PATH='test_set/' #test_set #training_set\n",
    "prefix=''#'_0_201807[1-2]'\n",
    "\n",
    "print(os.listdir(PATH))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:38:52.604278Z",
     "start_time": "2019-01-03T23:38:52.588936Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<dask.config.set at 0x7ffae3c89a20>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from multiprocessing.pool import ThreadPool\n",
    "import dask\n",
    "dask.config.set(pool=ThreadPool(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:49:59.083792Z",
     "start_time": "2018-12-17T11:49:59.080943Z"
    }
   },
   "source": [
    "# Read in training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:38:52.609109Z",
     "start_time": "2019-01-03T23:38:52.606090Z"
    }
   },
   "outputs": [],
   "source": [
    "# Training\n",
    "# fin = '{}/training_set/log{}*.csv.gz'.format(PATH,prefix)\n",
    "\n",
    "# Test\n",
    "# fin=['{}/{}/log_prehistory_20180715_000000000000.csv.gz'.format(PATH,SUB_PATH),\n",
    "#      '{}/{}/log_prehistory_20180716_000000000000.csv.gz'.format(PATH,SUB_PATH)\n",
    "#     ]\n",
    "fin = '{}/{}/log_input_*.csv.gz'.format(PATH,SUB_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:38:52.617334Z",
     "start_time": "2019-01-03T23:38:52.613055Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data//test_set//log_input_*.csv.gz'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:39:04.518145Z",
     "start_time": "2019-01-03T23:39:04.465066Z"
    }
   },
   "outputs": [],
   "source": [
    "dd_trn = dd.read_csv(fin,\n",
    "#                      usecols=[i for i in range(21) if i != 16], # disable this to process data/test_set/log_input_\n",
    "#                      usecols=[i for i in range(5)],\n",
    "                     compression='gzip')[['session_id', 'track_id_clean']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:39:05.322144Z",
     "start_time": "2019-01-03T23:39:05.318410Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd_trn.npartitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:39:05.903949Z",
     "start_time": "2019-01-03T23:39:05.900947Z"
    }
   },
   "outputs": [],
   "source": [
    "# dd_trn.memory_usage(deep=True).compute() / 1024**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:39:06.182572Z",
     "start_time": "2019-01-03T23:39:06.177463Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "session_id        object\n",
       "track_id_clean    object\n",
       "dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd_trn.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:39:06.775308Z",
     "start_time": "2019-01-03T23:39:06.773157Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#dd_trn.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:03:14.866890Z",
     "start_time": "2018-12-17T11:03:14.861479Z"
    }
   },
   "source": [
    "# Read in track features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:39:50.080665Z",
     "start_time": "2019-01-03T23:39:07.881620Z"
    }
   },
   "outputs": [],
   "source": [
    "df_trk = pd.concat([pd.read_csv(PATH+'track_features/'+f, \n",
    "                     usecols=[22+i for i in range(8)]+[0], \n",
    "                     compression='gzip'\n",
    "                    ) for f in os.listdir(PATH+'track_features/') if f.startswith('tf_')\n",
    "         ], axis=0).set_index('track_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:39:50.325020Z",
     "start_time": "2019-01-03T23:39:50.082643Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index                335.795269\n",
       "acoustic_vector_0     28.277496\n",
       "acoustic_vector_1     28.277496\n",
       "acoustic_vector_2     28.277496\n",
       "acoustic_vector_3     28.277496\n",
       "acoustic_vector_4     28.277496\n",
       "acoustic_vector_5     28.277496\n",
       "acoustic_vector_6     28.277496\n",
       "acoustic_vector_7     28.277496\n",
       "dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trk.memory_usage(deep=True)/1024**2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-17T11:53:39.360659Z",
     "start_time": "2018-12-17T11:53:39.355981Z"
    }
   },
   "source": [
    "# Merge DS with Track features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:40:00.162952Z",
     "start_time": "2019-01-03T23:39:50.326981Z"
    }
   },
   "outputs": [],
   "source": [
    "dd_out = dd_trn.merge(df_trk, \n",
    "                      how='left', \n",
    "                      right_index=True, \n",
    "                      left_on='track_id_clean'\n",
    "                     ).drop(['track_id_clean', 'session_id'], \n",
    "                            axis=1\n",
    "                           )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cast boolean into `unit8` to reduce file size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:40:00.190940Z",
     "start_time": "2019-01-03T23:40:00.164744Z"
    }
   },
   "outputs": [],
   "source": [
    "for c in dd_out.columns:\n",
    "    if c.startswith('acoustic_vector_'):\n",
    "        dd_out[c] = dd_out[c].astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:40:00.198626Z",
     "start_time": "2019-01-03T23:40:00.192854Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "acoustic_vector_0    float32\n",
       "acoustic_vector_1    float32\n",
       "acoustic_vector_2    float32\n",
       "acoustic_vector_3    float32\n",
       "acoustic_vector_4    float32\n",
       "acoustic_vector_5    float32\n",
       "acoustic_vector_6    float32\n",
       "acoustic_vector_7    float32\n",
       "dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd_out.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store the DD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:40:00.204849Z",
     "start_time": "2019-01-03T23:40:00.201851Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd_out.npartitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T23:54:29.173267Z",
     "start_time": "2019-01-03T23:40:00.206122Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data//test_set//subDD_v4_00.h5',\n",
       " 'data//test_set//subDD_v4_01.h5',\n",
       " 'data//test_set//subDD_v4_02.h5',\n",
       " 'data//test_set//subDD_v4_03.h5',\n",
       " 'data//test_set//subDD_v4_04.h5',\n",
       " 'data//test_set//subDD_v4_05.h5',\n",
       " 'data//test_set//subDD_v4_06.h5',\n",
       " 'data//test_set//subDD_v4_07.h5',\n",
       " 'data//test_set//subDD_v4_08.h5',\n",
       " 'data//test_set//subDD_v4_09.h5',\n",
       " 'data//test_set//subDD_v4_10.h5',\n",
       " 'data//test_set//subDD_v4_11.h5',\n",
       " 'data//test_set//subDD_v4_12.h5',\n",
       " 'data//test_set//subDD_v4_13.h5',\n",
       " 'data//test_set//subDD_v4_14.h5',\n",
       " 'data//test_set//subDD_v4_15.h5',\n",
       " 'data//test_set//subDD_v4_16.h5',\n",
       " 'data//test_set//subDD_v4_17.h5',\n",
       " 'data//test_set//subDD_v4_18.h5',\n",
       " 'data//test_set//subDD_v4_19.h5',\n",
       " 'data//test_set//subDD_v4_20.h5',\n",
       " 'data//test_set//subDD_v4_21.h5',\n",
       " 'data//test_set//subDD_v4_22.h5',\n",
       " 'data//test_set//subDD_v4_23.h5',\n",
       " 'data//test_set//subDD_v4_24.h5',\n",
       " 'data//test_set//subDD_v4_25.h5',\n",
       " 'data//test_set//subDD_v4_26.h5',\n",
       " 'data//test_set//subDD_v4_27.h5',\n",
       " 'data//test_set//subDD_v4_28.h5',\n",
       " 'data//test_set//subDD_v4_29.h5',\n",
       " 'data//test_set//subDD_v4_30.h5',\n",
       " 'data//test_set//subDD_v4_31.h5',\n",
       " 'data//test_set//subDD_v4_32.h5',\n",
       " 'data//test_set//subDD_v4_33.h5',\n",
       " 'data//test_set//subDD_v4_34.h5',\n",
       " 'data//test_set//subDD_v4_35.h5',\n",
       " 'data//test_set//subDD_v4_36.h5',\n",
       " 'data//test_set//subDD_v4_37.h5',\n",
       " 'data//test_set//subDD_v4_38.h5',\n",
       " 'data//test_set//subDD_v4_39.h5',\n",
       " 'data//test_set//subDD_v4_40.h5',\n",
       " 'data//test_set//subDD_v4_41.h5',\n",
       " 'data//test_set//subDD_v4_42.h5',\n",
       " 'data//test_set//subDD_v4_43.h5',\n",
       " 'data//test_set//subDD_v4_44.h5',\n",
       " 'data//test_set//subDD_v4_45.h5',\n",
       " 'data//test_set//subDD_v4_46.h5',\n",
       " 'data//test_set//subDD_v4_47.h5',\n",
       " 'data//test_set//subDD_v4_48.h5',\n",
       " 'data//test_set//subDD_v4_49.h5',\n",
       " 'data//test_set//subDD_v4_50.h5',\n",
       " 'data//test_set//subDD_v4_51.h5',\n",
       " 'data//test_set//subDD_v4_52.h5',\n",
       " 'data//test_set//subDD_v4_53.h5',\n",
       " 'data//test_set//subDD_v4_54.h5',\n",
       " 'data//test_set//subDD_v4_55.h5',\n",
       " 'data//test_set//subDD_v4_56.h5',\n",
       " 'data//test_set//subDD_v4_57.h5',\n",
       " 'data//test_set//subDD_v4_58.h5',\n",
       " 'data//test_set//subDD_v4_59.h5',\n",
       " 'data//test_set//subDD_v4_60.h5',\n",
       " 'data//test_set//subDD_v4_61.h5',\n",
       " 'data//test_set//subDD_v4_62.h5',\n",
       " 'data//test_set//subDD_v4_63.h5',\n",
       " 'data//test_set//subDD_v4_64.h5',\n",
       " 'data//test_set//subDD_v4_65.h5']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dd_out.to_csv(PATH+'/'+SUB_PATH+'/outDD_v2_*.csv.gz', index=False, float_format='%.5f', compression='gzip')\n",
    "dd_out.to_hdf(PATH+'/'+SUB_PATH+'/subDD_v4_*.h5', key='df')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}