Model_Building_Iterations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T13:41:42.294329Z",
     "start_time": "2019-01-03T13:41:41.170910Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import keggler as kg\n",
    "from helpers import *\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "\n",
    "import gc\n",
    "gc.enable()\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=Warning)\n",
    "\n",
    "import os, psutil\n",
    "import glob\n",
    "\n",
    "# Set up a logger to dump messages to both log file and notebook\n",
    "import logging as logging\n",
    "def ini_log(filename):\n",
    "    logger = logging.getLogger(__name__)\n",
    "    ## avoid multiple printouts due to same handlers added several times\n",
    "    if not logger.handlers:\n",
    "        logger.setLevel(logging.DEBUG)\n",
    "\n",
    "        handlers = [#logging.StreamHandler(None), \n",
    "            logging.FileHandler(filename, 'a')\n",
    "        ]\n",
    "\n",
    "        fmt=logging.Formatter('%(asctime)-15s: %(levelname)s  %(message)s')\n",
    "        for h in handlers:\n",
    "            h.setFormatter(fmt)\n",
    "            logger.addHandler(h)\n",
    "    return logger\n",
    "        \n",
    "log = ini_log('out.log')\n",
    "\n",
    "#PATH='data_mini/'\n",
    "#prefix='_mini'\n",
    "\n",
    "prefix=''\n",
    "\n",
    "n_files = 6\n",
    "n_iter  = 5\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lists of files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T13:41:42.300832Z",
     "start_time": "2019-01-03T13:41:42.296026Z"
    }
   },
   "outputs": [],
   "source": [
    "data_path = 'data/' # point this to your data folder\n",
    "trn_path = data_path + 'training_set/'\n",
    "\n",
    "# trn_input_logs = sorted(glob.glob(trn_path + \"outDD_v1*.csv.gz\"))\n",
    "trn_input_logs = sorted(glob.glob(trn_path + \"outDD_v2_*.h5\"))\n",
    "trn_extra = sorted(glob.glob(trn_path + \"outDD_v4_*.h5\"))\n",
    "ys   = sorted(glob.glob(trn_path + \"y_*.h5\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T13:41:42.323853Z",
     "start_time": "2019-01-03T13:41:42.302270Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data/training_set/outDD_v2_00.h5',\n",
       " 'data/training_set/outDD_v2_01.h5',\n",
       " 'data/training_set/outDD_v2_02.h5',\n",
       " 'data/training_set/outDD_v2_03.h5',\n",
       " 'data/training_set/outDD_v2_04.h5',\n",
       " 'data/training_set/outDD_v2_05.h5',\n",
       " 'data/training_set/outDD_v2_06.h5',\n",
       " 'data/training_set/outDD_v2_07.h5',\n",
       " 'data/training_set/outDD_v2_08.h5',\n",
       " 'data/training_set/outDD_v2_09.h5',\n",
       " 'data/training_set/outDD_v2_10.h5',\n",
       " 'data/training_set/outDD_v2_11.h5',\n",
       " 'data/training_set/outDD_v2_12.h5',\n",
       " 'data/training_set/outDD_v2_13.h5',\n",
       " 'data/training_set/outDD_v2_14.h5',\n",
       " 'data/training_set/outDD_v2_15.h5',\n",
       " 'data/training_set/outDD_v2_16.h5',\n",
       " 'data/training_set/outDD_v2_17.h5',\n",
       " 'data/training_set/outDD_v2_18.h5',\n",
       " 'data/training_set/outDD_v2_19.h5',\n",
       " 'data/training_set/outDD_v2_20.h5',\n",
       " 'data/training_set/outDD_v2_21.h5',\n",
       " 'data/training_set/outDD_v2_22.h5',\n",
       " 'data/training_set/outDD_v2_23.h5',\n",
       " 'data/training_set/outDD_v2_24.h5',\n",
       " 'data/training_set/outDD_v2_25.h5',\n",
       " 'data/training_set/outDD_v2_26.h5',\n",
       " 'data/training_set/outDD_v2_27.h5',\n",
       " 'data/training_set/outDD_v2_28.h5',\n",
       " 'data/training_set/outDD_v2_29.h5',\n",
       " 'data/training_set/outDD_v2_30.h5',\n",
       " 'data/training_set/outDD_v2_31.h5',\n",
       " 'data/training_set/outDD_v2_32.h5',\n",
       " 'data/training_set/outDD_v2_33.h5',\n",
       " 'data/training_set/outDD_v2_34.h5',\n",
       " 'data/training_set/outDD_v2_35.h5',\n",
       " 'data/training_set/outDD_v2_36.h5',\n",
       " 'data/training_set/outDD_v2_37.h5',\n",
       " 'data/training_set/outDD_v2_38.h5',\n",
       " 'data/training_set/outDD_v2_39.h5',\n",
       " 'data/training_set/outDD_v2_40.h5',\n",
       " 'data/training_set/outDD_v2_41.h5',\n",
       " 'data/training_set/outDD_v2_42.h5',\n",
       " 'data/training_set/outDD_v2_43.h5',\n",
       " 'data/training_set/outDD_v2_44.h5',\n",
       " 'data/training_set/outDD_v2_45.h5',\n",
       " 'data/training_set/outDD_v2_46.h5',\n",
       " 'data/training_set/outDD_v2_47.h5',\n",
       " 'data/training_set/outDD_v2_48.h5',\n",
       " 'data/training_set/outDD_v2_49.h5',\n",
       " 'data/training_set/outDD_v2_50.h5',\n",
       " 'data/training_set/outDD_v2_51.h5',\n",
       " 'data/training_set/outDD_v2_52.h5',\n",
       " 'data/training_set/outDD_v2_53.h5',\n",
       " 'data/training_set/outDD_v2_54.h5',\n",
       " 'data/training_set/outDD_v2_55.h5',\n",
       " 'data/training_set/outDD_v2_56.h5',\n",
       " 'data/training_set/outDD_v2_57.h5',\n",
       " 'data/training_set/outDD_v2_58.h5',\n",
       " 'data/training_set/outDD_v2_59.h5',\n",
       " 'data/training_set/outDD_v2_60.h5',\n",
       " 'data/training_set/outDD_v2_61.h5',\n",
       " 'data/training_set/outDD_v2_62.h5',\n",
       " 'data/training_set/outDD_v2_63.h5',\n",
       " 'data/training_set/outDD_v2_64.h5',\n",
       " 'data/training_set/outDD_v2_65.h5']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_input_logs[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T13:41:42.347027Z",
     "start_time": "2019-01-03T13:41:42.325271Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data/training_set/outDD_v4_00.h5',\n",
       " 'data/training_set/outDD_v4_01.h5',\n",
       " 'data/training_set/outDD_v4_02.h5',\n",
       " 'data/training_set/outDD_v4_03.h5',\n",
       " 'data/training_set/outDD_v4_04.h5',\n",
       " 'data/training_set/outDD_v4_05.h5',\n",
       " 'data/training_set/outDD_v4_06.h5',\n",
       " 'data/training_set/outDD_v4_07.h5',\n",
       " 'data/training_set/outDD_v4_08.h5',\n",
       " 'data/training_set/outDD_v4_09.h5',\n",
       " 'data/training_set/outDD_v4_10.h5',\n",
       " 'data/training_set/outDD_v4_11.h5',\n",
       " 'data/training_set/outDD_v4_12.h5',\n",
       " 'data/training_set/outDD_v4_13.h5',\n",
       " 'data/training_set/outDD_v4_14.h5',\n",
       " 'data/training_set/outDD_v4_15.h5',\n",
       " 'data/training_set/outDD_v4_16.h5',\n",
       " 'data/training_set/outDD_v4_17.h5',\n",
       " 'data/training_set/outDD_v4_18.h5',\n",
       " 'data/training_set/outDD_v4_19.h5',\n",
       " 'data/training_set/outDD_v4_20.h5',\n",
       " 'data/training_set/outDD_v4_21.h5',\n",
       " 'data/training_set/outDD_v4_22.h5',\n",
       " 'data/training_set/outDD_v4_23.h5',\n",
       " 'data/training_set/outDD_v4_24.h5',\n",
       " 'data/training_set/outDD_v4_25.h5',\n",
       " 'data/training_set/outDD_v4_26.h5',\n",
       " 'data/training_set/outDD_v4_27.h5',\n",
       " 'data/training_set/outDD_v4_28.h5',\n",
       " 'data/training_set/outDD_v4_29.h5',\n",
       " 'data/training_set/outDD_v4_30.h5',\n",
       " 'data/training_set/outDD_v4_31.h5',\n",
       " 'data/training_set/outDD_v4_32.h5',\n",
       " 'data/training_set/outDD_v4_33.h5',\n",
       " 'data/training_set/outDD_v4_34.h5',\n",
       " 'data/training_set/outDD_v4_35.h5',\n",
       " 'data/training_set/outDD_v4_36.h5',\n",
       " 'data/training_set/outDD_v4_37.h5',\n",
       " 'data/training_set/outDD_v4_38.h5',\n",
       " 'data/training_set/outDD_v4_39.h5',\n",
       " 'data/training_set/outDD_v4_40.h5',\n",
       " 'data/training_set/outDD_v4_41.h5',\n",
       " 'data/training_set/outDD_v4_42.h5',\n",
       " 'data/training_set/outDD_v4_43.h5',\n",
       " 'data/training_set/outDD_v4_44.h5',\n",
       " 'data/training_set/outDD_v4_45.h5',\n",
       " 'data/training_set/outDD_v4_46.h5',\n",
       " 'data/training_set/outDD_v4_47.h5',\n",
       " 'data/training_set/outDD_v4_48.h5',\n",
       " 'data/training_set/outDD_v4_49.h5',\n",
       " 'data/training_set/outDD_v4_50.h5',\n",
       " 'data/training_set/outDD_v4_51.h5',\n",
       " 'data/training_set/outDD_v4_52.h5',\n",
       " 'data/training_set/outDD_v4_53.h5',\n",
       " 'data/training_set/outDD_v4_54.h5',\n",
       " 'data/training_set/outDD_v4_55.h5',\n",
       " 'data/training_set/outDD_v4_56.h5',\n",
       " 'data/training_set/outDD_v4_57.h5',\n",
       " 'data/training_set/outDD_v4_58.h5',\n",
       " 'data/training_set/outDD_v4_59.h5',\n",
       " 'data/training_set/outDD_v4_60.h5',\n",
       " 'data/training_set/outDD_v4_61.h5',\n",
       " 'data/training_set/outDD_v4_62.h5',\n",
       " 'data/training_set/outDD_v4_63.h5',\n",
       " 'data/training_set/outDD_v4_64.h5',\n",
       " 'data/training_set/outDD_v4_65.h5']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_extra"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T13:41:42.369804Z",
     "start_time": "2019-01-03T13:41:42.349412Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgb\n",
    "mdl = lgb.LGBMClassifier(max_depth=-1, min_child_samples=400, \n",
    "              random_state=314, silent=True, metric='None', \n",
    "              n_jobs=4, n_estimators=500, learning_rate=0.1,\n",
    "              **{'colsample_bytree': 0.75, 'min_child_weight': 1, \n",
    "               'num_leaves': 60, 'subsample': 0.75}\n",
    "             )\n",
    "\n",
    "def learning_rate_decay_power_0995(current_iter):\n",
    "    base_learning_rate = 0.15\n",
    "    lr = base_learning_rate  * np.power(.998, current_iter)\n",
    "    return lr if lr > 1e-2 else 1e-2\n",
    "\n",
    "def print_RAM():\n",
    "    print('Memory = {:.2f} GB'. format(psutil.Process(os.getpid()).memory_info().rss / 1024**3))\n",
    "\n",
    "def train_models_for_each_track(i_iter, i_step):\n",
    "    print('==========================================')\n",
    "    print('Train {}-th iteration'.format(i_iter))\n",
    "    start_file = i_iter*i_step\n",
    "    print(trn_input_logs[start_file:start_file+i_step])\n",
    "    df_trn = pd.concat([read_log(f) \n",
    "                        for f in trn_input_logs[start_file:start_file+i_step]\n",
    "                       ], axis=0, ignore_index=True)\n",
    "#     print(df_trn.shape)\n",
    "    df_trn.rename({'not_skipped': 'skip_4'}, axis=1, inplace=True)\n",
    "    gc.collect()\n",
    "    print(trn_extra[start_file:start_file+i_step])\n",
    "    df_xtr = pd.concat([read_log(f) \n",
    "                        for f in trn_extra[start_file:start_file+i_step]\n",
    "                       ], axis=0, ignore_index=True)\n",
    "#     print(df_xtr.shape)\n",
    "    df_trn = pd.concat([df_trn, df_xtr], axis=1)\n",
    "    print(df_trn.shape)\n",
    "    \n",
    "#     display(df_trn.head())\n",
    "\n",
    "    from sklearn.preprocessing import LabelEncoder\n",
    "    df_trn['session_id'] = LabelEncoder().fit_transform(df_trn['session_id'])\n",
    "    df_trn['session_id'] = df_trn['session_id'].astype(np.uint32)\n",
    "    \n",
    "    print_RAM()\n",
    "    \n",
    "    # parameters for the train/test split\n",
    "    split_params = dict(test_size=0.10, random_state=314, shuffle=True)\n",
    "\n",
    "    # competition metric format\n",
    "    print(ys[start_file:start_file+i_step])\n",
    "    y_competition_truth = pd.concat([pd.read_hdf(f, key='df') \n",
    "                                     for f in ys[start_file:start_file+i_step]\n",
    "                                    ], axis=0)\n",
    "    y_lists_trn, y_lists_stp = train_test_split(y_competition_truth, **split_params)\n",
    "\n",
    "\n",
    "    l_prob = []\n",
    "    X_prob=[]\n",
    "\n",
    "    for i_ in list(range(10)):\n",
    "        print('----------- {} -------------'.format(i_))\n",
    "        print('Full dataframe length = {}'.format(len(df_trn)))\n",
    "        X_trn, y_trn, X_trk = get_XY(df_trn, aggs, reset_index=False, \n",
    "                                      list_musik_qualities_=list_musik_qualities, \n",
    "                                      aggs_music_qualities_=aggs_music_qualities,\n",
    "                                      i_=i_,\n",
    "                                     aggs_trkvec_=aggs_trkvec, list_trkvec_=list_trkvec)\n",
    "        \n",
    "        id_trn, id_stp = train_test_split(X_trn.index, **split_params)\n",
    "\n",
    "        i_trk = 0\n",
    "\n",
    "        # merge track aggregates\n",
    "        X = pd.concat([X_trn, X_trk[i_trk]], axis=1)\n",
    "        # add predictions for the last modelled track\n",
    "        if len(X_prob) > 0:\n",
    "            X = pd.concat([X, pd.DataFrame({'pred_trk{}'.format(j): X_prob[j] \n",
    "                                            for j in range(len(X_prob))\n",
    "                                           })],\n",
    "                          axis=1)\n",
    "        # get training and early-stop data and targets\n",
    "        X_trn_, y_trn_ = X.loc[id_trn,:], y_trn[i_trk].loc[id_trn]\n",
    "        X_stp_, y_stp_ = X.loc[id_stp,:], y_trn[i_trk].loc[id_stp]\n",
    "\n",
    "        # limit yourself to long-enough sessions\n",
    "        if i_ >= 5:\n",
    "            orig_len = len(X_trn_)\n",
    "            is_long_session = X_trn_['session_length'] >= (2*i_+1)\n",
    "            X_trn_, y_trn_ = (X_trn_[is_long_session],\n",
    "                              y_trn_[is_long_session])\n",
    "            is_long_session = X_stp_['session_length'] >= (2*i_+1)\n",
    "            X_stp_, y_stp_ = (X_stp_[is_long_session],\n",
    "                              y_stp_[is_long_session])\n",
    "            y_lists_stp_ = y_lists_stp[is_long_session.values]\n",
    "\n",
    "            final_len = len(X_trn_)\n",
    "            print('Kept {:.2f}% of data'.format(100. * final_len / orig_len))\n",
    "        else:\n",
    "            y_lists_stp_ = y_lists_stp\n",
    "\n",
    "    #     display(y_trn_.head(30))\n",
    "\n",
    "        fit_params = {'eval_names': ['train', 'early_stop'],\n",
    "                      'eval_set': [(X_trn_, y_trn_), \n",
    "                                   (X_stp_, y_stp_)],\n",
    "                      'eval_metric': 'binary_error',\n",
    "                      'verbose':100, 'early_stopping_rounds':60,\n",
    "                      'callbacks':[lgb.reset_parameter(learning_rate=learning_rate_decay_power_0995)]}    \n",
    "\n",
    "        mdl.fit(X_trn_, y_trn_, \n",
    "                **fit_params)\n",
    "\n",
    "        del X_trn_, y_trn_, X_stp_, y_stp_\n",
    "        gc.collect()\n",
    "\n",
    "        # save full (trn+val) prediction to be used in modelling\n",
    "        X_prob.append(mdl.predict_proba(X)[:,1])\n",
    "\n",
    "\n",
    "        # store the model\n",
    "        import joblib\n",
    "        joblib.dump(mdl, 'models/model_v2_m05i{}_{}f_{}.pkl'.format(i_iter, n_files, i_))\n",
    "        del X\n",
    "        gc.collect()\n",
    "        print_RAM()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-03T14:51:33.780971Z",
     "start_time": "2019-01-03T13:41:42.371859Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================================\n",
      "Train 0-th iteration\n",
      "['data/training_set/outDD_v2_00.h5', 'data/training_set/outDD_v2_01.h5', 'data/training_set/outDD_v2_02.h5', 'data/training_set/outDD_v2_03.h5', 'data/training_set/outDD_v2_04.h5', 'data/training_set/outDD_v2_05.h5']\n",
      "['data/training_set/outDD_v4_00.h5', 'data/training_set/outDD_v4_01.h5', 'data/training_set/outDD_v4_02.h5', 'data/training_set/outDD_v4_03.h5', 'data/training_set/outDD_v4_04.h5', 'data/training_set/outDD_v4_05.h5']\n",
      "(20107425, 46)\n",
      "Memory = 2.86 GB\n",
      "['data/training_set/y_00_.h5', 'data/training_set/y_01_.h5', 'data/training_set/y_02_.h5', 'data/training_set/y_03_.h5', 'data/training_set/y_04_.h5', 'data/training_set/y_05_.h5']\n",
      "----------- 0 -------------\n",
      "Full dataframe length = 20107425\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.217083\tearly_stop's binary_error: 0.220502\n",
      "Early stopping, best iteration is:\n",
      "[69]\ttrain's binary_error: 0.218226\tearly_stop's binary_error: 0.220203\n",
      "Memory = 5.12 GB\n",
      "----------- 1 -------------\n",
      "Full dataframe length = 20107425\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.290338\tearly_stop's binary_error: 0.295926\n",
      "[200]\ttrain's binary_error: 0.285367\tearly_stop's binary_error: 0.295412\n",
      "[300]\ttrain's binary_error: 0.281404\tearly_stop's binary_error: 0.295645\n",
      "Early stopping, best iteration is:\n",
      "[251]\ttrain's binary_error: 0.283304\tearly_stop's binary_error: 0.29523\n",
      "Memory = 5.61 GB\n",
      "----------- 2 -------------\n",
      "Full dataframe length = 20107425\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.316824\tearly_stop's binary_error: 0.330939\n",
      "Early stopping, best iteration is:\n",
      "[110]\ttrain's binary_error: 0.31603\tearly_stop's binary_error: 0.330815\n",
      "Memory = 5.75 GB\n",
      "----------- 3 -------------\n",
      "Full dataframe length = 20107425\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.330789\tearly_stop's binary_error: 0.345983\n",
      "Early stopping, best iteration is:\n",
      "[71]\ttrain's binary_error: 0.333065\tearly_stop's binary_error: 0.345643\n",
      "Memory = 5.87 GB\n",
      "----------- 4 -------------\n",
      "Full dataframe length = 20107425\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.330168\tearly_stop's binary_error: 0.34143\n",
      "[200]\ttrain's binary_error: 0.32451\tearly_stop's binary_error: 0.341331\n",
      "Early stopping, best iteration is:\n",
      "[142]\ttrain's binary_error: 0.327595\tearly_stop's binary_error: 0.340816\n",
      "Memory = 5.88 GB\n",
      "----------- 5 -------------\n",
      "Full dataframe length = 20107425\n",
      "Kept 91.53% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.322632\tearly_stop's binary_error: 0.334975\n",
      "Early stopping, best iteration is:\n",
      "[76]\ttrain's binary_error: 0.32426\tearly_stop's binary_error: 0.33473\n",
      "Memory = 6.84 GB\n",
      "----------- 6 -------------\n",
      "Full dataframe length = 20107425\n",
      "Kept 77.55% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.323706\tearly_stop's binary_error: 0.338989\n",
      "Early stopping, best iteration is:\n",
      "[73]\ttrain's binary_error: 0.32575\tearly_stop's binary_error: 0.338839\n",
      "Memory = 7.62 GB\n",
      "----------- 7 -------------\n",
      "Full dataframe length = 20107425\n",
      "Kept 66.69% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[4]\ttrain's binary_error: 0.335914\tearly_stop's binary_error: 0.34341\n",
      "Memory = 8.51 GB\n",
      "----------- 8 -------------\n",
      "Full dataframe length = 20107425\n",
      "Kept 58.32% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[37]\ttrain's binary_error: 0.338094\tearly_stop's binary_error: 0.346682\n",
      "Memory = 8.44 GB\n",
      "----------- 9 -------------\n",
      "Full dataframe length = 20107425\n",
      "Kept 51.69% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.337873\tearly_stop's binary_error: 0.361313\n",
      "Early stopping, best iteration is:\n",
      "[61]\ttrain's binary_error: 0.343494\tearly_stop's binary_error: 0.360575\n",
      "Memory = 8.31 GB\n",
      "==========================================\n",
      "Train 1-th iteration\n",
      "['data/training_set/outDD_v2_06.h5', 'data/training_set/outDD_v2_07.h5', 'data/training_set/outDD_v2_08.h5', 'data/training_set/outDD_v2_09.h5', 'data/training_set/outDD_v2_10.h5', 'data/training_set/outDD_v2_11.h5']\n",
      "['data/training_set/outDD_v4_06.h5', 'data/training_set/outDD_v4_07.h5', 'data/training_set/outDD_v4_08.h5', 'data/training_set/outDD_v4_09.h5', 'data/training_set/outDD_v4_10.h5', 'data/training_set/outDD_v4_11.h5']\n",
      "(19198509, 46)\n",
      "Memory = 6.09 GB\n",
      "['data/training_set/y_06_.h5', 'data/training_set/y_07_.h5', 'data/training_set/y_08_.h5', 'data/training_set/y_09_.h5', 'data/training_set/y_10_.h5', 'data/training_set/y_11_.h5']\n",
      "----------- 0 -------------\n",
      "Full dataframe length = 19198509\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.214783\tearly_stop's binary_error: 0.22\n",
      "Early stopping, best iteration is:\n",
      "[86]\ttrain's binary_error: 0.215316\tearly_stop's binary_error: 0.219722\n",
      "Memory = 7.23 GB\n",
      "----------- 1 -------------\n",
      "Full dataframe length = 19198509\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.289093\tearly_stop's binary_error: 0.297822\n",
      "[200]\ttrain's binary_error: 0.284085\tearly_stop's binary_error: 0.296927\n",
      "Early stopping, best iteration is:\n",
      "[223]\ttrain's binary_error: 0.282919\tearly_stop's binary_error: 0.296771\n",
      "Memory = 7.74 GB\n",
      "----------- 2 -------------\n",
      "Full dataframe length = 19198509\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.317405\tearly_stop's binary_error: 0.329158\n",
      "Early stopping, best iteration is:\n",
      "[76]\ttrain's binary_error: 0.31893\tearly_stop's binary_error: 0.328906\n",
      "Memory = 7.72 GB\n",
      "----------- 3 -------------\n",
      "Full dataframe length = 19198509\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.33086\tearly_stop's binary_error: 0.343257\n",
      "Early stopping, best iteration is:\n",
      "[78]\ttrain's binary_error: 0.332854\tearly_stop's binary_error: 0.34304\n",
      "Memory = 7.75 GB\n",
      "----------- 4 -------------\n",
      "Full dataframe length = 19198509\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.328573\tearly_stop's binary_error: 0.340851\n",
      "Early stopping, best iteration is:\n",
      "[40]\ttrain's binary_error: 0.33323\tearly_stop's binary_error: 0.340078\n",
      "Memory = 7.72 GB\n",
      "----------- 5 -------------\n",
      "Full dataframe length = 19198509\n",
      "Kept 91.51% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.323492\tearly_stop's binary_error: 0.336091\n",
      "Early stopping, best iteration is:\n",
      "[63]\ttrain's binary_error: 0.326207\tearly_stop's binary_error: 0.335626\n",
      "Memory = 8.24 GB\n",
      "----------- 6 -------------\n",
      "Full dataframe length = 19198509\n",
      "Kept 77.47% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.323699\tearly_stop's binary_error: 0.340378\n",
      "Early stopping, best iteration is:\n",
      "[80]\ttrain's binary_error: 0.325844\tearly_stop's binary_error: 0.339918\n",
      "Memory = 8.18 GB\n",
      "----------- 7 -------------\n",
      "Full dataframe length = 19198509\n",
      "Kept 66.72% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.323413\tearly_stop's binary_error: 0.343085\n",
      "Early stopping, best iteration is:\n",
      "[60]\ttrain's binary_error: 0.327582\tearly_stop's binary_error: 0.341743\n",
      "Memory = 7.97 GB\n",
      "----------- 8 -------------\n",
      "Full dataframe length = 19198509\n",
      "Kept 58.39% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.325081\tearly_stop's binary_error: 0.347348\n",
      "Early stopping, best iteration is:\n",
      "[64]\ttrain's binary_error: 0.329298\tearly_stop's binary_error: 0.346453\n",
      "Memory = 7.93 GB\n",
      "----------- 9 -------------\n",
      "Full dataframe length = 19198509\n",
      "Kept 51.79% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[6]\ttrain's binary_error: 0.349873\tearly_stop's binary_error: 0.360721\n",
      "Memory = 7.67 GB\n",
      "==========================================\n",
      "Train 2-th iteration\n",
      "['data/training_set/outDD_v2_12.h5', 'data/training_set/outDD_v2_13.h5', 'data/training_set/outDD_v2_14.h5', 'data/training_set/outDD_v2_15.h5', 'data/training_set/outDD_v2_16.h5', 'data/training_set/outDD_v2_17.h5']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['data/training_set/outDD_v4_12.h5', 'data/training_set/outDD_v4_13.h5', 'data/training_set/outDD_v4_14.h5', 'data/training_set/outDD_v4_15.h5', 'data/training_set/outDD_v4_16.h5', 'data/training_set/outDD_v4_17.h5']\n",
      "(19047719, 46)\n",
      "Memory = 6.50 GB\n",
      "['data/training_set/y_12_.h5', 'data/training_set/y_13_.h5', 'data/training_set/y_14_.h5', 'data/training_set/y_15_.h5', 'data/training_set/y_16_.h5', 'data/training_set/y_17_.h5']\n",
      "----------- 0 -------------\n",
      "Full dataframe length = 19047719\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.216258\tearly_stop's binary_error: 0.219531\n",
      "Early stopping, best iteration is:\n",
      "[108]\ttrain's binary_error: 0.215988\tearly_stop's binary_error: 0.219426\n",
      "Memory = 7.13 GB\n",
      "----------- 1 -------------\n",
      "Full dataframe length = 19047719\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.289021\tearly_stop's binary_error: 0.296752\n",
      "[200]\ttrain's binary_error: 0.28373\tearly_stop's binary_error: 0.29578\n",
      "Early stopping, best iteration is:\n",
      "[183]\ttrain's binary_error: 0.284464\tearly_stop's binary_error: 0.295597\n",
      "Memory = 7.66 GB\n",
      "----------- 2 -------------\n",
      "Full dataframe length = 19047719\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.316258\tearly_stop's binary_error: 0.33015\n",
      "Early stopping, best iteration is:\n",
      "[56]\ttrain's binary_error: 0.319581\tearly_stop's binary_error: 0.329826\n",
      "Memory = 7.15 GB\n",
      "----------- 3 -------------\n",
      "Full dataframe length = 19047719\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.331192\tearly_stop's binary_error: 0.341845\n",
      "Early stopping, best iteration is:\n",
      "[50]\ttrain's binary_error: 0.33478\tearly_stop's binary_error: 0.341294\n",
      "Memory = 7.55 GB\n",
      "----------- 4 -------------\n",
      "Full dataframe length = 19047719\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.327935\tearly_stop's binary_error: 0.339438\n",
      "Early stopping, best iteration is:\n",
      "[84]\ttrain's binary_error: 0.329132\tearly_stop's binary_error: 0.338913\n",
      "Memory = 7.17 GB\n",
      "----------- 5 -------------\n",
      "Full dataframe length = 19047719\n",
      "Kept 91.47% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.321958\tearly_stop's binary_error: 0.33626\n",
      "Early stopping, best iteration is:\n",
      "[77]\ttrain's binary_error: 0.323718\tearly_stop's binary_error: 0.335533\n",
      "Memory = 7.62 GB\n",
      "----------- 6 -------------\n",
      "Full dataframe length = 19047719\n",
      "Kept 77.45% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.322923\tearly_stop's binary_error: 0.338102\n",
      "Early stopping, best iteration is:\n",
      "[76]\ttrain's binary_error: 0.324946\tearly_stop's binary_error: 0.337808\n",
      "Memory = 7.11 GB\n",
      "----------- 7 -------------\n",
      "Full dataframe length = 19047719\n",
      "Kept 66.69% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.323095\tearly_stop's binary_error: 0.340745\n",
      "Early stopping, best iteration is:\n",
      "[50]\ttrain's binary_error: 0.328372\tearly_stop's binary_error: 0.34018\n",
      "Memory = 7.89 GB\n",
      "----------- 8 -------------\n",
      "Full dataframe length = 19047719\n",
      "Kept 58.35% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.327086\tearly_stop's binary_error: 0.349899\n",
      "[200]\ttrain's binary_error: 0.316973\tearly_stop's binary_error: 0.349252\n",
      "Early stopping, best iteration is:\n",
      "[193]\ttrain's binary_error: 0.31752\tearly_stop's binary_error: 0.348726\n",
      "Memory = 7.80 GB\n",
      "----------- 9 -------------\n",
      "Full dataframe length = 19047719\n",
      "Kept 51.74% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[6]\ttrain's binary_error: 0.341894\tearly_stop's binary_error: 0.363693\n",
      "Memory = 8.03 GB\n",
      "==========================================\n",
      "Train 3-th iteration\n",
      "['data/training_set/outDD_v2_18.h5', 'data/training_set/outDD_v2_19.h5', 'data/training_set/outDD_v2_20.h5', 'data/training_set/outDD_v2_21.h5', 'data/training_set/outDD_v2_22.h5', 'data/training_set/outDD_v2_23.h5']\n",
      "['data/training_set/outDD_v4_18.h5', 'data/training_set/outDD_v4_19.h5', 'data/training_set/outDD_v4_20.h5', 'data/training_set/outDD_v4_21.h5', 'data/training_set/outDD_v4_22.h5', 'data/training_set/outDD_v4_23.h5']\n",
      "(18856902, 46)\n",
      "Memory = 6.67 GB\n",
      "['data/training_set/y_18_.h5', 'data/training_set/y_19_.h5', 'data/training_set/y_20_.h5', 'data/training_set/y_21_.h5', 'data/training_set/y_22_.h5', 'data/training_set/y_23_.h5']\n",
      "----------- 0 -------------\n",
      "Full dataframe length = 18856902\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.217323\tearly_stop's binary_error: 0.219161\n",
      "[200]\ttrain's binary_error: 0.213882\tearly_stop's binary_error: 0.219029\n",
      "Early stopping, best iteration is:\n",
      "[143]\ttrain's binary_error: 0.21569\tearly_stop's binary_error: 0.218517\n",
      "Memory = 7.84 GB\n",
      "----------- 1 -------------\n",
      "Full dataframe length = 18856902\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.290145\tearly_stop's binary_error: 0.296721\n",
      "[200]\ttrain's binary_error: 0.284669\tearly_stop's binary_error: 0.296448\n",
      "Early stopping, best iteration is:\n",
      "[158]\ttrain's binary_error: 0.286737\tearly_stop's binary_error: 0.296227\n",
      "Memory = 8.29 GB\n",
      "----------- 2 -------------\n",
      "Full dataframe length = 18856902\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.318816\tearly_stop's binary_error: 0.33032\n",
      "Early stopping, best iteration is:\n",
      "[139]\ttrain's binary_error: 0.316201\tearly_stop's binary_error: 0.330003\n",
      "Memory = 8.29 GB\n",
      "----------- 3 -------------\n",
      "Full dataframe length = 18856902\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.330781\tearly_stop's binary_error: 0.345233\n",
      "[200]\ttrain's binary_error: 0.323945\tearly_stop's binary_error: 0.344263\n",
      "Early stopping, best iteration is:\n",
      "[199]\ttrain's binary_error: 0.32403\tearly_stop's binary_error: 0.344263\n",
      "Memory = 7.84 GB\n",
      "----------- 4 -------------\n",
      "Full dataframe length = 18856902\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.324946\tearly_stop's binary_error: 0.342428\n",
      "[200]\ttrain's binary_error: 0.31888\tearly_stop's binary_error: 0.342499\n",
      "Early stopping, best iteration is:\n",
      "[157]\ttrain's binary_error: 0.321001\tearly_stop's binary_error: 0.342076\n",
      "Memory = 8.29 GB\n",
      "----------- 5 -------------\n",
      "Full dataframe length = 18856902\n",
      "Kept 91.30% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.319523\tearly_stop's binary_error: 0.338853\n",
      "Early stopping, best iteration is:\n",
      "[71]\ttrain's binary_error: 0.321407\tearly_stop's binary_error: 0.338282\n",
      "Memory = 8.53 GB\n",
      "----------- 6 -------------\n",
      "Full dataframe length = 18856902\n",
      "Kept 77.09% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.321171\tearly_stop's binary_error: 0.344255\n",
      "Early stopping, best iteration is:\n",
      "[56]\ttrain's binary_error: 0.324839\tearly_stop's binary_error: 0.3439\n",
      "Memory = 8.52 GB\n",
      "----------- 7 -------------\n",
      "Full dataframe length = 18856902\n",
      "Kept 66.18% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.321499\tearly_stop's binary_error: 0.343242\n",
      "Early stopping, best iteration is:\n",
      "[49]\ttrain's binary_error: 0.327285\tearly_stop's binary_error: 0.342455\n",
      "Memory = 8.48 GB\n",
      "----------- 8 -------------\n",
      "Full dataframe length = 18856902\n",
      "Kept 57.77% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.325797\tearly_stop's binary_error: 0.346153\n",
      "Early stopping, best iteration is:\n",
      "[46]\ttrain's binary_error: 0.332389\tearly_stop's binary_error: 0.345174\n",
      "Memory = 8.08 GB\n",
      "----------- 9 -------------\n",
      "Full dataframe length = 18856902\n",
      "Kept 51.20% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.335369\tearly_stop's binary_error: 0.359564\n",
      "Early stopping, best iteration is:\n",
      "[73]\ttrain's binary_error: 0.339634\tearly_stop's binary_error: 0.358805\n",
      "Memory = 8.45 GB\n",
      "==========================================\n",
      "Train 4-th iteration\n",
      "['data/training_set/outDD_v2_24.h5', 'data/training_set/outDD_v2_25.h5', 'data/training_set/outDD_v2_26.h5', 'data/training_set/outDD_v2_27.h5', 'data/training_set/outDD_v2_28.h5', 'data/training_set/outDD_v2_29.h5']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['data/training_set/outDD_v4_24.h5', 'data/training_set/outDD_v4_25.h5', 'data/training_set/outDD_v4_26.h5', 'data/training_set/outDD_v4_27.h5', 'data/training_set/outDD_v4_28.h5', 'data/training_set/outDD_v4_29.h5']\n",
      "(18734510, 46)\n",
      "Memory = 7.30 GB\n",
      "['data/training_set/y_24_.h5', 'data/training_set/y_25_.h5', 'data/training_set/y_26_.h5', 'data/training_set/y_27_.h5', 'data/training_set/y_28_.h5', 'data/training_set/y_29_.h5']\n",
      "----------- 0 -------------\n",
      "Full dataframe length = 18734510\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.215924\tearly_stop's binary_error: 0.218282\n",
      "Early stopping, best iteration is:\n",
      "[95]\ttrain's binary_error: 0.216067\tearly_stop's binary_error: 0.217998\n",
      "Memory = 7.85 GB\n",
      "----------- 1 -------------\n",
      "Full dataframe length = 18734510\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.289138\tearly_stop's binary_error: 0.294118\n",
      "Early stopping, best iteration is:\n",
      "[51]\ttrain's binary_error: 0.292416\tearly_stop's binary_error: 0.293913\n",
      "Memory = 8.31 GB\n",
      "----------- 2 -------------\n",
      "Full dataframe length = 18734510\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.31968\tearly_stop's binary_error: 0.327645\n",
      "Early stopping, best iteration is:\n",
      "[96]\ttrain's binary_error: 0.319968\tearly_stop's binary_error: 0.327343\n",
      "Memory = 8.31 GB\n",
      "----------- 3 -------------\n",
      "Full dataframe length = 18734510\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.331917\tearly_stop's binary_error: 0.345003\n",
      "Early stopping, best iteration is:\n",
      "[96]\ttrain's binary_error: 0.332194\tearly_stop's binary_error: 0.344533\n",
      "Memory = 8.31 GB\n",
      "----------- 4 -------------\n",
      "Full dataframe length = 18734510\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.328439\tearly_stop's binary_error: 0.342349\n",
      "Early stopping, best iteration is:\n",
      "[70]\ttrain's binary_error: 0.330563\tearly_stop's binary_error: 0.342206\n",
      "Memory = 8.31 GB\n",
      "----------- 5 -------------\n",
      "Full dataframe length = 18734510\n",
      "Kept 91.35% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.321849\tearly_stop's binary_error: 0.334701\n",
      "Early stopping, best iteration is:\n",
      "[89]\ttrain's binary_error: 0.322621\tearly_stop's binary_error: 0.334099\n",
      "Memory = 9.12 GB\n",
      "----------- 6 -------------\n",
      "Full dataframe length = 18734510\n",
      "Kept 77.15% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.323044\tearly_stop's binary_error: 0.335882\n",
      "Early stopping, best iteration is:\n",
      "[103]\ttrain's binary_error: 0.322879\tearly_stop's binary_error: 0.335733\n",
      "Memory = 8.57 GB\n",
      "----------- 7 -------------\n",
      "Full dataframe length = 18734510\n",
      "Kept 66.23% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.321505\tearly_stop's binary_error: 0.342076\n",
      "Early stopping, best iteration is:\n",
      "[57]\ttrain's binary_error: 0.325548\tearly_stop's binary_error: 0.3417\n",
      "Memory = 8.72 GB\n",
      "----------- 8 -------------\n",
      "Full dataframe length = 18734510\n",
      "Kept 57.80% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[4]\ttrain's binary_error: 0.338659\tearly_stop's binary_error: 0.344954\n",
      "Memory = 8.61 GB\n",
      "----------- 9 -------------\n",
      "Full dataframe length = 18734510\n",
      "Kept 51.22% of data\n",
      "Training until validation scores don't improve for 60 rounds.\n",
      "[100]\ttrain's binary_error: 0.336241\tearly_stop's binary_error: 0.359379\n",
      "Early stopping, best iteration is:\n",
      "[47]\ttrain's binary_error: 0.344742\tearly_stop's binary_error: 0.35872\n",
      "Memory = 8.61 GB\n"
     ]
    }
   ],
   "source": [
    "for i in range(n_iter):\n",
    "    train_models_for_each_track(i_iter=i, i_step=n_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "185px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}