GradientBoostClassifiers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from tqdm import tqdm_notebook\n",
    "import sklearn\n",
    "from sklearn.ensemble import GradientBoostingClassifier \n",
    "prefix='../'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_event(event):\n",
    "    hits= pd.read_csv(prefix+'train_1/%s-hits.csv'%event)\n",
    "    cells= pd.read_csv(prefix+'train_1/%s-cells.csv'%event)\n",
    "    truth= pd.read_csv(prefix+'train_1/%s-truth.csv'%event)\n",
    "    particles= pd.read_csv(prefix+'train_1/%s-particles.csv'%event)\n",
    "    return hits, cells, truth, particles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_cylindrical_coordinates(hits):\n",
    "    x, y = hits['x'], hits['y']\n",
    "    hits['r'] = np.sqrt(x**2 + y**2)\n",
    "    hits['a0'] = np.arctan2(y, x)\n",
    "    return hits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcb863ddfe6a4362973c765da1901f10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "event000001010 (905342, 17)\n",
      "event000001011 (1049940, 17)\n",
      "event000001012 (975962, 17)\n",
      "event000001013 (937140, 17)\n",
      "event000001014 (1138386, 17)\n",
      "event000001015 (1096946, 17)\n",
      "event000001016 (1054472, 17)\n",
      "event000001017 (1125976, 17)\n",
      "event000001018 (787588, 17)\n",
      "event000001019 (1099946, 17)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# you can jump to step4 for test only.\n",
    "train = True\n",
    "if train:\n",
    "    Train = []\n",
    "    for i in tqdm_notebook(range(10,20)):\n",
    "        event = 'event0000010%02d'%i\n",
    "        hits, cells, truth, particles = get_event(event)\n",
    "        hit_cells = cells.groupby(['hit_id']).value.count().values\n",
    "        hit_value = cells.groupby(['hit_id']).value.sum().values\n",
    "        #features = np.array(hits[['x','y','z']]/1000).reshape(len(hit_cells), 3)#, hit_cells.reshape(len(hit_cells),1)/10,hit_value.reshape(len(hit_cells),1)))\n",
    "        hits = convert_to_cylindrical_coordinates(hits)\n",
    "        features = np.hstack((np.hstack((np.array(hits['z']/1000).reshape(np.array(hits['x']).shape[0], 1), \n",
    "                                         np.array(hits['r']/1000).reshape(np.array(hits['r']).shape[0], 1))), \n",
    "                              np.array(hits['a0']).reshape(len(hits['a0'].iloc[:]), 1)))\n",
    "        hit_signals = np.hstack((hit_cells.reshape(len(hit_cells),1)/10,hit_value.reshape(len(hit_cells),1)))\n",
    "        features = np.hstack((features, hit_signals))\n",
    "        features = np.hstack((features, np.array(hits[['x','y','z']]/1000).reshape(len(hit_cells), 3)))\n",
    "        particle_ids = truth.particle_id.unique()\n",
    "        particle_ids = particle_ids[np.where(particle_ids!=0)[0]]\n",
    "\n",
    "        pair = []\n",
    "        for particle_id in particle_ids:\n",
    "            hit_ids = truth[truth.particle_id == particle_id].hit_id.values-1\n",
    "            for i in hit_ids:\n",
    "                for j in hit_ids:\n",
    "                    if i != j:\n",
    "                        pair.append([i,j])\n",
    "        pair = np.array(pair)   \n",
    "        Train1 = np.hstack((features[pair[:,0]], features[pair[:,1]], np.ones((len(pair),1))))\n",
    "\n",
    "        if len(Train) == 0:\n",
    "            Train = Train1\n",
    "        else:\n",
    "            Train = np.vstack((Train,Train1))\n",
    "\n",
    "        n = len(hits)\n",
    "        size = len(Train1)*3\n",
    "        p_id = truth.particle_id.values\n",
    "        i =np.random.randint(n, size=size)\n",
    "        j =np.random.randint(n, size=size)\n",
    "        pair = np.hstack((i.reshape(size,1),j.reshape(size,1)))\n",
    "        pair = pair[((p_id[i]==0) | (p_id[i]!=p_id[j]))]\n",
    "\n",
    "        Train0 = np.hstack((features[pair[:,0]], features[pair[:,1]], np.zeros((len(pair),1))))\n",
    "\n",
    "        print(event, Train1.shape)\n",
    "\n",
    "        Train = np.vstack((Train,Train0))\n",
    "    del Train0, Train1\n",
    "\n",
    "    np.random.shuffle(Train)\n",
    "    print(Train.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_Train = np.load('train10events.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = GradientBoostingClassifier(max_depth=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 30min 8s, sys: 14.1 s, total: 30min 22s\n",
      "Wall time: 42min 42s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=10,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "              n_iter_no_change=None, presort='auto', random_state=None,\n",
       "              subsample=1.0, tol=0.0001, validation_fraction=0.1,\n",
       "              verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "clf.fit(Train[:20000,:-1], Train[:20000, -1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.09 s, sys: 47.8 ms, total: 1.14 s\n",
      "Wall time: 1.15 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.942485"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "clf.score(Train[-200000:, :-1], Train[-200000:, -1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using 1000 trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.getsizeof(Train)\n",
    "with open('train10events.npy', 'wb') as file:\n",
    "    np.save(file, Train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf2 = GradientBoostingClassifier(max_depth = None, n_estimators = 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13min 23s, sys: 3.11 s, total: 13min 26s\n",
      "Wall time: 13min 30s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=None,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
       "              presort='auto', random_state=None, subsample=1.0, verbose=0,\n",
       "              warm_start=False)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "clf2.fit(Train[:100000,:-1], Train[:100000, -1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 559 ms, sys: 41.5 ms, total: 600 ms\n",
      "Wall time: 619 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.97953"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "#clf.score(Train[-150000:, :-1], Train[-150000:, -1])\n",
    "clf.score(s_Train[:100000,:-1], s_Train[:100000, -1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('100_Gradient_Boost_96-5.p', 'wb') as file:\n",
    "    pickle.dump(clf, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/base.py:251: UserWarning: Trying to unpickle estimator DecisionTreeRegressor from version 0.19.1 when using version 0.20.2. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/base.py:251: UserWarning: Trying to unpickle estimator GradientBoostingClassifier from version 0.19.1 when using version 0.20.2. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "with open('100_Gradient_Boost_96-5.p', 'rb') as file:\n",
    "    clf = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = True\n",
    "if train:\n",
    "    Train = []\n",
    "    for i in tqdm_notebook(range(10,20)):\n",
    "        event = 'event0000010%02d'%i\n",
    "        hits, cells, truth, particles = get_event(event)\n",
    "        hit_cells = cells.groupby(['hit_id']).value.count().values\n",
    "        hit_value = cells.groupby(['hit_id']).value.sum().values\n",
    "        features = np.hstack((hits[['x','y','z']]/1000, hit_cells.reshape(len(hit_cells),1)/10,hit_value.reshape(len(hit_cells),1)))\n",
    "        particle_ids = truth.particle_id.unique()\n",
    "        particle_ids = particle_ids[np.where(particle_ids!=0)[0]]\n",
    "\n",
    "        pair = []\n",
    "        for particle_id in particle_ids:\n",
    "            hit_ids = truth[truth.particle_id == particle_id].hit_id.values-1\n",
    "            for i in hit_ids:\n",
    "                for j in hit_ids:\n",
    "                    if i != j:\n",
    "                        pair.append([i,j])\n",
    "        pair = np.array(pair)   \n",
    "        Train1 = np.hstack((features[pair[:,0]], features[pair[:,1]], np.ones((len(pair),1))))\n",
    "\n",
    "        if len(Train) == 0:\n",
    "            Train = Train1\n",
    "        else:\n",
    "            Train = np.vstack((Train,Train1))\n",
    "\n",
    "        n = len(hits)\n",
    "        size = len(Train1)*3\n",
    "        p_id = truth.particle_id.values\n",
    "        i = np.random.randint(n, size=size)\n",
    "        j = np.random.randint(n, size=size)\n",
    "        pair = np.hstack((i.reshape(size,1),j.reshape(size,1)))\n",
    "        pair = pair[((p_id[i]==0) | (p_id[i]!=p_id[j]))]\n",
    "\n",
    "        Train0 = np.hstack((features[pair[:,0]], features[pair[:,1]], np.zeros((len(pair),1))))\n",
    "\n",
    "        print(event, Train1.shape)\n",
    "\n",
    "        Train = np.vstack((Train,Train0))\n",
    "    del Train0, Train1\n",
    "\n",
    "    np.random.shuffle(Train)\n",
    "    print(Train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "hits, cells, truth, particles = get_event('event000001002')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "hit_cells = cells.groupby(['hit_id']).value.count().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "hit_value = cells.groupby(['hit_id']).value.sum().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = np.hstack((hits[['x','y','z']]/1000, hit_cells.reshape(len(hit_cells),1)/10,hit_value.reshape(len(hit_cells),1)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125504, 5)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "Train_standard = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "94.88 only x, y, z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "95.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.5025    ,  0.09576462],\n",
       "       [-1.5025    ,  0.0557332 ],\n",
       "       [-1.5025    ,  0.05809028],\n",
       "       ...,\n",
       "       [ 2.9525    ,  0.90042692],\n",
       "       [ 2.9525    ,  0.91105782],\n",
       "       [ 2.9525    ,  0.95292453]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}