3-Intro a DeepLearning Final Submission.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Cargamos-librerias\" data-toc-modified-id=\"Cargamos-librerias-1\">Cargamos librerias</a></span><ul class=\"toc-item\"><li><span><a href=\"#metricas-de-evaluacion-(sigmas)-+-funciones-de-utilidad\" data-toc-modified-id=\"metricas-de-evaluacion-(sigmas)-+-funciones-de-utilidad-1.1\">metricas de evaluacion (sigmas) + funciones de utilidad</a></span></li><li><span><a href=\"#Datos-de-entrenamiento!\" data-toc-modified-id=\"Datos-de-entrenamiento!-1.2\">Datos de entrenamiento!</a></span></li><li><span><a href=\"#preprocesamiento-para-X-y-Y\" data-toc-modified-id=\"preprocesamiento-para-X-y-Y-1.3\">preprocesamiento para X y Y</a></span></li></ul></li><li><span><a href=\"#Tensorflow\" data-toc-modified-id=\"Tensorflow-2\">Tensorflow</a></span></li><li><span><a href=\"#Keras\" data-toc-modified-id=\"Keras-3\">Keras</a></span><ul class=\"toc-item\"><li><span><a href=\"#Red-sencillita\" data-toc-modified-id=\"Red-sencillita-3.1\">Red sencillita</a></span></li><li><span><a href=\"#Calidad-de-vida:-liveloss!\" data-toc-modified-id=\"Calidad-de-vida:-liveloss!-3.2\">Calidad de vida: liveloss!</a></span></li><li><span><a href=\"#Calidad-de-vida:-agregando-un-tensorboard\" data-toc-modified-id=\"Calidad-de-vida:-agregando-un-tensorboard-3.3\">Calidad de vida: agregando un tensorboard</a></span></li><li><span><a href=\"#Trucos:-Regularizadores-de-redes\" data-toc-modified-id=\"Trucos:-Regularizadores-de-redes-3.4\">Trucos: Regularizadores de redes</a></span></li><li><span><a href=\"#Actividad:-Crea-una-red-profunda\" data-toc-modified-id=\"Actividad:-Crea-una-red-profunda-3.5\">Actividad: Crea una red profunda</a></span></li><li><span><a href=\"#trucos:-Dropout\" data-toc-modified-id=\"trucos:-Dropout-3.6\">trucos: Dropout</a></span></li><li><span><a href=\"#Trucos:-BatchNormalization-y-Activacion!\" data-toc-modified-id=\"Trucos:-BatchNormalization-y-Activacion!-3.7\">Trucos: BatchNormalization y Activacion!</a></span></li><li><span><a href=\"#Keras-Functional\" data-toc-modified-id=\"Keras-Functional-3.8\">Keras Functional</a></span></li><li><span><a href=\"#Truco:-Skip-connections\" data-toc-modified-id=\"Truco:-Skip-connections-3.9\">Truco: Skip connections</a></span></li><li><span><a href=\"#Optimizador\" data-toc-modified-id=\"Optimizador-3.10\">Optimizador</a></span></li><li><span><a href=\"#Controlando-el-learning-rate\" data-toc-modified-id=\"Controlando-el-learning-rate-3.11\">Controlando el learning rate</a></span></li><li><span><a href=\"#Actividad:-Crea-una-red-poderosa!\" data-toc-modified-id=\"Actividad:-Crea-una-red-poderosa!-3.12\">Actividad: Crea una red poderosa!</a></span></li></ul></li><li><span><a href=\"#Pytorch\" data-toc-modified-id=\"Pytorch-4\">Pytorch</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](extra/atlas.png)\n",
    "# Cargamos librerias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:47.151173Z",
     "start_time": "2018-08-24T08:17:46.814786Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:47.725328Z",
     "start_time": "2018-08-24T08:17:47.152976Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sc\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "from IPython.display import display\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## metricas de evaluacion (sigmas) + funciones de utilidad\n",
    "\n",
    "![http://i.imgur.com/Hflz2lG.jpg](http://i.imgur.com/Hflz2lG.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:34:42.166891Z",
     "start_time": "2018-08-24T08:34:42.157723Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "\n",
    "    \n",
    "def AMSScore(s,b): \n",
    "    return np.sqrt (2.*( (s + b + 10.)*np.log(1.+s/(b+10.))-s))\n",
    "\n",
    "def eval_model(Y_true_train,Y_pred_train,w_train,Y_true_test,Y_pred_test,w_test):\n",
    "    ratio =  float(len(X_train)) /float(len(X_test))\n",
    "    TruePositive_train = w_train*(Y_true_train==1.0)*(1.0/ratio)\n",
    "    TrueNegative_train = w_train*(Y_true_train==0.0)*(1.0/ratio)\n",
    "    TruePositive_valid = w_test*(Y_true_test==1.0)*(1.0/(1-ratio))\n",
    "    TrueNegative_valid = w_test*(Y_true_test==0.0)*(1.0/(1-ratio))\n",
    "    s_train = sum ( TruePositive_train*(Y_pred_train==1.0) )\n",
    "    b_train = sum ( TrueNegative_train*(Y_pred_train==1.0) )\n",
    "    s_test = sum ( TruePositive_valid*(Y_pred_test==1.0) )\n",
    "    b_test = sum ( TrueNegative_valid*(Y_pred_test==1.0) )\n",
    "    score_train = AMSScore(s_train,b_train)\n",
    "    score_test = AMSScore(s_test,b_test)\n",
    "    print('--- Resultados --')\n",
    "    print('- AUC train: {:.3f} '.format(roc_auc_score(Y_train,Y_train_pred)))\n",
    "    print('- AUC test : {:.3f} '.format(roc_auc_score(Y_test,Y_test_pred)))\n",
    "    print('- AMS train: {:.3f} sigma'.format(score_train))\n",
    "    print('- AMS test : {:.3f} sigma'.format(score_test))\n",
    "    return score_train, score_test\n",
    "\n",
    "def plot_roc(clf,Y_test,Y_test_prob):\n",
    "    fpr, tpr, thresholds = roc_curve(Y_test, Y_test_prob)\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    plt.plot(fpr, tpr, lw=1, alpha=0.3, label=str(clf.__class__.__name__))\n",
    "    plt.plot(np.linspace(0,1,100),np.linspace(0,1,100), lw=2, alpha=0.3, label='Suerte')\n",
    "    plt.legend(loc='lower right')\n",
    "    plt.xlim([0,1])\n",
    "    plt.ylim([0,1])\n",
    "    plt.tight_layout()\n",
    "    return\n",
    "\n",
    "def keras_graph(model):\n",
    "    from IPython.display import SVG\n",
    "    from keras.utils.vis_utils import model_to_dot\n",
    "    return SVG(model_to_dot(model).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datos de entrenamiento!\n",
    "\n",
    "Quieres saber mas? Visita"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:49.192476Z",
     "start_time": "2018-08-24T08:17:48.223762Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(250000, 33)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EventId</th>\n",
       "      <th>DER_mass_MMC</th>\n",
       "      <th>DER_mass_transverse_met_lep</th>\n",
       "      <th>DER_mass_vis</th>\n",
       "      <th>DER_pt_h</th>\n",
       "      <th>DER_deltaeta_jet_jet</th>\n",
       "      <th>DER_mass_jet_jet</th>\n",
       "      <th>DER_prodeta_jet_jet</th>\n",
       "      <th>DER_deltar_tau_lep</th>\n",
       "      <th>DER_pt_tot</th>\n",
       "      <th>...</th>\n",
       "      <th>PRI_jet_num</th>\n",
       "      <th>PRI_jet_leading_pt</th>\n",
       "      <th>PRI_jet_leading_eta</th>\n",
       "      <th>PRI_jet_leading_phi</th>\n",
       "      <th>PRI_jet_subleading_pt</th>\n",
       "      <th>PRI_jet_subleading_eta</th>\n",
       "      <th>PRI_jet_subleading_phi</th>\n",
       "      <th>PRI_jet_all_pt</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100000</td>\n",
       "      <td>138.47</td>\n",
       "      <td>51.655</td>\n",
       "      <td>97.827</td>\n",
       "      <td>27.98</td>\n",
       "      <td>0.91</td>\n",
       "      <td>124.711</td>\n",
       "      <td>2.666</td>\n",
       "      <td>3.064</td>\n",
       "      <td>41.928</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>67.435</td>\n",
       "      <td>2.15</td>\n",
       "      <td>0.444</td>\n",
       "      <td>46.062</td>\n",
       "      <td>1.24</td>\n",
       "      <td>-2.475</td>\n",
       "      <td>113.497</td>\n",
       "      <td>0.002653</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   EventId  DER_mass_MMC  DER_mass_transverse_met_lep  DER_mass_vis  DER_pt_h  \\\n",
       "0   100000        138.47                       51.655        97.827     27.98   \n",
       "\n",
       "   DER_deltaeta_jet_jet  DER_mass_jet_jet  DER_prodeta_jet_jet  \\\n",
       "0                  0.91           124.711                2.666   \n",
       "\n",
       "   DER_deltar_tau_lep  DER_pt_tot  ...    PRI_jet_num  PRI_jet_leading_pt  \\\n",
       "0               3.064      41.928  ...              2              67.435   \n",
       "\n",
       "   PRI_jet_leading_eta  PRI_jet_leading_phi  PRI_jet_subleading_pt  \\\n",
       "0                 2.15                0.444                 46.062   \n",
       "\n",
       "   PRI_jet_subleading_eta  PRI_jet_subleading_phi  PRI_jet_all_pt    Weight  \\\n",
       "0                    1.24                  -2.475         113.497  0.002653   \n",
       "\n",
       "   Label  \n",
       "0      s  \n",
       "\n",
       "[1 rows x 33 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('C:\\\\Users\\\\tkalnik\\\\PycharmProjects\\\\RIIAA_Escuela18-master\\\\datos\\\\training.csv')\n",
    "print(df.shape)\n",
    "df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## preprocesamiento para X y Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:49.327953Z",
     "start_time": "2018-08-24T08:17:49.193912Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing  # to normalize and scale the data\n",
    "Y = df['Label'].replace(to_replace=['s','b'],value=[1,0]).values\n",
    "weights = df['Weight'].values\n",
    "X = df.drop(['EventId','Label','Weight'],axis=1).values\n",
    "normalized_X = preprocessing.normalize(X)\n",
    "scaled_X = preprocessing.scale(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:17:49.531551Z",
     "start_time": "2018-08-24T08:17:49.381349Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(75000, 30) (75000,) (75000,)\n",
      "(175000, 30) (175000,) (175000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\tkalnik\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,Y_train,Y_test,w_train,w_test = train_test_split(Normalized_X,Y,weights,train_size=0.3)\n",
    "print(X_train.shape,Y_train.shape,w_train.shape)\n",
    "print(X_test.shape,Y_test.shape,w_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow\n",
    "[![](extra/tf_logo.jpg)](https://www.tensorflow.org/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T07:38:39.497500Z",
     "start_time": "2018-08-24T07:38:38.473420Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras\n",
    "\n",
    "[![](extra/keras-logo.png)](https://keras.io/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:18:17.391905Z",
     "start_time": "2018-08-24T08:18:16.014973Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras import backend as K\n",
    "import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Red sencillita"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:18:55.181558Z",
     "start_time": "2018-08-24T08:18:44.314653Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 80)                2480      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 80)                6480      \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 80)                6480      \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 80)                6480      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 80)                6480      \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 80)                6480      \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 80)                6480      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 80)                0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 1)                 81        \n",
      "=================================================================\n",
      "Total params: 41,441\n",
      "Trainable params: 41,441\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.5591 - acc: 0.7107\n",
      "Epoch 2/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.5244 - acc: 0.7328\n",
      "Epoch 3/170\n",
      "75000/75000 [==============================] - 6s 83us/step - loss: 0.5147 - acc: 0.7383\n",
      "Epoch 4/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.5063 - acc: 0.7463\n",
      "Epoch 5/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.4840 - acc: 0.7676\n",
      "Epoch 6/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.4631 - acc: 0.7830\n",
      "Epoch 7/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.4575 - acc: 0.7857\n",
      "Epoch 8/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.4504 - acc: 0.7909\n",
      "Epoch 9/170\n",
      "75000/75000 [==============================] - 6s 79us/step - loss: 0.4493 - acc: 0.7905\n",
      "Epoch 10/170\n",
      "75000/75000 [==============================] - 6s 83us/step - loss: 0.4459 - acc: 0.7923: 0s - loss: 0.4464 - acc:\n",
      "Epoch 11/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.4432 - acc: 0.7907\n",
      "Epoch 12/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.4410 - acc: 0.7946\n",
      "Epoch 13/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.4384 - acc: 0.7948\n",
      "Epoch 14/170\n",
      "75000/75000 [==============================] - 7s 88us/step - loss: 0.4365 - acc: 0.7969\n",
      "Epoch 15/170\n",
      "75000/75000 [==============================] - 7s 87us/step - loss: 0.4347 - acc: 0.7976\n",
      "Epoch 16/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.4342 - acc: 0.7987\n",
      "Epoch 17/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.4305 - acc: 0.8005\n",
      "Epoch 18/170\n",
      "75000/75000 [==============================] - 5s 73us/step - loss: 0.4280 - acc: 0.8026\n",
      "Epoch 19/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.4278 - acc: 0.8023\n",
      "Epoch 20/170\n",
      "75000/75000 [==============================] - 6s 74us/step - loss: 0.4256 - acc: 0.8046\n",
      "Epoch 21/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.4232 - acc: 0.8059\n",
      "Epoch 22/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.4208 - acc: 0.8052\n",
      "Epoch 23/170\n",
      "75000/75000 [==============================] - 6s 83us/step - loss: 0.4185 - acc: 0.8087\n",
      "Epoch 24/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.4173 - acc: 0.8087\n",
      "Epoch 25/170\n",
      "75000/75000 [==============================] - 6s 79us/step - loss: 0.4142 - acc: 0.8106\n",
      "Epoch 26/170\n",
      "75000/75000 [==============================] - 6s 74us/step - loss: 0.4136 - acc: 0.8109\n",
      "Epoch 27/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.4123 - acc: 0.8121\n",
      "Epoch 28/170\n",
      "75000/75000 [==============================] - 5s 70us/step - loss: 0.4107 - acc: 0.8133\n",
      "Epoch 29/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.4095 - acc: 0.8135: 0s - loss: 0.4\n",
      "Epoch 30/170\n",
      "75000/75000 [==============================] - 6s 79us/step - loss: 0.4077 - acc: 0.8141\n",
      "Epoch 31/170\n",
      "75000/75000 [==============================] - 6s 74us/step - loss: 0.4055 - acc: 0.8160\n",
      "Epoch 32/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.4045 - acc: 0.8167\n",
      "Epoch 33/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.4029 - acc: 0.8173\n",
      "Epoch 34/170\n",
      "75000/75000 [==============================] - 6s 82us/step - loss: 0.4038 - acc: 0.8173\n",
      "Epoch 35/170\n",
      "75000/75000 [==============================] - 6s 82us/step - loss: 0.4016 - acc: 0.8167\n",
      "Epoch 36/170\n",
      "75000/75000 [==============================] - 6s 83us/step - loss: 0.4010 - acc: 0.8178\n",
      "Epoch 37/170\n",
      "75000/75000 [==============================] - 6s 82us/step - loss: 0.4016 - acc: 0.8174\n",
      "Epoch 38/170\n",
      "75000/75000 [==============================] - 6s 83us/step - loss: 0.4000 - acc: 0.8176\n",
      "Epoch 39/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3995 - acc: 0.8187\n",
      "Epoch 40/170\n",
      "75000/75000 [==============================] - 7s 99us/step - loss: 0.3996 - acc: 0.8190\n",
      "Epoch 41/170\n",
      "75000/75000 [==============================] - 5s 73us/step - loss: 0.3977 - acc: 0.8193\n",
      "Epoch 42/170\n",
      "75000/75000 [==============================] - 5s 72us/step - loss: 0.3973 - acc: 0.8195\n",
      "Epoch 43/170\n",
      "75000/75000 [==============================] - 7s 87us/step - loss: 0.3969 - acc: 0.8195\n",
      "Epoch 44/170\n",
      "75000/75000 [==============================] - 7s 89us/step - loss: 0.3960 - acc: 0.8198\n",
      "Epoch 45/170\n",
      "75000/75000 [==============================] - 8s 100us/step - loss: 0.3951 - acc: 0.8215\n",
      "Epoch 46/170\n",
      "75000/75000 [==============================] - 6s 82us/step - loss: 0.3944 - acc: 0.8224\n",
      "Epoch 47/170\n",
      "75000/75000 [==============================] - 6s 82us/step - loss: 0.3945 - acc: 0.8213\n",
      "Epoch 48/170\n",
      "75000/75000 [==============================] - 6s 82us/step - loss: 0.3945 - acc: 0.8201\n",
      "Epoch 49/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3950 - acc: 0.8218\n",
      "Epoch 50/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.3922 - acc: 0.8224\n",
      "Epoch 51/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3915 - acc: 0.8215\n",
      "Epoch 52/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3924 - acc: 0.8212\n",
      "Epoch 53/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3918 - acc: 0.8223\n",
      "Epoch 54/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3912 - acc: 0.8237\n",
      "Epoch 55/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3917 - acc: 0.8216\n",
      "Epoch 56/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3899 - acc: 0.8232\n",
      "Epoch 57/170\n",
      "75000/75000 [==============================] - 7s 88us/step - loss: 0.3907 - acc: 0.8230\n",
      "Epoch 58/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3889 - acc: 0.8232\n",
      "Epoch 59/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3886 - acc: 0.8240\n",
      "Epoch 60/170\n",
      "75000/75000 [==============================] - 7s 87us/step - loss: 0.3892 - acc: 0.8238\n",
      "Epoch 61/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3890 - acc: 0.8231\n",
      "Epoch 62/170\n",
      "75000/75000 [==============================] - 7s 89us/step - loss: 0.3876 - acc: 0.8244\n",
      "Epoch 63/170\n",
      "75000/75000 [==============================] - 7s 92us/step - loss: 0.3878 - acc: 0.8242\n",
      "Epoch 64/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3877 - acc: 0.8246\n",
      "Epoch 65/170\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75000/75000 [==============================] - 6s 83us/step - loss: 0.3871 - acc: 0.8252: 1\n",
      "Epoch 66/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.3870 - acc: 0.8245\n",
      "Epoch 67/170\n",
      "75000/75000 [==============================] - 6s 77us/step - loss: 0.3860 - acc: 0.8249\n",
      "Epoch 68/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.3869 - acc: 0.8251\n",
      "Epoch 69/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.3847 - acc: 0.8254\n",
      "Epoch 70/170\n",
      "75000/75000 [==============================] - 6s 77us/step - loss: 0.3860 - acc: 0.8239\n",
      "Epoch 71/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.3851 - acc: 0.8252\n",
      "Epoch 72/170\n",
      "75000/75000 [==============================] - 6s 77us/step - loss: 0.3849 - acc: 0.8255\n",
      "Epoch 73/170\n",
      "75000/75000 [==============================] - 7s 89us/step - loss: 0.3839 - acc: 0.8257\n",
      "Epoch 74/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.3835 - acc: 0.8260\n",
      "Epoch 75/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.3830 - acc: 0.8270\n",
      "Epoch 76/170\n",
      "75000/75000 [==============================] - 6s 79us/step - loss: 0.3823 - acc: 0.8265\n",
      "Epoch 77/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3824 - acc: 0.8272\n",
      "Epoch 78/170\n",
      "75000/75000 [==============================] - 5s 66us/step - loss: 0.3836 - acc: 0.8258\n",
      "Epoch 79/170\n",
      "75000/75000 [==============================] - 5s 66us/step - loss: 0.3812 - acc: 0.8272\n",
      "Epoch 80/170\n",
      "75000/75000 [==============================] - 5s 73us/step - loss: 0.3816 - acc: 0.8268\n",
      "Epoch 81/170\n",
      "75000/75000 [==============================] - 5s 71us/step - loss: 0.3811 - acc: 0.8272\n",
      "Epoch 82/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3807 - acc: 0.8273\n",
      "Epoch 83/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.3807 - acc: 0.8271\n",
      "Epoch 84/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.3805 - acc: 0.8276\n",
      "Epoch 85/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3789 - acc: 0.8282\n",
      "Epoch 86/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3789 - acc: 0.8283\n",
      "Epoch 87/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3789 - acc: 0.8284\n",
      "Epoch 88/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.3788 - acc: 0.8284\n",
      "Epoch 89/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.3784 - acc: 0.8279\n",
      "Epoch 90/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3793 - acc: 0.8277\n",
      "Epoch 91/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3774 - acc: 0.8299\n",
      "Epoch 92/170\n",
      "75000/75000 [==============================] - 6s 79us/step - loss: 0.3768 - acc: 0.8291: 1s - lo\n",
      "Epoch 93/170\n",
      "75000/75000 [==============================] - 6s 79us/step - loss: 0.3761 - acc: 0.8297\n",
      "Epoch 94/170\n",
      "75000/75000 [==============================] - 6s 82us/step - loss: 0.3753 - acc: 0.8300\n",
      "Epoch 95/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.3751 - acc: 0.8298\n",
      "Epoch 96/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.3762 - acc: 0.8290\n",
      "Epoch 97/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3756 - acc: 0.8298\n",
      "Epoch 98/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3745 - acc: 0.8296\n",
      "Epoch 99/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.3749 - acc: 0.8291\n",
      "Epoch 100/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.3728 - acc: 0.8295\n",
      "Epoch 101/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.3733 - acc: 0.8304\n",
      "Epoch 102/170\n",
      "75000/75000 [==============================] - 7s 90us/step - loss: 0.3719 - acc: 0.8290\n",
      "Epoch 103/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3717 - acc: 0.8323\n",
      "Epoch 104/170\n",
      "75000/75000 [==============================] - 6s 73us/step - loss: 0.3733 - acc: 0.8305\n",
      "Epoch 105/170\n",
      "75000/75000 [==============================] - 5s 70us/step - loss: 0.3727 - acc: 0.8315\n",
      "Epoch 106/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.3710 - acc: 0.8322\n",
      "Epoch 107/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3718 - acc: 0.8306\n",
      "Epoch 108/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.3692 - acc: 0.8320\n",
      "Epoch 109/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3706 - acc: 0.8324\n",
      "Epoch 110/170\n",
      "75000/75000 [==============================] - 7s 90us/step - loss: 0.3701 - acc: 0.8323\n",
      "Epoch 111/170\n",
      "75000/75000 [==============================] - 6s 82us/step - loss: 0.3692 - acc: 0.8325\n",
      "Epoch 112/170\n",
      "75000/75000 [==============================] - 7s 89us/step - loss: 0.3709 - acc: 0.8321\n",
      "Epoch 113/170\n",
      "75000/75000 [==============================] - 7s 97us/step - loss: 0.3694 - acc: 0.8328\n",
      "Epoch 114/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3680 - acc: 0.8329\n",
      "Epoch 115/170\n",
      "75000/75000 [==============================] - 7s 98us/step - loss: 0.3702 - acc: 0.8335\n",
      "Epoch 116/170\n",
      "75000/75000 [==============================] - 7s 89us/step - loss: 0.3682 - acc: 0.8317\n",
      "Epoch 117/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3685 - acc: 0.8321\n",
      "Epoch 118/170\n",
      "75000/75000 [==============================] - 7s 88us/step - loss: 0.3696 - acc: 0.8310\n",
      "Epoch 119/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.3646 - acc: 0.8346: 0s - loss: 0.3641 - acc: \n",
      "Epoch 120/170\n",
      "75000/75000 [==============================] - 6s 83us/step - loss: 0.3665 - acc: 0.8357\n",
      "Epoch 121/170\n",
      "75000/75000 [==============================] - 7s 88us/step - loss: 0.3685 - acc: 0.8329\n",
      "Epoch 122/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3658 - acc: 0.8349\n",
      "Epoch 123/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.3686 - acc: 0.8336\n",
      "Epoch 124/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.3656 - acc: 0.8349\n",
      "Epoch 125/170\n",
      "75000/75000 [==============================] - ETA: 0s - loss: 0.3648 - acc: 0.835 - 6s 85us/step - loss: 0.3646 - acc: 0.8350\n",
      "Epoch 126/170\n",
      "75000/75000 [==============================] - 7s 89us/step - loss: 0.3656 - acc: 0.8346\n",
      "Epoch 127/170\n",
      "75000/75000 [==============================] - 7s 88us/step - loss: 0.3679 - acc: 0.8341\n",
      "Epoch 128/170\n",
      "75000/75000 [==============================] - 7s 87us/step - loss: 0.3647 - acc: 0.8341\n",
      "Epoch 129/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.3683 - acc: 0.8331\n",
      "Epoch 130/170\n",
      "75000/75000 [==============================] - 7s 88us/step - loss: 0.3636 - acc: 0.8354\n",
      "Epoch 131/170\n",
      "75000/75000 [==============================] - 6s 85us/step - loss: 0.3631 - acc: 0.8345\n",
      "Epoch 132/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3703 - acc: 0.8331\n",
      "Epoch 133/170\n",
      "75000/75000 [==============================] - 7s 90us/step - loss: 0.3746 - acc: 0.8316\n",
      "Epoch 134/170\n",
      "75000/75000 [==============================] - 7s 89us/step - loss: 0.3699 - acc: 0.8319\n",
      "Epoch 135/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3639 - acc: 0.8349\n",
      "Epoch 136/170\n",
      "75000/75000 [==============================] - 7s 88us/step - loss: 0.3622 - acc: 0.8354\n",
      "Epoch 137/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3656 - acc: 0.8347\n",
      "Epoch 138/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3630 - acc: 0.8351\n",
      "Epoch 139/170\n",
      "75000/75000 [==============================] - 6s 86us/step - loss: 0.3659 - acc: 0.8351\n",
      "Epoch 140/170\n",
      "75000/75000 [==============================] - 7s 89us/step - loss: 0.3623 - acc: 0.8361\n",
      "Epoch 141/170\n",
      "75000/75000 [==============================] - 7s 90us/step - loss: 0.3640 - acc: 0.8351\n",
      "Epoch 142/170\n",
      "75000/75000 [==============================] - 6s 87us/step - loss: 0.3629 - acc: 0.8352\n",
      "Epoch 143/170\n",
      "75000/75000 [==============================] - 6s 87us/step - loss: 0.3619 - acc: 0.8353\n",
      "Epoch 144/170\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75000/75000 [==============================] - 6s 77us/step - loss: 0.3598 - acc: 0.8371\n",
      "Epoch 145/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.3676 - acc: 0.8335\n",
      "Epoch 146/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.3636 - acc: 0.8345\n",
      "Epoch 147/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.3602 - acc: 0.8375\n",
      "Epoch 148/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.3678 - acc: 0.8353\n",
      "Epoch 149/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.4013 - acc: 0.8152\n",
      "Epoch 150/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.3806 - acc: 0.8291\n",
      "Epoch 151/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.3695 - acc: 0.8328\n",
      "Epoch 152/170\n",
      "75000/75000 [==============================] - 6s 75us/step - loss: 0.3657 - acc: 0.8327\n",
      "Epoch 153/170\n",
      "75000/75000 [==============================] - 6s 76us/step - loss: 0.3640 - acc: 0.8359\n",
      "Epoch 154/170\n",
      "75000/75000 [==============================] - 6s 77us/step - loss: 0.3628 - acc: 0.8361\n",
      "Epoch 155/170\n",
      "75000/75000 [==============================] - 6s 79us/step - loss: 0.3657 - acc: 0.8357\n",
      "Epoch 156/170\n",
      "75000/75000 [==============================] - 6s 79us/step - loss: 0.3634 - acc: 0.8358\n",
      "Epoch 157/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.3664 - acc: 0.8350\n",
      "Epoch 158/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3634 - acc: 0.8353\n",
      "Epoch 159/170\n",
      "75000/75000 [==============================] - 6s 83us/step - loss: 0.3648 - acc: 0.8350\n",
      "Epoch 160/170\n",
      "75000/75000 [==============================] - 7s 87us/step - loss: 0.3593 - acc: 0.8375: 1s - loss: \n",
      "Epoch 161/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.3590 - acc: 0.8366\n",
      "Epoch 162/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3665 - acc: 0.8353\n",
      "Epoch 163/170\n",
      "75000/75000 [==============================] - 6s 84us/step - loss: 0.3587 - acc: 0.8370\n",
      "Epoch 164/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3581 - acc: 0.8375\n",
      "Epoch 165/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3595 - acc: 0.8381\n",
      "Epoch 166/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3596 - acc: 0.8387\n",
      "Epoch 167/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3589 - acc: 0.8375\n",
      "Epoch 168/170\n",
      "75000/75000 [==============================] - 6s 78us/step - loss: 0.3609 - acc: 0.8366\n",
      "Epoch 169/170\n",
      "75000/75000 [==============================] - 6s 80us/step - loss: 0.3686 - acc: 0.8341\n",
      "Epoch 170/170\n",
      "75000/75000 [==============================] - 6s 81us/step - loss: 0.3624 - acc: 0.8380\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1a830f33080>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.layers import Dense, Activation, Dropout\n",
    "\n",
    "K.clear_session()\n",
    "tb_call = keras.callbacks.TensorBoard(log_dir='C:\\\\Users\\\\tkalnik\\\\PycharmProjects\\\\RIIAA_Escuela18-master\\\\logs3', histogram_freq=0, write_graph=True, write_images=True)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(units=80, input_dim=X_train.shape[-1], kernel_initializer='TruncatedNormal', activation='elu'))\n",
    "model.add(Dense(units=80, input_dim=X_train.shape[-1], kernel_initializer='TruncatedNormal', activation='elu'))\n",
    "model.add(Dense(units=80, input_dim=X_train.shape[-1], kernel_initializer='TruncatedNormal', activation='elu'))\n",
    "model.add(Dense(units=80, input_dim=X_train.shape[-1], kernel_initializer='TruncatedNormal', activation='elu'))\n",
    "model.add(Dense(units=80, input_dim=X_train.shape[-1], kernel_initializer='TruncatedNormal', activation='elu'))\n",
    "model.add(Dense(units=80, input_dim=X_train.shape[-1], kernel_initializer='TruncatedNormal', activation='elu'))\n",
    "model.add(Dense(units=80, input_dim=X_train.shape[-1], kernel_initializer='TruncatedNormal', activation='elu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, kernel_initializer='TruncatedNormal', activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer=\"Adam\", metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "model.fit(X_train, Y_train, epochs=170, batch_size=28, callbacks=[tb_call])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-24T08:04:10.725399Z",
     "start_time": "2018-08-24T08:04:04.945904Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Resultados --\n",
      "- AUC train: 0.804 \n",
      "- AUC test : 0.780 \n",
      "- AMS train: 2.529 sigma\n",
      "- AMS test : 3.111 sigma\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.528916246247552, 3.1110115290247444)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pcut=0.5\n",
    "Y_train_pred = model.predict(X_train).flatten() > pcut\n",
    "Y_test_pred = model.predict(X_test).flatten() > pcut\n",
    "Y_train_prob= model.predict_proba(X_train).flatten()\n",
    "Y_test_prob = model.predict_proba(X_test).flatten()\n",
    "eval_model(Y_train,Y_train_pred,w_train,Y_test,Y_test_pred,w_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "209px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}