[2] train_lstm_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/dfs/scratch0/paroma/anaconda2/envs/babble/lib/python2.7/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/dfs/scratch0/paroma/anaconda2/envs/babble/lib/python2.7/site-packages/sklearn/grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n",
      "/dfs/scratch0/paroma/anaconda2/envs/babble/lib/python2.7/site-packages/sklearn/learning_curve.py:22: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the functions are moved. This module will be removed in 0.20\n",
      "  DeprecationWarning)\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "from sklearn import *\n",
    "from lstm.imdb_lstm import *\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataset\n",
    "We reload the dataset with the plain text plots and the labels that reef generated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset='imdb'\n",
    "\n",
    "from data.loader import DataLoader\n",
    "dl = DataLoader()\n",
    "_, _, _, train_ground, val_ground, test_ground, train_text, val_text, test_text = dl.load_data(dataset=dataset)\n",
    "train_reef = np.load('./data/imdb_reef.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train an LSTM Model\n",
    "We now train a simple LSTM model with the labels generated by Reef. The following hyperparameter search is simplistic, and a more fine-tuned search and a more complex model can improve performance!\n",
    "\n",
    "__Note that this takes ~1 hour to run on CPU__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /dfs/scratch0/paroma/anaconda2/envs/babble/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py:1238: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "WARNING:tensorflow:From /dfs/scratch0/paroma/anaconda2/envs/babble/lib/python2.7/site-packages/keras/backend/tensorflow_backend.py:1340: calling reduce_mean (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/5\n",
      "1136/1136 [==============================] - 20s 18ms/step - loss: 0.6874 - acc: 0.0000e+00 - val_loss: 0.6868 - val_acc: 0.5282\n",
      "Epoch 2/5\n",
      "1136/1136 [==============================] - 20s 18ms/step - loss: 0.6708 - acc: 0.0000e+00 - val_loss: 0.6337 - val_acc: 0.5634\n",
      "Epoch 3/5\n",
      "1136/1136 [==============================] - 21s 19ms/step - loss: 0.6380 - acc: 0.0000e+00 - val_loss: 0.6108 - val_acc: 0.7958\n",
      "Epoch 4/5\n",
      "1136/1136 [==============================] - 21s 18ms/step - loss: 0.6214 - acc: 0.0000e+00 - val_loss: 0.5630 - val_acc: 0.7394\n",
      "Epoch 5/5\n",
      "1136/1136 [==============================] - 20s 17ms/step - loss: 0.6088 - acc: 0.0000e+00 - val_loss: 0.5435 - val_acc: 0.7606\n",
      "Accuracy: 76.06%\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/10\n",
      "1136/1136 [==============================] - 21s 18ms/step - loss: 0.6884 - acc: 0.0000e+00 - val_loss: 0.6896 - val_acc: 0.5282\n",
      "Epoch 2/10\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6808 - acc: 0.0000e+00 - val_loss: 0.6730 - val_acc: 0.5282\n",
      "Epoch 3/10\n",
      "1136/1136 [==============================] - 19s 17ms/step - loss: 0.6441 - acc: 0.0000e+00 - val_loss: 0.5587 - val_acc: 0.7782\n",
      "Epoch 4/10\n",
      "1136/1136 [==============================] - 19s 17ms/step - loss: 0.6199 - acc: 0.0000e+00 - val_loss: 0.5457 - val_acc: 0.7676\n",
      "Epoch 5/10\n",
      "1136/1136 [==============================] - 19s 16ms/step - loss: 0.6063 - acc: 0.0000e+00 - val_loss: 0.5513 - val_acc: 0.7923\n",
      "Epoch 6/10\n",
      "1136/1136 [==============================] - 19s 16ms/step - loss: 0.6018 - acc: 0.0000e+00 - val_loss: 0.5386 - val_acc: 0.7887\n",
      "Epoch 7/10\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.5988 - acc: 0.0000e+00 - val_loss: 0.5287 - val_acc: 0.7782\n",
      "Epoch 8/10\n",
      "1136/1136 [==============================] - 19s 17ms/step - loss: 0.5974 - acc: 0.0000e+00 - val_loss: 0.5366 - val_acc: 0.7746\n",
      "Epoch 9/10\n",
      "1136/1136 [==============================] - 19s 16ms/step - loss: 0.5968 - acc: 0.0000e+00 - val_loss: 0.5310 - val_acc: 0.7817\n",
      "Epoch 10/10\n",
      "1136/1136 [==============================] - 19s 17ms/step - loss: 0.5965 - acc: 0.0000e+00 - val_loss: 0.5309 - val_acc: 0.7711\n",
      "Accuracy: 77.11%\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/25\n",
      "1136/1136 [==============================] - 20s 18ms/step - loss: 0.6875 - acc: 0.0000e+00 - val_loss: 0.6880 - val_acc: 0.5282\n",
      "Epoch 2/25\n",
      "1136/1136 [==============================] - 19s 17ms/step - loss: 0.6725 - acc: 0.0000e+00 - val_loss: 0.6253 - val_acc: 0.6655\n",
      "Epoch 3/25\n",
      "1136/1136 [==============================] - 19s 17ms/step - loss: 0.6377 - acc: 0.0000e+00 - val_loss: 0.5554 - val_acc: 0.8063\n",
      "Epoch 4/25\n",
      "1136/1136 [==============================] - 20s 18ms/step - loss: 0.6142 - acc: 0.0000e+00 - val_loss: 0.5439 - val_acc: 0.7958\n",
      "Epoch 5/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6049 - acc: 0.0000e+00 - val_loss: 0.5175 - val_acc: 0.7958\n",
      "Epoch 6/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6015 - acc: 0.0000e+00 - val_loss: 0.5331 - val_acc: 0.7711\n",
      "Epoch 7/25\n",
      "1136/1136 [==============================] - 19s 17ms/step - loss: 0.5995 - acc: 0.0000e+00 - val_loss: 0.5226 - val_acc: 0.7923\n",
      "Epoch 8/25\n",
      "1136/1136 [==============================] - 18s 15ms/step - loss: 0.5976 - acc: 0.0000e+00 - val_loss: 0.5223 - val_acc: 0.7817\n",
      "Epoch 9/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.5969 - acc: 0.0000e+00 - val_loss: 0.5211 - val_acc: 0.7887\n",
      "Epoch 10/25\n",
      "1136/1136 [==============================] - 19s 17ms/step - loss: 0.5966 - acc: 0.0000e+00 - val_loss: 0.5146 - val_acc: 0.8063\n",
      "Epoch 11/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5964 - acc: 0.0000e+00 - val_loss: 0.5216 - val_acc: 0.7852\n",
      "Epoch 12/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.5962 - acc: 0.0000e+00 - val_loss: 0.5219 - val_acc: 0.7782\n",
      "Epoch 13/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.5961 - acc: 0.0000e+00 - val_loss: 0.5216 - val_acc: 0.7782\n",
      "Epoch 14/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.5960 - acc: 0.0000e+00 - val_loss: 0.5233 - val_acc: 0.7923\n",
      "Epoch 15/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5959 - acc: 0.0000e+00 - val_loss: 0.5219 - val_acc: 0.7887\n",
      "Epoch 16/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5958 - acc: 0.0000e+00 - val_loss: 0.5236 - val_acc: 0.7887\n",
      "Epoch 17/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.5957 - acc: 0.0000e+00 - val_loss: 0.5245 - val_acc: 0.7852\n",
      "Epoch 18/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5957 - acc: 0.0000e+00 - val_loss: 0.5236 - val_acc: 0.7852\n",
      "Epoch 19/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5957 - acc: 0.0000e+00 - val_loss: 0.5225 - val_acc: 0.7887\n",
      "Epoch 20/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5957 - acc: 0.0000e+00 - val_loss: 0.5226 - val_acc: 0.7852\n",
      "Epoch 21/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5957 - acc: 0.0000e+00 - val_loss: 0.5253 - val_acc: 0.7852\n",
      "Epoch 22/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5958 - acc: 0.0000e+00 - val_loss: 0.5215 - val_acc: 0.7852\n",
      "Epoch 23/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5958 - acc: 0.0000e+00 - val_loss: 0.5221 - val_acc: 0.7852\n",
      "Epoch 24/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5958 - acc: 0.0000e+00 - val_loss: 0.5254 - val_acc: 0.7852\n",
      "Epoch 25/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5959 - acc: 0.0000e+00 - val_loss: 0.5241 - val_acc: 0.7852\n",
      "Accuracy: 78.52%\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_4 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/5\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6899 - acc: 0.0000e+00 - val_loss: 0.6902 - val_acc: 0.5282\n",
      "Epoch 2/5\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6836 - acc: 0.0000e+00 - val_loss: 0.6856 - val_acc: 0.5282\n",
      "Epoch 3/5\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6736 - acc: 0.0000e+00 - val_loss: 0.6506 - val_acc: 0.5528\n",
      "Epoch 4/5\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 1.2241 - acc: 0.0000e+00 - val_loss: 0.6202 - val_acc: 0.7782\n",
      "Epoch 5/5\n",
      "1136/1136 [==============================] - 18s 15ms/step - loss: 0.6547 - acc: 0.0000e+00 - val_loss: 0.6485 - val_acc: 0.7852\n",
      "Accuracy: 78.52%\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_5 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_5 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/10\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6903 - acc: 0.0000e+00 - val_loss: 0.6907 - val_acc: 0.5282\n",
      "Epoch 2/10\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6845 - acc: 0.0000e+00 - val_loss: 0.6864 - val_acc: 0.5282\n",
      "Epoch 3/10\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6801 - acc: 0.0000e+00 - val_loss: 0.6780 - val_acc: 0.5282\n",
      "Epoch 4/10\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6663 - acc: 0.0000e+00 - val_loss: 0.6308 - val_acc: 0.6268\n",
      "Epoch 5/10\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6340 - acc: 0.0000e+00 - val_loss: 0.5952 - val_acc: 0.7289\n",
      "Epoch 6/10\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6196 - acc: 0.0000e+00 - val_loss: 0.5597 - val_acc: 0.8204\n",
      "Epoch 7/10\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6112 - acc: 0.0000e+00 - val_loss: 0.5619 - val_acc: 0.7887\n",
      "Epoch 8/10\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6046 - acc: 0.0000e+00 - val_loss: 0.5418 - val_acc: 0.8028\n",
      "Epoch 9/10\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6012 - acc: 0.0000e+00 - val_loss: 0.5378 - val_acc: 0.7852\n",
      "Epoch 10/10\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.5992 - acc: 0.0000e+00 - val_loss: 0.5359 - val_acc: 0.7852\n",
      "Accuracy: 78.52%\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_6 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_6 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6894 - acc: 0.0000e+00 - val_loss: 0.6928 - val_acc: 0.5282\n",
      "Epoch 2/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6833 - acc: 0.0000e+00 - val_loss: 0.6853 - val_acc: 0.5282\n",
      "Epoch 3/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6717 - acc: 0.0000e+00 - val_loss: 0.6431 - val_acc: 0.5458\n",
      "Epoch 4/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6411 - acc: 0.0000e+00 - val_loss: 0.5994 - val_acc: 0.7746\n",
      "Epoch 5/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6214 - acc: 0.0000e+00 - val_loss: 0.5483 - val_acc: 0.8028\n",
      "Epoch 6/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6121 - acc: 0.0000e+00 - val_loss: 0.5323 - val_acc: 0.7711\n",
      "Epoch 7/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6053 - acc: 0.0000e+00 - val_loss: 0.5493 - val_acc: 0.7641\n",
      "Epoch 8/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6090 - acc: 0.0000e+00 - val_loss: 0.6028 - val_acc: 0.6972\n",
      "Epoch 9/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6169 - acc: 0.0000e+00 - val_loss: 0.5600 - val_acc: 0.7218\n",
      "Epoch 10/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6065 - acc: 0.0000e+00 - val_loss: 0.5377 - val_acc: 0.8134\n",
      "Epoch 11/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6021 - acc: 0.0000e+00 - val_loss: 0.5536 - val_acc: 0.7958\n",
      "Epoch 12/25\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.5998 - acc: 0.0000e+00 - val_loss: 0.5481 - val_acc: 0.7887\n",
      "Epoch 13/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5984 - acc: 0.0000e+00 - val_loss: 0.5351 - val_acc: 0.7887\n",
      "Epoch 14/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5974 - acc: 0.0000e+00 - val_loss: 0.5408 - val_acc: 0.7817\n",
      "Epoch 15/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5970 - acc: 0.0000e+00 - val_loss: 0.5347 - val_acc: 0.7958\n",
      "Epoch 16/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5967 - acc: 0.0000e+00 - val_loss: 0.5391 - val_acc: 0.7993\n",
      "Epoch 17/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5964 - acc: 0.0000e+00 - val_loss: 0.5370 - val_acc: 0.7958\n",
      "Epoch 18/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5962 - acc: 0.0000e+00 - val_loss: 0.5366 - val_acc: 0.7958\n",
      "Epoch 19/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5961 - acc: 0.0000e+00 - val_loss: 0.5363 - val_acc: 0.7887\n",
      "Epoch 20/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5960 - acc: 0.0000e+00 - val_loss: 0.5364 - val_acc: 0.7958\n",
      "Epoch 21/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5959 - acc: 0.0000e+00 - val_loss: 0.5359 - val_acc: 0.7923\n",
      "Epoch 22/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5958 - acc: 0.0000e+00 - val_loss: 0.5362 - val_acc: 0.7923\n",
      "Epoch 23/25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5958 - acc: 0.0000e+00 - val_loss: 0.5353 - val_acc: 0.7923\n",
      "Epoch 24/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5957 - acc: 0.0000e+00 - val_loss: 0.5367 - val_acc: 0.7887\n",
      "Epoch 25/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5957 - acc: 0.0000e+00 - val_loss: 0.5356 - val_acc: 0.7852\n",
      "Accuracy: 78.52%\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_7 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_7 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/5\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6916 - acc: 0.0000e+00 - val_loss: 0.6908 - val_acc: 0.5282\n",
      "Epoch 2/5\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6866 - acc: 0.0000e+00 - val_loss: 0.6923 - val_acc: 0.5282\n",
      "Epoch 3/5\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6828 - acc: 0.0000e+00 - val_loss: 0.6861 - val_acc: 0.5282\n",
      "Epoch 4/5\n",
      "1136/1136 [==============================] - 15s 13ms/step - loss: 0.6782 - acc: 0.0000e+00 - val_loss: 0.6777 - val_acc: 0.5282\n",
      "Epoch 5/5\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6669 - acc: 0.0000e+00 - val_loss: 0.6430 - val_acc: 0.5493\n",
      "Accuracy: 54.93%\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_8 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_8 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/10\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6918 - acc: 0.0000e+00 - val_loss: 0.6909 - val_acc: 0.5282\n",
      "Epoch 2/10\n",
      "1136/1136 [==============================] - 15s 14ms/step - loss: 0.6869 - acc: 0.0000e+00 - val_loss: 0.6927 - val_acc: 0.5282\n",
      "Epoch 3/10\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6833 - acc: 0.0000e+00 - val_loss: 0.6872 - val_acc: 0.5282\n",
      "Epoch 4/10\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6789 - acc: 0.0000e+00 - val_loss: 0.6800 - val_acc: 0.5282\n",
      "Epoch 5/10\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6689 - acc: 0.0000e+00 - val_loss: 0.6520 - val_acc: 0.5317\n",
      "Epoch 6/10\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6464 - acc: 0.0000e+00 - val_loss: 0.6070 - val_acc: 0.6866\n",
      "Epoch 7/10\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6309 - acc: 0.0000e+00 - val_loss: 0.5872 - val_acc: 0.7465\n",
      "Epoch 8/10\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6214 - acc: 0.0000e+00 - val_loss: 0.5766 - val_acc: 0.8239\n",
      "Epoch 9/10\n",
      "1136/1136 [==============================] - 15s 13ms/step - loss: 0.6160 - acc: 0.0000e+00 - val_loss: 0.5585 - val_acc: 0.8099\n",
      "Epoch 10/10\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6098 - acc: 0.0000e+00 - val_loss: 0.5521 - val_acc: 0.7887\n",
      "Accuracy: 78.87%\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_9 (Embedding)      (None, 500, 32)           225536    \n",
      "_________________________________________________________________\n",
      "lstm_9 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 278,837\n",
      "Trainable params: 278,837\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 284 samples\n",
      "Epoch 1/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6920 - acc: 0.0000e+00 - val_loss: 0.6911 - val_acc: 0.5282\n",
      "Epoch 2/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6873 - acc: 0.0000e+00 - val_loss: 0.6914 - val_acc: 0.5282\n",
      "Epoch 3/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6837 - acc: 0.0000e+00 - val_loss: 0.6875 - val_acc: 0.5282\n",
      "Epoch 4/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6792 - acc: 0.0000e+00 - val_loss: 0.6802 - val_acc: 0.5282\n",
      "Epoch 5/25\n",
      "1136/1136 [==============================] - 15s 14ms/step - loss: 0.6707 - acc: 0.0000e+00 - val_loss: 0.6606 - val_acc: 0.5282\n",
      "Epoch 6/25\n",
      "1136/1136 [==============================] - 15s 13ms/step - loss: 0.6485 - acc: 0.0000e+00 - val_loss: 0.6020 - val_acc: 0.7430\n",
      "Epoch 7/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6287 - acc: 0.0000e+00 - val_loss: 0.5843 - val_acc: 0.7641\n",
      "Epoch 8/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6200 - acc: 0.0000e+00 - val_loss: 0.5667 - val_acc: 0.8134\n",
      "Epoch 9/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6141 - acc: 0.0000e+00 - val_loss: 0.5623 - val_acc: 0.8134\n",
      "Epoch 10/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6095 - acc: 0.0000e+00 - val_loss: 0.5465 - val_acc: 0.7500\n",
      "Epoch 11/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6054 - acc: 0.0000e+00 - val_loss: 0.5600 - val_acc: 0.7923\n",
      "Epoch 12/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.6028 - acc: 0.0000e+00 - val_loss: 0.5460 - val_acc: 0.7641\n",
      "Epoch 13/25\n",
      "1136/1136 [==============================] - 15s 13ms/step - loss: 0.6008 - acc: 0.0000e+00 - val_loss: 0.5526 - val_acc: 0.7817\n",
      "Epoch 14/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5996 - acc: 0.0000e+00 - val_loss: 0.5431 - val_acc: 0.7782\n",
      "Epoch 15/25\n",
      "1136/1136 [==============================] - 15s 14ms/step - loss: 0.5986 - acc: 0.0000e+00 - val_loss: 0.5451 - val_acc: 0.7782\n",
      "Epoch 16/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5979 - acc: 0.0000e+00 - val_loss: 0.5445 - val_acc: 0.7746\n",
      "Epoch 17/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5974 - acc: 0.0000e+00 - val_loss: 0.5419 - val_acc: 0.7711\n",
      "Epoch 18/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5970 - acc: 0.0000e+00 - val_loss: 0.5416 - val_acc: 0.7711\n",
      "Epoch 19/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5967 - acc: 0.0000e+00 - val_loss: 0.5379 - val_acc: 0.7746\n",
      "Epoch 20/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5964 - acc: 0.0000e+00 - val_loss: 0.5398 - val_acc: 0.7676\n",
      "Epoch 21/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5963 - acc: 0.0000e+00 - val_loss: 0.5393 - val_acc: 0.7711\n",
      "Epoch 22/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5961 - acc: 0.0000e+00 - val_loss: 0.5389 - val_acc: 0.7676\n",
      "Epoch 23/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5960 - acc: 0.0000e+00 - val_loss: 0.5371 - val_acc: 0.7711\n",
      "Epoch 24/25\n",
      "1136/1136 [==============================] - 16s 14ms/step - loss: 0.5960 - acc: 0.0000e+00 - val_loss: 0.5385 - val_acc: 0.7746\n",
      "Epoch 25/25\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.5959 - acc: 0.0000e+00 - val_loss: 0.5385 - val_acc: 0.7676\n",
      "Accuracy: 76.76%\n"
     ]
    }
   ],
   "source": [
    "f1_all = []\n",
    "pr_all = []\n",
    "re_all = []\n",
    "val_acc_all = []\n",
    "\n",
    "\n",
    "bs_arr = [64,128,256]\n",
    "n_epochs_arr = [5,10,25]\n",
    "\n",
    "for bs in bs_arr:\n",
    "    for n in n_epochs_arr:\n",
    "        y_pred = lstm_simple(train_text, train_reef, val_text, val_ground, bs=bs, n=n)\n",
    "        predictions = np.round(y_pred)\n",
    "        \n",
    "        val_acc_all.append(np.sum(predictions == val_ground)/float(np.shape(val_ground)[0]))\n",
    "        f1_all.append(metrics.f1_score(val_ground, predictions))\n",
    "        pr_all.append(metrics.precision_score(val_ground, predictions))\n",
    "        re_all.append(metrics.recall_score(val_ground, predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Batch Size:  128\n",
      "Best Epochs:  5\n",
      "Validation F1 Score:  0.7874564459930314\n",
      "Validation Best Pr:  0.738562091503268\n",
      "Validation Best Re:  0.8432835820895522\n"
     ]
    }
   ],
   "source": [
    "ii,jj = np.unravel_index(np.argmax(f1_all), (3,3))\n",
    "print 'Best Batch Size: ', bs_arr[ii]\n",
    "print 'Best Epochs: ', n_epochs_arr[jj]\n",
    "\n",
    "print 'Validation F1 Score: ', max(f1_all)\n",
    "print 'Validation Best Pr: ', pr_all[np.argmax(f1_all)]\n",
    "print 'Validation Best Re: ', re_all[np.argmax(f1_all)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Performance\n",
    "We re-train the model with the best validation performance since we don't save weights for the models currently."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_10 (Embedding)     (None, 500, 32)           245344    \n",
      "_________________________________________________________________\n",
      "lstm_10 (LSTM)               (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 298,645\n",
      "Trainable params: 298,645\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 500 samples\n",
      "Epoch 1/5\n",
      "1136/1136 [==============================] - 18s 16ms/step - loss: 0.6901 - acc: 0.0000e+00 - val_loss: 0.6913 - val_acc: 0.5260\n",
      "Epoch 2/5\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6835 - acc: 0.0000e+00 - val_loss: 0.6851 - val_acc: 0.5260\n",
      "Epoch 3/5\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6741 - acc: 0.0000e+00 - val_loss: 0.6614 - val_acc: 0.5260\n",
      "Epoch 4/5\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6414 - acc: 0.0000e+00 - val_loss: 0.5875 - val_acc: 0.7680\n",
      "Epoch 5/5\n",
      "1136/1136 [==============================] - 17s 15ms/step - loss: 0.6236 - acc: 0.0000e+00 - val_loss: 0.5699 - val_acc: 0.8000\n",
      "Accuracy: 80.00%\n"
     ]
    }
   ],
   "source": [
    "y_pred = lstm_simple(train_text, train_reef, test_text, test_ground, bs=bs_arr[ii], n=n_epochs_arr[jj])\n",
    "predictions = np.round(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test F1 Score:  0.7907949790794979\n",
      "Test Precision:  0.7842323651452282\n",
      "Test Recall:  0.7974683544303798\n"
     ]
    }
   ],
   "source": [
    "print 'Test F1 Score: ', metrics.f1_score(test_ground, predictions)\n",
    "print 'Test Precision: ', metrics.precision_score(test_ground, predictions)\n",
    "print 'Test Recall: ', metrics.recall_score(test_ground, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## [Optional] Ground Truth Performance\n",
    "We can also train the same model with ground truth labels for the train set to see how far Reef labels are from the best possible performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 500, 32)           245344    \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 298,645\n",
      "Trainable params: 298,645\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 1136 samples, validate on 500 samples\n",
      "Epoch 1/10\n",
      "1136/1136 [==============================] - 168s 148ms/step - loss: 0.6747 - acc: 0.6444 - val_loss: 0.4742 - val_acc: 0.8060\n",
      "Epoch 2/10\n",
      "1136/1136 [==============================] - 164s 145ms/step - loss: 0.4225 - acc: 0.8415 - val_loss: 0.7756 - val_acc: 0.5340\n",
      "Epoch 3/10\n",
      "1136/1136 [==============================] - 162s 143ms/step - loss: 0.4225 - acc: 0.8468 - val_loss: 0.3589 - val_acc: 0.8420\n",
      "Epoch 4/10\n",
      "1136/1136 [==============================] - 162s 143ms/step - loss: 0.0911 - acc: 0.9762 - val_loss: 0.3115 - val_acc: 0.8680\n",
      "Epoch 5/10\n",
      "1136/1136 [==============================] - 158s 139ms/step - loss: 0.0235 - acc: 0.9974 - val_loss: 0.3189 - val_acc: 0.8740\n",
      "Epoch 6/10\n",
      "1136/1136 [==============================] - 162s 142ms/step - loss: 0.0114 - acc: 0.9991 - val_loss: 0.3369 - val_acc: 0.8640\n",
      "Epoch 7/10\n",
      "1136/1136 [==============================] - 158s 139ms/step - loss: 0.0085 - acc: 0.9982 - val_loss: 0.3438 - val_acc: 0.8720\n",
      "Epoch 8/10\n",
      "1136/1136 [==============================] - 160s 141ms/step - loss: 0.0067 - acc: 0.9982 - val_loss: 0.3508 - val_acc: 0.8560\n",
      "Epoch 9/10\n",
      "1136/1136 [==============================] - 161s 142ms/step - loss: 0.0052 - acc: 0.9991 - val_loss: 0.3781 - val_acc: 0.8720\n",
      "Epoch 10/10\n",
      "1136/1136 [==============================] - 162s 142ms/step - loss: 0.0040 - acc: 0.9991 - val_loss: 0.4283 - val_acc: 0.8500\n",
      "Accuracy: 85.00%\n"
     ]
    }
   ],
   "source": [
    "y_pred = lstm_simple(train_text, train_ground, test_text, test_ground, bs=5, n=10)\n",
    "predictions = np.round(y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test F1 Score:  0.8508946322067595\n",
      "Test Precision:  0.8045112781954887\n",
      "Test Recall:  0.9029535864978903\n"
     ]
    }
   ],
   "source": [
    "print 'Test F1 Score: ', metrics.f1_score(test_ground, predictions)\n",
    "print 'Test Precision: ', metrics.precision_score(test_ground, predictions)\n",
    "print 'Test Recall: ', metrics.recall_score(test_ground, predictions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}