2.2 attend_encode_attent_input_part1_params_solo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Setup Logging\n",
    "import logging\n",
    "logging_format = '%(asctime)s - %(levelname)s - %(message)s'\n",
    "logging.basicConfig(level=logging.INFO,filename=None, format=logging_format)\n",
    "log = logging.getLogger(__name__)\n",
    "    \n",
    "from att_emb_enc import preproc as pp\n",
    "from att_emb_enc import utils\n",
    "from att_emb_enc import embed\n",
    "from att_emb_enc import modelling as mod\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Sklearn\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Datetime\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:05:12,913 - INFO - Input Data Loaded From data/processed_inputs/X_Y_word_index_maxfeat20000_maxlen300.pkl\n",
      "2019-06-11 07:05:12,951 - INFO - Embeddings File Loaded From data/processed_embeddings/avg_embeddings_maxfeat20000.pkl\n",
      "\n",
      "2019-06-11 07:05:12,951 - INFO - Inputs and Embedding Matrix Files Loaded\n"
     ]
    }
   ],
   "source": [
    "# Get Data\n",
    "X, Y, vocabulary = utils.create_or_load_preproc(20000, 300, False)\n",
    "embed_matrix = utils.create_or_load_embeddings(vocabulary, 20000, 300, False)\n",
    "log.info(\"Inputs and Embedding Matrix Files Loaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:05:12,990 - INFO - Training Shape = (31397, 37)\n",
      "2019-06-11 07:05:12,990 - INFO - Testing Shape = (3489, 37)\n"
     ]
    }
   ],
   "source": [
    "# Split Test Set From All Articles\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X, Y, test_size=0.1, shuffle=True, random_state=42)\n",
    "log.info(f\"Training Shape = {y_train.shape}\")\n",
    "log.info(f\"Testing Shape = {y_test.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "6 * 4 * 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:05:13,027 - WARNING - From /home/jmoore/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "2019-06-11 07:05:14,346 - WARNING - From /home/jmoore/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00023: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:05:25,293 - INFO - Model 0 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:05:25,295 - INFO - Settings - n_samples = 1000, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00047: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:05:59,545 - INFO - Model 1 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:05:59,547 - INFO - Settings - n_samples = 1875, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00029: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:06:38,793 - INFO - Model 2 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:06:38,795 - INFO - Settings - n_samples = 3750, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00028: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:07:52,578 - INFO - Model 3 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:07:52,580 - INFO - Settings - n_samples = 7500, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00033: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:10:42,960 - INFO - Model 4 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:10:42,962 - INFO - Settings - n_samples = 15000, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00026: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:15:09,184 - INFO - Model 5 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:15:09,185 - INFO - Settings - n_samples = 30000, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00032: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:15:26,696 - INFO - Model 6 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:15:26,697 - INFO - Settings - n_samples = 1000, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00015: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:15:42,406 - INFO - Model 7 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:15:42,407 - INFO - Settings - n_samples = 1875, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00024: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:16:24,298 - INFO - Model 8 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:16:24,300 - INFO - Settings - n_samples = 3750, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00031: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:18:04,985 - INFO - Model 9 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:18:04,986 - INFO - Settings - n_samples = 7500, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00033: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:21:35,114 - INFO - Model 10 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:21:35,115 - INFO - Settings - n_samples = 15000, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00026: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:27:03,554 - INFO - Model 11 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:27:03,555 - INFO - Settings - n_samples = 30000, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00027: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:27:25,876 - INFO - Model 12 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:27:25,877 - INFO - Settings - n_samples = 1000, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00025: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:27:59,758 - INFO - Model 13 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:27:59,759 - INFO - Settings - n_samples = 1875, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00010: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:28:28,881 - INFO - Model 14 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:28:28,882 - INFO - Settings - n_samples = 3750, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00026: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:30:27,014 - INFO - Model 15 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:30:27,016 - INFO - Settings - n_samples = 7500, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00029: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:34:40,971 - INFO - Model 16 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:34:40,972 - INFO - Settings - n_samples = 15000, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00019: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:40:12,547 - INFO - Model 17 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:40:12,549 - INFO - Settings - n_samples = 30000, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00020: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:40:37,942 - INFO - Model 18 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:40:37,943 - INFO - Settings - n_samples = 1000, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00011: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:41:03,638 - INFO - Model 19 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:41:03,639 - INFO - Settings - n_samples = 1875, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00018: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:42:03,936 - INFO - Model 20 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:42:03,937 - INFO - Settings - n_samples = 3750, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00019: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:43:59,467 - INFO - Model 21 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:43:59,468 - INFO - Settings - n_samples = 7500, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00014: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:46:45,619 - INFO - Model 22 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:46:45,620 - INFO - Settings - n_samples = 15000, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00015: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:52:50,004 - INFO - Model 23 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:52:50,006 - INFO - Settings - n_samples = 30000, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00065: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:53:30,731 - INFO - Model 24 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:53:30,732 - INFO - Settings - n_samples = 1000, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00043: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:54:15,907 - INFO - Model 25 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:54:15,908 - INFO - Settings - n_samples = 1875, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00014: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:54:50,254 - INFO - Model 26 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 07:54:50,254 - INFO - Settings - n_samples = 3750, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00032: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:56:32,621 - INFO - Model 27 - Prec 0.4423076914571006 - Rec 0.27380952348356014\n",
      "2019-06-11 07:56:32,622 - INFO - Settings - n_samples = 7500, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00032: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 07:59:43,390 - INFO - Model 28 - Prec 0.6279069752839372 - Rec 0.3214285710459184\n",
      "2019-06-11 07:59:43,391 - INFO - Settings - n_samples = 15000, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00026: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:04:32,684 - INFO - Model 29 - Prec 0.5098039205690119 - Rec 0.30952380915532884\n",
      "2019-06-11 08:04:32,685 - INFO - Settings - n_samples = 30000, n_hidden = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00040: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:05:08,276 - INFO - Model 30 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 08:05:08,276 - INFO - Settings - n_samples = 1000, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00039: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:05:59,294 - INFO - Model 31 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 08:05:59,295 - INFO - Settings - n_samples = 1875, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00018: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:06:48,115 - INFO - Model 32 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 08:06:48,116 - INFO - Settings - n_samples = 3750, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00023: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:08:19,701 - INFO - Model 33 - Prec 0.49999999375 - Rec 0.04761904756235828\n",
      "2019-06-11 08:08:19,702 - INFO - Settings - n_samples = 7500, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00020: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:10:45,767 - INFO - Model 34 - Prec 0.38461538165680476 - Rec 0.05952380945294785\n",
      "2019-06-11 08:10:45,768 - INFO - Settings - n_samples = 15000, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00028: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:17:09,321 - INFO - Model 35 - Prec 0.605263157098338 - Rec 0.5476190469671203\n",
      "2019-06-11 08:17:09,321 - INFO - Settings - n_samples = 30000, n_hidden = 32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00022: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:17:43,228 - INFO - Model 36 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 08:17:43,229 - INFO - Settings - n_samples = 1000, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00033: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:18:42,449 - INFO - Model 37 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 08:18:42,450 - INFO - Settings - n_samples = 1875, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00032: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:20:16,399 - INFO - Model 38 - Prec 0.4482758605231867 - Rec 0.15476190457766442\n",
      "2019-06-11 08:20:16,400 - INFO - Settings - n_samples = 3750, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00028: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:22:43,560 - INFO - Model 39 - Prec 0.6923076896449704 - Rec 0.21428571403061225\n",
      "2019-06-11 08:22:43,562 - INFO - Settings - n_samples = 7500, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00023: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:27:02,318 - INFO - Model 40 - Prec 0.553191488184699 - Rec 0.30952380915532884\n",
      "2019-06-11 08:27:02,320 - INFO - Settings - n_samples = 15000, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00020: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:34:24,704 - INFO - Model 41 - Prec 0.7878787866850322 - Rec 0.6190476183106577\n",
      "2019-06-11 08:34:24,706 - INFO - Settings - n_samples = 30000, n_hidden = 64\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00020: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:35:29,789 - INFO - Model 42 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 08:35:29,790 - INFO - Settings - n_samples = 1000, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00018: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:36:20,978 - INFO - Model 43 - Prec 0.0 - Rec 0.0\n",
      "2019-06-11 08:36:20,980 - INFO - Settings - n_samples = 1875, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00030: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:38:31,077 - INFO - Model 44 - Prec 0.5185185165980796 - Rec 0.16666666646825398\n",
      "2019-06-11 08:38:31,078 - INFO - Settings - n_samples = 3750, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00022: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:42:00,613 - INFO - Model 45 - Prec 0.49999999886363633 - Rec 0.26190476159297055\n",
      "2019-06-11 08:42:00,615 - INFO - Settings - n_samples = 7500, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00015: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:46:34,432 - INFO - Model 46 - Prec 0.4411764692906574 - Rec 0.17857142835884354\n",
      "2019-06-11 08:46:34,434 - INFO - Settings - n_samples = 15000, n_hidden = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00017: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 08:54:56,325 - INFO - Model 47 - Prec 0.6521739120982988 - Rec 0.5357142850765306\n",
      "2019-06-11 08:54:56,328 - INFO - Settings - n_samples = 30000, n_hidden = 96\n"
     ]
    }
   ],
   "source": [
    "# Create Parameter Grid\n",
    "param_grid = {\n",
    "    'n_hidden': [16, 32, 64, 96], \n",
    "    'number_of_texts': [1000, 1875, 3750, 7500, 15000, 30000], \n",
    "    'activation': ['softmax', 'sigmoid']\n",
    "}\n",
    "param_grid = list(ParameterGrid(param_grid))\n",
    "\n",
    "n_samples_li = []\n",
    "n_hidden_li = []\n",
    "activation_li = []\n",
    "training_time_li = []\n",
    "val_loss_li = []\n",
    "val_acc_li = []\n",
    "val_prec_li = []\n",
    "val_rec_li = []\n",
    "test_loss_li = []\n",
    "test_acc_li = []\n",
    "test_prec_li = []\n",
    "test_rec_li = []\n",
    "\n",
    "for ix, grid in enumerate(param_grid):\n",
    "    start = datetime.now().timestamp()\n",
    "    # Get Data for sample size\n",
    "    X_train_grid = X_train[0:grid['number_of_texts']].copy()\n",
    "    Y_train_grid = y_train[0:grid['number_of_texts']].copy()\n",
    "    Xtr, Xval, ytr, yval = train_test_split(\n",
    "        X_train_grid, Y_train_grid, test_size=0.25, shuffle=False, random_state=42)\n",
    "    \n",
    "    # Train Model\n",
    "    model, history = mod.train_and_return_model(\n",
    "        Xtr, Xval, ytr, yval, embed_matrix, 300, 20000, 300, grid['n_hidden'], grid['activation'])\n",
    "    \n",
    "    # Extract Results\n",
    "    val_loss, val_acc, val_prec, val_rec = utils.extract_validation_results(history)\n",
    "    test_loss, test_acc, test_prec, test_rec = model.evaluate(X_test, y_test, batch_size=1024, verbose=0)\n",
    "    end = datetime.now().timestamp()\n",
    "    \n",
    "    # Save To Lists\n",
    "    n_samples_li.append(grid['number_of_texts'])\n",
    "    n_hidden_li.append(grid['n_hidden'])\n",
    "    training_time_li.append(end - start)\n",
    "    activation_li.append(grid['activation'])\n",
    "    val_loss_li.append(val_loss)\n",
    "    val_acc_li.append(val_acc)\n",
    "    val_prec_li.append(val_prec)\n",
    "    val_rec_li.append(val_rec)\n",
    "    test_loss_li.append(test_loss)\n",
    "    test_acc_li.append(test_acc)\n",
    "    test_prec_li.append(test_prec)\n",
    "    test_rec_li.append(test_rec)\n",
    "    \n",
    "    # Log Results\n",
    "    log.info(f\"Model {ix} - Prec {test_prec} - Rec {test_rec}\")\n",
    "    log.info(f\"Settings - n_samples = {grid['number_of_texts']}, n_hidden = {grid['n_hidden']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "results_df = pd.DataFrame({\n",
    "    'n_samples': n_samples_li,\n",
    "    'n_hidden': n_hidden_li,\n",
    "    'activation':activation_li,\n",
    "    'training_time': training_time_li,\n",
    "    'val_loss': val_loss_li,\n",
    "    'val_accuracy': val_acc_li,\n",
    "    'val_precision': val_prec_li,\n",
    "    'val_recall': val_rec_li,\n",
    "    'test_loss': test_loss_li,\n",
    "    'test_accuracy': test_acc_li,\n",
    "    'test_precision': test_prec_li,\n",
    "    'test_recall': test_rec_li,\n",
    "})\n",
    "# Append F1 - Score\n",
    "results_df['val_f1'] = results_df.apply(lambda x: utils.calc_f1(x.val_precision, x.val_recall), axis=1)\n",
    "results_df['test_f1'] = results_df.apply(lambda x: utils.calc_f1(x.test_precision, x.test_recall), axis=1)\n",
    "results_df.sort_values('test_loss', ascending=True).to_csv('data/results_enc_emb_att.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_samples</th>\n",
       "      <th>n_hidden</th>\n",
       "      <th>activation</th>\n",
       "      <th>training_time</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>val_accuracy</th>\n",
       "      <th>val_precision</th>\n",
       "      <th>val_recall</th>\n",
       "      <th>test_loss</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_precision</th>\n",
       "      <th>test_recall</th>\n",
       "      <th>val_f1</th>\n",
       "      <th>test_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>30000</td>\n",
       "      <td>96</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>501.888605</td>\n",
       "      <td>0.160303</td>\n",
       "      <td>0.935950</td>\n",
       "      <td>0.664384</td>\n",
       "      <td>0.625806</td>\n",
       "      <td>0.163766</td>\n",
       "      <td>0.934776</td>\n",
       "      <td>0.652174</td>\n",
       "      <td>0.535714</td>\n",
       "      <td>0.644518</td>\n",
       "      <td>0.588235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>30000</td>\n",
       "      <td>64</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>442.382329</td>\n",
       "      <td>0.165130</td>\n",
       "      <td>0.936807</td>\n",
       "      <td>0.837838</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.168144</td>\n",
       "      <td>0.936162</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>0.619048</td>\n",
       "      <td>0.699248</td>\n",
       "      <td>0.693333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>30000</td>\n",
       "      <td>32</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>383.550871</td>\n",
       "      <td>0.166100</td>\n",
       "      <td>0.934094</td>\n",
       "      <td>0.617834</td>\n",
       "      <td>0.625806</td>\n",
       "      <td>0.169503</td>\n",
       "      <td>0.931801</td>\n",
       "      <td>0.605263</td>\n",
       "      <td>0.547619</td>\n",
       "      <td>0.621795</td>\n",
       "      <td>0.575000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30000</td>\n",
       "      <td>16</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>289.291727</td>\n",
       "      <td>0.171320</td>\n",
       "      <td>0.929452</td>\n",
       "      <td>0.567010</td>\n",
       "      <td>0.354839</td>\n",
       "      <td>0.175454</td>\n",
       "      <td>0.927053</td>\n",
       "      <td>0.509804</td>\n",
       "      <td>0.309524</td>\n",
       "      <td>0.436508</td>\n",
       "      <td>0.385185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>15000</td>\n",
       "      <td>96</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>273.815420</td>\n",
       "      <td>0.175283</td>\n",
       "      <td>0.931596</td>\n",
       "      <td>0.480769</td>\n",
       "      <td>0.287356</td>\n",
       "      <td>0.184181</td>\n",
       "      <td>0.925751</td>\n",
       "      <td>0.441176</td>\n",
       "      <td>0.178571</td>\n",
       "      <td>0.359712</td>\n",
       "      <td>0.254237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>15000</td>\n",
       "      <td>16</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>190.766743</td>\n",
       "      <td>0.177503</td>\n",
       "      <td>0.928872</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.275862</td>\n",
       "      <td>0.185247</td>\n",
       "      <td>0.925790</td>\n",
       "      <td>0.627907</td>\n",
       "      <td>0.321429</td>\n",
       "      <td>0.355556</td>\n",
       "      <td>0.425197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>15000</td>\n",
       "      <td>32</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>146.063606</td>\n",
       "      <td>0.177813</td>\n",
       "      <td>0.928274</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.068966</td>\n",
       "      <td>0.185629</td>\n",
       "      <td>0.925635</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>0.059524</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.103093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>15000</td>\n",
       "      <td>64</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>258.754622</td>\n",
       "      <td>0.175295</td>\n",
       "      <td>0.930422</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.287356</td>\n",
       "      <td>0.185879</td>\n",
       "      <td>0.927277</td>\n",
       "      <td>0.553191</td>\n",
       "      <td>0.309524</td>\n",
       "      <td>0.364964</td>\n",
       "      <td>0.396947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>7500</td>\n",
       "      <td>96</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>209.534456</td>\n",
       "      <td>0.197488</td>\n",
       "      <td>0.920620</td>\n",
       "      <td>0.592593</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.190144</td>\n",
       "      <td>0.924078</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.261905</td>\n",
       "      <td>0.426667</td>\n",
       "      <td>0.343750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>7500</td>\n",
       "      <td>64</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>147.158974</td>\n",
       "      <td>0.197551</td>\n",
       "      <td>0.920332</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.270833</td>\n",
       "      <td>0.191587</td>\n",
       "      <td>0.922382</td>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.214286</td>\n",
       "      <td>0.382353</td>\n",
       "      <td>0.327273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>3750</td>\n",
       "      <td>64</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>93.947803</td>\n",
       "      <td>0.200405</td>\n",
       "      <td>0.915173</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.206580</td>\n",
       "      <td>0.914372</td>\n",
       "      <td>0.448276</td>\n",
       "      <td>0.154762</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.230088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>7500</td>\n",
       "      <td>16</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>102.366249</td>\n",
       "      <td>0.216317</td>\n",
       "      <td>0.907416</td>\n",
       "      <td>0.514286</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.207481</td>\n",
       "      <td>0.913837</td>\n",
       "      <td>0.442308</td>\n",
       "      <td>0.273810</td>\n",
       "      <td>0.433735</td>\n",
       "      <td>0.338235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>3750</td>\n",
       "      <td>96</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>130.096667</td>\n",
       "      <td>0.206588</td>\n",
       "      <td>0.913531</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.209444</td>\n",
       "      <td>0.912296</td>\n",
       "      <td>0.518519</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.518519</td>\n",
       "      <td>0.252252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>7500</td>\n",
       "      <td>32</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>91.583729</td>\n",
       "      <td>0.217421</td>\n",
       "      <td>0.908252</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>0.210370</td>\n",
       "      <td>0.911947</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>0.074074</td>\n",
       "      <td>0.086957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1875</td>\n",
       "      <td>64</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>59.219416</td>\n",
       "      <td>0.227473</td>\n",
       "      <td>0.910621</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.226964</td>\n",
       "      <td>0.910080</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1875</td>\n",
       "      <td>32</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>51.017422</td>\n",
       "      <td>0.230207</td>\n",
       "      <td>0.908719</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.228599</td>\n",
       "      <td>0.909275</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1875</td>\n",
       "      <td>96</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>51.186965</td>\n",
       "      <td>0.228113</td>\n",
       "      <td>0.910563</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.230876</td>\n",
       "      <td>0.906819</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>3750</td>\n",
       "      <td>32</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>48.818243</td>\n",
       "      <td>0.240126</td>\n",
       "      <td>0.905578</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.238414</td>\n",
       "      <td>0.907036</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1000</td>\n",
       "      <td>32</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>35.590092</td>\n",
       "      <td>0.253021</td>\n",
       "      <td>0.896541</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.242984</td>\n",
       "      <td>0.901877</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1875</td>\n",
       "      <td>16</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>45.174892</td>\n",
       "      <td>0.243413</td>\n",
       "      <td>0.906472</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.244736</td>\n",
       "      <td>0.904054</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1000</td>\n",
       "      <td>16</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>40.722947</td>\n",
       "      <td>0.260653</td>\n",
       "      <td>0.892108</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.252119</td>\n",
       "      <td>0.898778</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>3750</td>\n",
       "      <td>16</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>34.344338</td>\n",
       "      <td>0.276092</td>\n",
       "      <td>0.889126</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.275049</td>\n",
       "      <td>0.889173</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>1000</td>\n",
       "      <td>96</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>65.081391</td>\n",
       "      <td>0.286579</td>\n",
       "      <td>0.882811</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.280397</td>\n",
       "      <td>0.886377</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1000</td>\n",
       "      <td>64</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>33.905847</td>\n",
       "      <td>0.294455</td>\n",
       "      <td>0.873622</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.289161</td>\n",
       "      <td>0.875144</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>30000</td>\n",
       "      <td>96</td>\n",
       "      <td>softmax</td>\n",
       "      <td>364.382111</td>\n",
       "      <td>0.458740</td>\n",
       "      <td>0.812047</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.459075</td>\n",
       "      <td>0.812531</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>30000</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>328.438214</td>\n",
       "      <td>0.460644</td>\n",
       "      <td>0.812007</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.460406</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>30000</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>331.574224</td>\n",
       "      <td>0.461732</td>\n",
       "      <td>0.812014</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.460706</td>\n",
       "      <td>0.812531</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>30000</td>\n",
       "      <td>16</td>\n",
       "      <td>softmax</td>\n",
       "      <td>266.220382</td>\n",
       "      <td>0.467084</td>\n",
       "      <td>0.812007</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.466879</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>15000</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>253.954471</td>\n",
       "      <td>0.464458</td>\n",
       "      <td>0.811416</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.469589</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>15000</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>210.126895</td>\n",
       "      <td>0.467879</td>\n",
       "      <td>0.811416</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.470793</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15000</td>\n",
       "      <td>16</td>\n",
       "      <td>softmax</td>\n",
       "      <td>170.379706</td>\n",
       "      <td>0.472363</td>\n",
       "      <td>0.811416</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.473275</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>7500</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>118.131372</td>\n",
       "      <td>0.479287</td>\n",
       "      <td>0.812382</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.473779</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7500</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>100.684439</td>\n",
       "      <td>0.484382</td>\n",
       "      <td>0.812382</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.481142</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>15000</td>\n",
       "      <td>96</td>\n",
       "      <td>softmax</td>\n",
       "      <td>166.150431</td>\n",
       "      <td>0.480116</td>\n",
       "      <td>0.811416</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.482733</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7500</td>\n",
       "      <td>16</td>\n",
       "      <td>softmax</td>\n",
       "      <td>73.782132</td>\n",
       "      <td>0.490364</td>\n",
       "      <td>0.812382</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.484168</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>7500</td>\n",
       "      <td>96</td>\n",
       "      <td>softmax</td>\n",
       "      <td>115.529006</td>\n",
       "      <td>0.493493</td>\n",
       "      <td>0.812382</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.487169</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3750</td>\n",
       "      <td>16</td>\n",
       "      <td>softmax</td>\n",
       "      <td>39.245193</td>\n",
       "      <td>0.497875</td>\n",
       "      <td>0.811992</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.496862</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>3750</td>\n",
       "      <td>96</td>\n",
       "      <td>softmax</td>\n",
       "      <td>60.296383</td>\n",
       "      <td>0.501025</td>\n",
       "      <td>0.811992</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.499214</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1875</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>33.879440</td>\n",
       "      <td>0.499665</td>\n",
       "      <td>0.811790</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.501546</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3750</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>41.890195</td>\n",
       "      <td>0.503229</td>\n",
       "      <td>0.811992</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.503404</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1875</td>\n",
       "      <td>16</td>\n",
       "      <td>softmax</td>\n",
       "      <td>34.249058</td>\n",
       "      <td>0.508308</td>\n",
       "      <td>0.811790</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.503761</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>3750</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>29.121247</td>\n",
       "      <td>0.512358</td>\n",
       "      <td>0.811992</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.510451</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1000</td>\n",
       "      <td>64</td>\n",
       "      <td>softmax</td>\n",
       "      <td>22.319825</td>\n",
       "      <td>0.525676</td>\n",
       "      <td>0.810595</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.510826</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1000</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>17.509727</td>\n",
       "      <td>0.526373</td>\n",
       "      <td>0.810595</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.515355</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1875</td>\n",
       "      <td>96</td>\n",
       "      <td>softmax</td>\n",
       "      <td>25.693755</td>\n",
       "      <td>0.529101</td>\n",
       "      <td>0.811790</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.527550</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1875</td>\n",
       "      <td>32</td>\n",
       "      <td>softmax</td>\n",
       "      <td>15.707598</td>\n",
       "      <td>0.529115</td>\n",
       "      <td>0.811790</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.528120</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1000</td>\n",
       "      <td>96</td>\n",
       "      <td>softmax</td>\n",
       "      <td>25.392356</td>\n",
       "      <td>0.541163</td>\n",
       "      <td>0.810595</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.535186</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000</td>\n",
       "      <td>16</td>\n",
       "      <td>softmax</td>\n",
       "      <td>12.280781</td>\n",
       "      <td>0.547511</td>\n",
       "      <td>0.810595</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.538136</td>\n",
       "      <td>0.812523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    n_samples  n_hidden activation  training_time  val_loss  val_accuracy  \\\n",
       "47      30000        96    sigmoid     501.888605  0.160303      0.935950   \n",
       "41      30000        64    sigmoid     442.382329  0.165130      0.936807   \n",
       "35      30000        32    sigmoid     383.550871  0.166100      0.934094   \n",
       "29      30000        16    sigmoid     289.291727  0.171320      0.929452   \n",
       "46      15000        96    sigmoid     273.815420  0.175283      0.931596   \n",
       "28      15000        16    sigmoid     190.766743  0.177503      0.928872   \n",
       "34      15000        32    sigmoid     146.063606  0.177813      0.928274   \n",
       "40      15000        64    sigmoid     258.754622  0.175295      0.930422   \n",
       "45       7500        96    sigmoid     209.534456  0.197488      0.920620   \n",
       "39       7500        64    sigmoid     147.158974  0.197551      0.920332   \n",
       "38       3750        64    sigmoid      93.947803  0.200405      0.915173   \n",
       "27       7500        16    sigmoid     102.366249  0.216317      0.907416   \n",
       "44       3750        96    sigmoid     130.096667  0.206588      0.913531   \n",
       "33       7500        32    sigmoid      91.583729  0.217421      0.908252   \n",
       "37       1875        64    sigmoid      59.219416  0.227473      0.910621   \n",
       "31       1875        32    sigmoid      51.017422  0.230207      0.908719   \n",
       "43       1875        96    sigmoid      51.186965  0.228113      0.910563   \n",
       "32       3750        32    sigmoid      48.818243  0.240126      0.905578   \n",
       "30       1000        32    sigmoid      35.590092  0.253021      0.896541   \n",
       "25       1875        16    sigmoid      45.174892  0.243413      0.906472   \n",
       "24       1000        16    sigmoid      40.722947  0.260653      0.892108   \n",
       "26       3750        16    sigmoid      34.344338  0.276092      0.889126   \n",
       "42       1000        96    sigmoid      65.081391  0.286579      0.882811   \n",
       "36       1000        64    sigmoid      33.905847  0.294455      0.873622   \n",
       "23      30000        96    softmax     364.382111  0.458740      0.812047   \n",
       "11      30000        32    softmax     328.438214  0.460644      0.812007   \n",
       "17      30000        64    softmax     331.574224  0.461732      0.812014   \n",
       "5       30000        16    softmax     266.220382  0.467084      0.812007   \n",
       "16      15000        64    softmax     253.954471  0.464458      0.811416   \n",
       "10      15000        32    softmax     210.126895  0.467879      0.811416   \n",
       "4       15000        16    softmax     170.379706  0.472363      0.811416   \n",
       "15       7500        64    softmax     118.131372  0.479287      0.812382   \n",
       "9        7500        32    softmax     100.684439  0.484382      0.812382   \n",
       "22      15000        96    softmax     166.150431  0.480116      0.811416   \n",
       "3        7500        16    softmax      73.782132  0.490364      0.812382   \n",
       "21       7500        96    softmax     115.529006  0.493493      0.812382   \n",
       "2        3750        16    softmax      39.245193  0.497875      0.811992   \n",
       "20       3750        96    softmax      60.296383  0.501025      0.811992   \n",
       "13       1875        64    softmax      33.879440  0.499665      0.811790   \n",
       "8        3750        32    softmax      41.890195  0.503229      0.811992   \n",
       "1        1875        16    softmax      34.249058  0.508308      0.811790   \n",
       "14       3750        64    softmax      29.121247  0.512358      0.811992   \n",
       "12       1000        64    softmax      22.319825  0.525676      0.810595   \n",
       "6        1000        32    softmax      17.509727  0.526373      0.810595   \n",
       "19       1875        96    softmax      25.693755  0.529101      0.811790   \n",
       "7        1875        32    softmax      15.707598  0.529115      0.811790   \n",
       "18       1000        96    softmax      25.392356  0.541163      0.810595   \n",
       "0        1000        16    softmax      12.280781  0.547511      0.810595   \n",
       "\n",
       "    val_precision  val_recall  test_loss  test_accuracy  test_precision  \\\n",
       "47       0.664384    0.625806   0.163766       0.934776        0.652174   \n",
       "41       0.837838    0.600000   0.168144       0.936162        0.787879   \n",
       "35       0.617834    0.625806   0.169503       0.931801        0.605263   \n",
       "29       0.567010    0.354839   0.175454       0.927053        0.509804   \n",
       "46       0.480769    0.287356   0.184181       0.925751        0.441176   \n",
       "28       0.500000    0.275862   0.185247       0.925790        0.627907   \n",
       "34       0.666667    0.068966   0.185629       0.925635        0.384615   \n",
       "40       0.500000    0.287356   0.185879       0.927277        0.553191   \n",
       "45       0.592593    0.333333   0.190144       0.924078        0.500000   \n",
       "39       0.650000    0.270833   0.191587       0.922382        0.692308   \n",
       "38       0.666667    0.117647   0.206580       0.914372        0.448276   \n",
       "27       0.514286    0.375000   0.207481       0.913837        0.442308   \n",
       "44       0.700000    0.411765   0.209444       0.912296        0.518519   \n",
       "33       0.333333    0.041667   0.210370       0.911947        0.500000   \n",
       "37       0.000000    0.000000   0.226964       0.910080        0.000000   \n",
       "31       0.000000    0.000000   0.228599       0.909275        0.000000   \n",
       "43       0.000000    0.000000   0.230876       0.906819        0.000000   \n",
       "32       0.000000    0.000000   0.238414       0.907036        0.000000   \n",
       "30       0.000000    0.000000   0.242984       0.901877        0.000000   \n",
       "25       0.000000    0.000000   0.244736       0.904054        0.000000   \n",
       "24       0.000000    0.000000   0.252119       0.898778        0.000000   \n",
       "26       0.000000    0.000000   0.275049       0.889173        0.000000   \n",
       "42       0.000000    0.000000   0.280397       0.886377        0.000000   \n",
       "36       0.000000    0.000000   0.289161       0.875144        0.000000   \n",
       "23       0.000000    0.000000   0.459075       0.812531        0.000000   \n",
       "11       0.000000    0.000000   0.460406       0.812523        0.000000   \n",
       "17       0.000000    0.000000   0.460706       0.812531        0.000000   \n",
       "5        0.000000    0.000000   0.466879       0.812523        0.000000   \n",
       "16       0.000000    0.000000   0.469589       0.812523        0.000000   \n",
       "10       0.000000    0.000000   0.470793       0.812523        0.000000   \n",
       "4        0.000000    0.000000   0.473275       0.812523        0.000000   \n",
       "15       0.000000    0.000000   0.473779       0.812523        0.000000   \n",
       "9        0.000000    0.000000   0.481142       0.812523        0.000000   \n",
       "22       0.000000    0.000000   0.482733       0.812523        0.000000   \n",
       "3        0.000000    0.000000   0.484168       0.812523        0.000000   \n",
       "21       0.000000    0.000000   0.487169       0.812523        0.000000   \n",
       "2        0.000000    0.000000   0.496862       0.812523        0.000000   \n",
       "20       0.000000    0.000000   0.499214       0.812523        0.000000   \n",
       "13       0.000000    0.000000   0.501546       0.812523        0.000000   \n",
       "8        0.000000    0.000000   0.503404       0.812523        0.000000   \n",
       "1        0.000000    0.000000   0.503761       0.812523        0.000000   \n",
       "14       0.000000    0.000000   0.510451       0.812523        0.000000   \n",
       "12       0.000000    0.000000   0.510826       0.812523        0.000000   \n",
       "6        0.000000    0.000000   0.515355       0.812523        0.000000   \n",
       "19       0.000000    0.000000   0.527550       0.812523        0.000000   \n",
       "7        0.000000    0.000000   0.528120       0.812523        0.000000   \n",
       "18       0.000000    0.000000   0.535186       0.812523        0.000000   \n",
       "0        0.000000    0.000000   0.538136       0.812523        0.000000   \n",
       "\n",
       "    test_recall    val_f1   test_f1  \n",
       "47     0.535714  0.644518  0.588235  \n",
       "41     0.619048  0.699248  0.693333  \n",
       "35     0.547619  0.621795  0.575000  \n",
       "29     0.309524  0.436508  0.385185  \n",
       "46     0.178571  0.359712  0.254237  \n",
       "28     0.321429  0.355556  0.425197  \n",
       "34     0.059524  0.125000  0.103093  \n",
       "40     0.309524  0.364964  0.396947  \n",
       "45     0.261905  0.426667  0.343750  \n",
       "39     0.214286  0.382353  0.327273  \n",
       "38     0.154762  0.200000  0.230088  \n",
       "27     0.273810  0.433735  0.338235  \n",
       "44     0.166667  0.518519  0.252252  \n",
       "33     0.047619  0.074074  0.086957  \n",
       "37     0.000000       NaN       NaN  \n",
       "31     0.000000       NaN       NaN  \n",
       "43     0.000000       NaN       NaN  \n",
       "32     0.000000       NaN       NaN  \n",
       "30     0.000000       NaN       NaN  \n",
       "25     0.000000       NaN       NaN  \n",
       "24     0.000000       NaN       NaN  \n",
       "26     0.000000       NaN       NaN  \n",
       "42     0.000000       NaN       NaN  \n",
       "36     0.000000       NaN       NaN  \n",
       "23     0.000000       NaN       NaN  \n",
       "11     0.000000       NaN       NaN  \n",
       "17     0.000000       NaN       NaN  \n",
       "5      0.000000       NaN       NaN  \n",
       "16     0.000000       NaN       NaN  \n",
       "10     0.000000       NaN       NaN  \n",
       "4      0.000000       NaN       NaN  \n",
       "15     0.000000       NaN       NaN  \n",
       "9      0.000000       NaN       NaN  \n",
       "22     0.000000       NaN       NaN  \n",
       "3      0.000000       NaN       NaN  \n",
       "21     0.000000       NaN       NaN  \n",
       "2      0.000000       NaN       NaN  \n",
       "20     0.000000       NaN       NaN  \n",
       "13     0.000000       NaN       NaN  \n",
       "8      0.000000       NaN       NaN  \n",
       "1      0.000000       NaN       NaN  \n",
       "14     0.000000       NaN       NaN  \n",
       "12     0.000000       NaN       NaN  \n",
       "6      0.000000       NaN       NaN  \n",
       "19     0.000000       NaN       NaN  \n",
       "7      0.000000       NaN       NaN  \n",
       "18     0.000000       NaN       NaN  \n",
       "0      0.000000       NaN       NaN  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df.sort_values('test_loss', ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}