Image classification (letters) with Keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline \n",
    "\n",
    "# define project paths\n",
    "projectpath = 'projectpath'\n",
    "\n",
    "train_data = pd.read_csv(projectpath + 'train_orig.csv',header=None)\n",
    "test_data = pd.read_csv(projectpath + 'test_orig.csv',header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>776</th>\n",
       "      <th>777</th>\n",
       "      <th>778</th>\n",
       "      <th>779</th>\n",
       "      <th>780</th>\n",
       "      <th>781</th>\n",
       "      <th>782</th>\n",
       "      <th>783</th>\n",
       "      <th>784</th>\n",
       "      <th>785</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>77</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>153</td>\n",
       "      <td>255</td>\n",
       "      <td>251</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>114</td>\n",
       "      <td>114</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>115</td>\n",
       "      <td>114</td>\n",
       "      <td>96</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>90</td>\n",
       "      <td>99</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 786 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0   1    2    3    4    5    6    7    8    9    ...  776  777  778  779  \\\n",
       "0    0   A    0    0    0    0    0    0    0    0  ...    0    0    9   19   \n",
       "1    1   A  255  255  255  255  255  255  255  255  ...    0    0    4    0   \n",
       "2    2   A    0    0    0    0    0    0    0    0  ...  114  114  117  117   \n",
       "3    3   A    0    0    0    0    0    0    0    0  ...    0    0    0    0   \n",
       "\n",
       "   780  781  782  783  784  785  \n",
       "0    4   30   11    8   77   13  \n",
       "1  153  255  251  255  255  255  \n",
       "2  117  117  115  114   96   19  \n",
       "3    3    0   24   90   99   28  \n",
       "\n",
       "[4 rows x 786 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show training data\n",
    "train_data.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>775</th>\n",
       "      <th>776</th>\n",
       "      <th>777</th>\n",
       "      <th>778</th>\n",
       "      <th>779</th>\n",
       "      <th>780</th>\n",
       "      <th>781</th>\n",
       "      <th>782</th>\n",
       "      <th>783</th>\n",
       "      <th>784</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>159</td>\n",
       "      <td>239</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>234</td>\n",
       "      <td>115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>106</td>\n",
       "      <td>182</td>\n",
       "      <td>183</td>\n",
       "      <td>176</td>\n",
       "      <td>171</td>\n",
       "      <td>171</td>\n",
       "      <td>171</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>97</td>\n",
       "      <td>201</td>\n",
       "      <td>240</td>\n",
       "      <td>243</td>\n",
       "      <td>210</td>\n",
       "      <td>102</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>195</td>\n",
       "      <td>235</td>\n",
       "      <td>255</td>\n",
       "      <td>254</td>\n",
       "      <td>255</td>\n",
       "      <td>247</td>\n",
       "      <td>199</td>\n",
       "      <td>197</td>\n",
       "      <td>207</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>96</td>\n",
       "      <td>181</td>\n",
       "      <td>227</td>\n",
       "      <td>248</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>190</td>\n",
       "      <td>248</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0    1    2    3    4    5    6    7    8    9    ...  775  776  777  778  \\\n",
       "0    0    0    0    0    0    0    0    0    2   22  ...    1    0   18  159   \n",
       "1    1    0    7  106  182  183  176  171  171  171  ...    3    0    3   97   \n",
       "2    2    0    0    0    0    0    0    0    0    0  ...  195  235  255  254   \n",
       "3    3    2    0   10   96  181  227  248  255  255  ...    0    0    0    0   \n",
       "\n",
       "   779  780  781  782  783  784  \n",
       "0  239  255  255  255  234  115  \n",
       "1  201  240  243  210  102    1  \n",
       "2  255  247  199  197  207  145  \n",
       "3    2    0   22  190  248  156  \n",
       "\n",
       "[4 rows x 785 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show test data\n",
    "test_data.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take labels into separate dataframe\n",
    "train_target = train_data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete unused columns\n",
    "del train_data[0]\n",
    "del train_data[1]\n",
    "del test_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>...</th>\n",
       "      <th>776</th>\n",
       "      <th>777</th>\n",
       "      <th>778</th>\n",
       "      <th>779</th>\n",
       "      <th>780</th>\n",
       "      <th>781</th>\n",
       "      <th>782</th>\n",
       "      <th>783</th>\n",
       "      <th>784</th>\n",
       "      <th>785</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>77</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>153</td>\n",
       "      <td>255</td>\n",
       "      <td>251</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>114</td>\n",
       "      <td>114</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>115</td>\n",
       "      <td>114</td>\n",
       "      <td>96</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>90</td>\n",
       "      <td>99</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 784 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   2    3    4    5    6    7    8    9    10   11   ...  776  777  778  779  \\\n",
       "0    0    0    0    0    0    0    0    0    0    0  ...    0    0    9   19   \n",
       "1  255  255  255  255  255  255  255  255  255  255  ...    0    0    4    0   \n",
       "2    0    0    0    0    0    0    0    0    0    0  ...  114  114  117  117   \n",
       "3    0    0    0    0    0    0    0    0    0   16  ...    0    0    0    0   \n",
       "\n",
       "   780  781  782  783  784  785  \n",
       "0    4   30   11    8   77   13  \n",
       "1  153  255  251  255  255  255  \n",
       "2  117  117  115  114   96   19  \n",
       "3    3    0   24   90   99   28  \n",
       "\n",
       "[4 rows x 784 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if columns were deleted correctly on training data\n",
    "train_data.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store values in numpy array\n",
    "train_data = train_data.values\n",
    "test_data = test_data.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0, ...,   8,  77,  13],\n",
       "       [255, 255, 255, ..., 255, 255, 255],\n",
       "       [  0,   0,   0, ..., 114,  96,  19],\n",
       "       ...,\n",
       "       [  0,   0,   0, ...,   0,   0,   0],\n",
       "       [  0,   0,   0, ...,   0,   0,   0],\n",
       "       [  0,   0,   0, ...,   0,   0,   0]], dtype=int64)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show training data array\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reshape training and test data for tf/keras\n",
    "X_train = train_data.reshape(-1, 28, 28 , 1).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = test_data.reshape(-1, 28, 28 , 1).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train / 255.0\n",
    "X_test = X_test / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split dataset into train (80%) and test data (20%); random_state=1 makes it reproducible (could be any number); stratify ensures that the proportion stays in test and training data sets\n",
    "data_train, data_test, labels_train, labels_test = train_test_split(X_train, train_labels, test_size=0.2, random_state=1, shuffle=True, stratify=train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# import libraries and classes for and from keras\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "np.random.seed(2)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "\n",
    "from keras.utils.np_utils import to_categorical # convert to one-hot-encoding\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "\n",
    "sns.set(style='white', context='notebook', palette='deep')sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encode labels into numeric values\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "enc = LabelEncoder()\n",
    "train_target = enc.fit_transform(train_target.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train = to_categorical(train_target, num_classes = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQMAAAEBCAYAAAB8GcDAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFXxJREFUeJzt3X9Q1Pedx/HXwgKiNbGmrCSVmmhszMX642obzA+pyUhsYUXvnKvKYSyT5tpT7+KZo8rhOWkjZTwn3jiatJ1Lp3ejvcjY6Kgn/pgkpY3YdrATDOmMdaoYCQZXTVUUlmX3e3/4dS/0+H6+sMuya/N8/OXy5vPdt1/gtd/d7+f7+Xosy7IE4BMvLdkNAEgNhAEASYQBABthAEASYQDARhgAkEQYALARBgAkEQYAbIQBAEmEAQCbd6ifsKurS83NzcrJyVF6evpQPz3wZy8cDisQCGjy5MkaNmxYv8fFFQb79u3TK6+8op6eHj399NMqLS11HdPc3Nyv7wMQnx07dmjGjBn9/v6Yw6C9vV2bN2/W66+/rszMTC1atEgPP/yw7r//fuO4nJwcSVLrB9fVE+aCSSRXmsf8TjliRYz18XfebazvGJdtrH8q/9OOtXdrjUO15I/H+/y6N92jsZ8dEf1b66+Yw6ChoUH5+fkaNWqUJOmpp57SwYMHtWLFCuO4W28NesKWenoIAyRXmsf8OxhxucLfEzGHSW66+U/sjhGZjrXz5hxy/fsZ6NvwmD9AvHDhQq/k8fl8am9vj3VzAJIs5jCIRCLyeDzRx5Zl9XoM4PYScxjk5uYqEAhEHwcCAfl8vkFpCsDQizkMHnnkER07dkyXL19WZ2enDh8+rFmzZg1mbwCGUMwfII4ZM0arVq3S0qVLFQqFtHDhQk2ZMmUwewMGhentq+tbW5fPuH+9yPyJ/bB1W8wbMLjrtX8wf8PlmDfdp7jmGfj9fvn9/sHqBUASMR0ZgCTCAICNMAAgiTAAYCMMAEgiDADYhnw9A2Coma5MDEfCxrGvjy4w1oete9FYj3R8ZKx7skc61n7Z7XxFYyJwZABAEmEAwEYYAJBEGACwEQYAJBEGAGycWsRtLz3NvNaf6fThX9/9JePYub/8Z2Pd6uk21k2nDiXJY1gjsTG9yzh2sHFkAEASYQDARhgAkEQYALARBgAkEQYAbIQBAEnMM8BtwG05c7ebo+bd4byc+X/+8Anj2LThdxrrrvMMXO61aPK70MWYx8aCIwMAkggDADbCAIAkwgCAjTAAIIkwAGAjDABIYp4BUoDbPALTUueS+zyDd0vHOta8M4qNYxM5j0CSItcuOdYuha7Fte2Biut/UlZWpsuXL8vrvbmZ7373u5o6deqgNAZgaMUcBpZlqaWlRW+99VY0DADcvmL+zOD06dOSpPLycs2bN0/bt28ftKYADL2YX9KvXr2qmTNnat26dQqFQlq6dKnuu+8+Pfroo4PZH4AhEnMYTJ8+XdOnT48+Xrhwoerr6wkD4DYV89uExsZGHTt2LPrYsiw+OwBuYzGHwbVr17Rx40YFg0F1dHRo9+7dmjNnzmD2BmAIxfxSPnv2bDU1NWn+/PmKRCJasmRJr7cNQH95Xe57EAr3GOu/GWO+98GwdS851qzuTuNYT2a2sS6XOQ5uIud+51i7FjL3NtjiOq5/7rnn9Nxzzw1WLwCSiOnIACQRBgBshAEASYQBABthAEASlzBjiJhum+526vA793zFWJ/S+D1j3XT60JORZRzrxoqYTy160s2vt1bbacfaleCNmHqKFUcGACQRBgBshAEASYQBABthAEASYQDARhgAkMQ8AwwSt+XMw5GwY+3LOZ83jv3XPUti6ukW18uQ4xHnJczW2fcda0GXZdoHG0cGACQRBgBshAEASYQBABthAEASYQDARhgAkMQ8A/ST223T3dw5bIRj7Y0X/tI4Nn3sg8Z6+IOTxrplWHLce+8041jXeQSG+RP9EX6/PeaxTmtEpKfF9rPiyACAJMIAgI0wACCJMABgIwwASCIMANgIAwCSmGeAfopnvQJJ+v1jdzvWMuYvN451u236tnk/Nda/vT7Xuegyz8DtvgjxCn0Q+70R0h1+JmlK4DyDjo4OFRcXq7W1VZLU0NAgv9+vwsJCbd68OaYnBpBaXMOgqalJixcvVktLiySpq6tLlZWVevnll3XgwAE1Nzervr4+0X0CSDDXMKitrdX69evl8/kkSSdOnNC4ceOUl5cnr9crv9+vgwcPJrxRAInl+pnBhg0bej2+cOGCcnJyoo99Pp/a22OfXw0gNQz4bEIkEul10YplWXFfxAIg+QYcBrm5uQoEAtHHgUAg+hYCwO1rwGEwdepUnTlzRmfPnlU4HNb+/fs1a9asRPQGYAgNeJ5BVlaWampqtHLlSgWDQRUUFGju3LmJ6A1DyOna+Fvc5hHsvOsrxvqd//W9gbYU9ZsvrjfWKy4dM9b//oGNMT+3J838emnFt5yBrrfGPtXHkuXw9dj0u5M333wz+u+ZM2dq7969MT4lgFTEdGQAkggDADbCAIAkwgCAjTAAIIlLmD8x4j11uOSefGPd/4vnB9zTLV0v/qOxXnDpt8b6PSPvMtY9w0cOuKd+c9mvbi5ecF5CfqhxZABAEmEAwEYYAJBEGACwEQYAJBEGAGyEAQBJzDP4s2JaztxtHsHnP/1ZY/1HPzZfpp42/E5jvedXexxrX9h+zjjWTVZahrHuGTE6ru0nUnv38GS3EMWRAQBJhAEAG2EAQBJhAMBGGACQRBgAsBEGACQxz+C24nbnKlM9I838o25c+YCx7p3ypLEe6fjIWP/b5c435z13NeBY648R6VnGuueOz8S1ffPG43s9/cCbOn+CHBkAkEQYALARBgAkEQYAbIQBAEmEAQAbYQBAEvMMbitelzX6Q+Eex9rJBx8yjs381gsx9XTLgZkvGet7LjU61kZmma/pD/aEjPV7s1zum+DNNNaT6QNvrDdQlyJW32NjfYXv97iOjg4VFxertbVVkrR27VoVFhaqpKREJSUlOnLkSIwtAEgF/ToyaGpqUlVVlVpaWqJfa25u1vbt2+Xz+RLVG4Ah1K8jg9raWq1fvz76h9/Z2am2tjZVVlbK7/dry5YtikQiCW0UQGL1Kww2bNigGTNmRB9fvHhR+fn5qq6uVm1trRobG7Vr166ENQkg8WL6rCEvL0/btm2Tz+dTdna2ysrKVF/vfCEKgNQXUxicPHlShw4dij62LEveFLr6CsDAxRQGlmWpurpaV65cUSgU0s6dOzVnzpzB7g3AEIrp5XzSpEl69tlntXjxYvX09KiwsFDFxcWD3dsnTka6+cdhmkcgSVvHPOFYG/uGeR6B1d1prJ+ds9pYX3jpXWPd5FrwRsxjJckj8zoPRpbLB9+u6xXE98H5FY/5fhZDaUBh8Oabb0b/XVpaqtLS0kFvCEByMB0ZgCTCAICNMAAgiTAAYCMMAEjiEuYhlR7HJciS5L/7i8Z6+RvfdqxZPd3GsZbLUueByyOM9ddHFxjrcZz8U9hlifhHH2iNY+vJdd1KnVOLHBkAkEQYALARBgAkEQYAbIQBAEmEAQAbYQBAEvMMBlWay+WuEZfLZceONN86/L9fcb5EWZLSRuUa6yae0fcY6196999i3nZKi/OW6p60+MbfEPMMAKQYwgCAJMIAgI0wACCJMABgIwwASCIMANiYZzBAHsO19aaadPN+EybN37jPWPd+eZ6xbtJz6tfmb3C7V6bLegiu4xPIM+ZeYz3dZ9ivcS+VHp9rlvl280OJIwMAkggDADbCAIAkwgCAjTAAIIkwAGAjDABI6uc8g61bt6qurk6SVFBQoIqKCjU0NOj73/++gsGgvvrVr2rVqlUJbTRVeA33PnC770FT3nRjfdial2Lq6ZZAyTOOtfG//YNxbLxrMcQjKz3DWA+Gzefif//QRGP97oM/cqxZoaBxrCcz21iP1/WIy/wNA6d5Ky7TWRy5Hhk0NDTo7bff1u7du7Vnzx6999572r9/vyorK/Xyyy/rwIEDam5uVn19fWwdAEgJrmGQk5OjNWvWKDMzUxkZGZowYYJaWlo0btw45eXlyev1yu/36+DBg0PRL4AEcQ2DiRMnatq0aZKklpYW1dXVyePxKCcnJ/o9Pp9P7e3tiesSQML1+wPEU6dOqby8XBUVFcrLy+s1D9+yLNd5+QBSW7/C4Pjx41q2bJlWr16tBQsWKDc3V4FAIFoPBALy+XwJaxJA4rmGwfnz57V8+XJt2rRJRUVFkqSpU6fqzJkzOnv2rMLhsPbv369Zs2YlvFkAieN6avHVV19VMBhUTU1N9GuLFi1STU2NVq5cqWAwqIKCAs2dOzehjQ6VeG6b/gOfeSnzB379Qkw9RZ+77j+M9b84cc55rMtpz2QKu1z+7HZaMyM7dZYbH6iwYjwPmACuYVBVVaWqqqo+a3v37h30hgAkBzMQAUgiDADYCAMAkggDADbCAIAkwgCA7RO3VHpGuvm/7HY+/pv3POpYe/pohXGs5bJtq+Oysf5X//KOsX4teMOx5jZ/Ihwxn6uPd7q5aZl4t217ZK5n33v7vqZlJHgp9oFInU4AJBVhAEASYQDARhgAkEQYALARBgAkEQYAbLflPAPTeemMNPN/qdtl2e0nxnzBWP/3PU8795U1wjjWzS8e3WysH7ncZKyb5hK4zSNw43Y7eTemn5lbb27LuGc8+NmYerq5cfP8i0Qb4XFeJt51/oVDPdY5IRwZAJBEGACwEQYAJBEGAGyEAQBJhAEAG2EAQFIS5xl4PB7Feom86Zy32zyCmb5JxvreH5cY6+ljHzQ0Zl7fv3v7RmO9+MqvzM/tck48kbdNj5dprkBE5r7dzpunfWlmTD3d3Hh8r4eWyz0fPOnm7X/ak+W8bbe5HYN8R0OODABIIgwA2AgDAJIIAwA2wgCAJMIAgI0wACCpn/MMtm7dqrq6OklSQUGBKioqtHbtWh0/flzZ2dmSpBUrVmjOnDn9fmLLsmK+Rv5TmdmOtec/k28c+/yPC4x175QnjfXIjSuONY/LPRnWbLlqrLvds+F2Fs96Ck+MmWysp90/w1i3ujtjfm7LZd6K3O6F4fI7sdLqdqztG36HceylG33/PnkisU1AcA2DhoYGvf3229q9e7c8Ho+eeeYZHTlyRM3Nzdq+fbt8Pl9MTwwgtbi+TcjJydGaNWuUmZmpjIwMTZgwQW1tbWpra1NlZaX8fr+2bNmiiMtMLACpzTUMJk6cqGnTpkmSWlpaVFdXp8cff1z5+fmqrq5WbW2tGhsbtWvXroQ3CyBx+v0B4qlTp1ReXq6KigqNHz9e27Ztk8/nU3Z2tsrKylRfX5/IPgEkWL/C4Pjx41q2bJlWr16tBQsW6OTJkzp06FC0blmWvN7bcm1VADbXMDh//ryWL1+uTZs2qaioSNLNP/7q6mpduXJFoVBIO3fuHNCZBACpx2O5nN978cUX9bOf/Uyf+9znol9btGiRIpGIduzYoZ6eHhUWFur555/v1xO2trbqySef1MPBScq2Mvv8ns3/dJdxG2mzih1rxkuMkyz84R+M9cjvG431vSveM9ZLL77lWHNbbjzey5+HZw4z1k9MmOBYG/Odh41jMwq/EVNPt7ue07811tv/7od9fv3DSI+eDp7XG2+8obFjx/b7+VyP7auqqlRVVdVnrbS0tN9PBCC1MQMRgCTCAICNMAAgiTAAYCMMAEgiDADYkjZt8FufuqYxnr6fPu2JbxrHhv/np461rhNnjWM7z5kvm+664nyLbDcRy3zpaO7fmK/wzPzWC8Z6mtYNuKdb0tNcct9lmoHbPISvjDbP78j9gfNp6MhbdcaxV58pN9a7L5v3ezjo/H8PdbktP++yTLvH/PvkNt5X6Hw5/rA1L5mfO73v53bryXF7MY0C8GeHMAAgiTAAYCMMAEgiDADYCAMAkpJwajEcvrlK7kXLsKrshUvGbUSudTnWQkHzarVdYfNpl6DLqSATy2VspCNorGecbzfWP/KYx3u9zs+fnmbuzeNSdztFFvKYVxH+4OIfHWuWy34Jdpt/pt1hl94jhlOLLit0u/1MPYpvfM8N59WRs1x+Hy5E+t4vAfvrt/7W+st1PYPB1tjYyKXPwBDYsWOHZswwLyP/cUMeBl1dXWpublZOTo7S080TPgAMXDgcViAQ0OTJkzVsmHnRmY8b8jAAkJr4ABGAJMIAgI0wACCJMABgIwwASCIMANgIAwCSkhwG+/bt09e+9jUVFhZqx44dyWzl/ykrK1NRUZFKSkpUUlKipqamZLekjo4OFRcXq7W1VZLU0NAgv9+vwsJCbd68OWX6Wrt2rQoLC6P77siRI0npa+vWrSoqKlJRUZE2btwoKXX2WV+9JX2/WUny4YcfWrNnz7Y++ugj6/r165bf77dOnTqVrHZ6iUQi1mOPPWaFQqFktxL1zjvvWMXFxdZDDz1knTt3zurs7LQKCgqs999/3wqFQlZ5ebn185//POl9WZZlFRcXW+3t7UPey8cdPXrU+vrXv24Fg0Gru7vbWrp0qbVv376U2Gd99Xb48OGk77ekHRk0NDQoPz9fo0aN0vDhw/XUU0/p4MGDyWqnl9OnT0uSysvLNW/ePG3fvj3JHUm1tbVav369fL6b6yieOHFC48aNU15enrxer/x+f1L235/21dnZqba2NlVWVsrv92vLli2KROK7j2MscnJytGbNGmVmZiojI0MTJkxQS0tLSuyzvnpra2tL+n5LWhhcuHBBOTk50cc+n0/t7eartIbK1atXNXPmTG3btk0/+clP9Nprr+no0aNJ7WnDhg29LjpJlf33p31dvHhR+fn5qq6uVm1trRobG7Vr164h72vixImaNm2aJKmlpUV1dXXyeDwpsc/66u3xxx9P+n5LWhhEIhF5PP93eadlWb0eJ9P06dO1ceNGjRw5UqNHj9bChQtVX1+f7LZ6SdX9l5eXp23btsnn8yk7O1tlZWVJ3XenTp1SeXm5KioqlJeXl1L77OO9jR8/Pun7LWlhkJubq0AgEH0cCASih5rJ1tjYqGPHjkUfW5Ylrzdpq8r3KVX338mTJ3Xo0KHo42Tuu+PHj2vZsmVavXq1FixYkFL77E97S4X9lrQweOSRR3Ts2DFdvnxZnZ2dOnz4sGbNmpWsdnq5du2aNm7cqGAwqI6ODu3evVtz5sxJdlu9TJ06VWfOnNHZs2cVDoe1f//+lNh/lmWpurpaV65cUSgU0s6dO5Oy786fP6/ly5dr06ZNKioqkpQ6+6yv3lJhvyXt5W7MmDFatWqVli5dqlAopIULF2rKlCnJaqeX2bNnq6mpSfPnz1ckEtGSJUs0ffr0ZLfVS1ZWlmpqarRy5UoFg0EVFBRo7ty5yW5LkyZN0rPPPqvFixerp6dHhYWFKi4uHvI+Xn31VQWDQdXU1ES/tmjRopTYZ069JXu/sZ4BAEnMQARgIwwASCIMANgIAwCSCAMANsIAgCTCAICNMAAgSfpfiMspsX5fCAEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# show an image from training data\n",
    "g = plt.imshow(X_train[25][:,:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check labels\n",
    "np.unique(train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import BatchNormalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From \\AppData\\Local\\conda\\conda\\envs\\v_conda\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From \\AppData\\Local\\conda\\conda\\envs\\v_conda\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "from keras.optimizers import RMSprop, Adam\n",
    "from keras.layers import DepthwiseConv2D, Reshape, Activation\n",
    "\n",
    "nets = 20\n",
    "model = [0] *nets\n",
    "\n",
    "\n",
    "for j in range(nets):\n",
    "    model[j] = Sequential()\n",
    "\n",
    "    model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = (28, 28, 1)))\n",
    "    model[j].add(BatchNormalization())\n",
    "    model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))\n",
    "    model[j].add(BatchNormalization())\n",
    "    model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))\n",
    "    model[j].add(BatchNormalization())\n",
    "    model[j].add(Dropout(0.35))\n",
    "\n",
    "    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n",
    "    model[j].add(BatchNormalization())\n",
    "    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n",
    "    model[j].add(BatchNormalization())\n",
    "    model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n",
    "    model[j].add(BatchNormalization())\n",
    "    model[j].add(Dropout(0.4))\n",
    "\n",
    "    model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))\n",
    "    model[j].add(BatchNormalization())\n",
    "    model[j].add(Flatten())\n",
    "    model[j].add(Dropout(0.4))\n",
    "    model[j].add(Dense(10, activation='softmax'))\n",
    "\n",
    "    # COMPILE WITH ADAM OPTIMIZER AND CROSS ENTROPY COST\n",
    "    # before: adagrad-optimizer\n",
    "    model[j].compile(optimizer='adam', loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "    \n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LR Reduction Callback\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n",
    "                                            patience=3, \n",
    "                                            verbose=0, \n",
    "                                            factor=0.5, \n",
    "                                            min_lr=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net : 1\n",
      "WARNING:tensorflow:From \\AppData\\Local\\conda\\conda\\envs\\v_conda\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 46s 351ms/step - loss: 1.0069 - acc: 0.6908 - val_loss: 0.3777 - val_acc: 0.8910\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.5710 - acc: 0.8276 - val_loss: 0.3035 - val_acc: 0.9071\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.4982 - acc: 0.8521 - val_loss: 0.2781 - val_acc: 0.9220\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.4417 - acc: 0.8652 - val_loss: 0.2458 - val_acc: 0.9295\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 40s 302ms/step - loss: 0.3998 - acc: 0.8825 - val_loss: 0.2387 - val_acc: 0.9306\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3629 - acc: 0.8924 - val_loss: 0.2119 - val_acc: 0.9306\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3473 - acc: 0.8938 - val_loss: 0.2207 - val_acc: 0.9380\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3313 - acc: 0.8996 - val_loss: 0.1956 - val_acc: 0.9444\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 309ms/step - loss: 0.3230 - acc: 0.9037 - val_loss: 0.2130 - val_acc: 0.9359\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.3227 - acc: 0.9004 - val_loss: 0.1997 - val_acc: 0.9455\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 40s 307ms/step - loss: 0.2992 - acc: 0.9088 - val_loss: 0.1947 - val_acc: 0.9466\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2967 - acc: 0.9073 - val_loss: 0.1863 - val_acc: 0.9402\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2772 - acc: 0.9156 - val_loss: 0.2942 - val_acc: 0.8942\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2767 - acc: 0.9165 - val_loss: 0.1855 - val_acc: 0.9434\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2607 - acc: 0.9169 - val_loss: 0.1737 - val_acc: 0.9455\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 41s 309ms/step - loss: 0.2625 - acc: 0.9204 - val_loss: 0.1735 - val_acc: 0.9434\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.2443 - acc: 0.9244 - val_loss: 0.1765 - val_acc: 0.9476\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2436 - acc: 0.9250 - val_loss: 0.1727 - val_acc: 0.9487\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2493 - acc: 0.9240 - val_loss: 0.1809 - val_acc: 0.9455\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2403 - acc: 0.9276 - val_loss: 0.1738 - val_acc: 0.9444\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2558 - acc: 0.9225 - val_loss: 0.1763 - val_acc: 0.9455\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2267 - acc: 0.9308 - val_loss: 0.1689 - val_acc: 0.9444\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2380 - acc: 0.9255 - val_loss: 0.1757 - val_acc: 0.9402\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 43s 324ms/step - loss: 0.2229 - acc: 0.9311 - val_loss: 0.1675 - val_acc: 0.9487\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 43s 331ms/step - loss: 0.2333 - acc: 0.9287 - val_loss: 0.1692 - val_acc: 0.9476\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2258 - acc: 0.9300 - val_loss: 0.1687 - val_acc: 0.9476\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2206 - acc: 0.9344 - val_loss: 0.1724 - val_acc: 0.9487\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2260 - acc: 0.9296 - val_loss: 0.1687 - val_acc: 0.9487\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2253 - acc: 0.9298 - val_loss: 0.1666 - val_acc: 0.9487\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2117 - acc: 0.9330 - val_loss: 0.1673 - val_acc: 0.9509\n",
      "CNN 1: Epochs=30, Train accuracy=0.93469, Validation accuracy=0.95085\n",
      "Net : 2\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 46s 348ms/step - loss: 1.0091 - acc: 0.7003 - val_loss: 0.3716 - val_acc: 0.8942\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.5674 - acc: 0.8297 - val_loss: 0.3148 - val_acc: 0.9113\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.4844 - acc: 0.8531 - val_loss: 0.2786 - val_acc: 0.9263\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.4349 - acc: 0.8729 - val_loss: 0.2452 - val_acc: 0.9359\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 43s 328ms/step - loss: 0.3842 - acc: 0.8828 - val_loss: 0.2385 - val_acc: 0.9274\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.3791 - acc: 0.8868 - val_loss: 0.2389 - val_acc: 0.9327\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.3489 - acc: 0.8926 - val_loss: 0.2215 - val_acc: 0.9391\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.3367 - acc: 0.8976 - val_loss: 0.2276 - val_acc: 0.9370\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.3267 - acc: 0.8993 - val_loss: 0.4843 - val_acc: 0.8002\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.3139 - acc: 0.9039 - val_loss: 0.2069 - val_acc: 0.9423\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2806 - acc: 0.9159 - val_loss: 0.2034 - val_acc: 0.9434\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2891 - acc: 0.9137 - val_loss: 0.2015 - val_acc: 0.9455\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2797 - acc: 0.9147 - val_loss: 0.1906 - val_acc: 0.9444\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2700 - acc: 0.9163 - val_loss: 0.1920 - val_acc: 0.9444\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2654 - acc: 0.9182 - val_loss: 0.1808 - val_acc: 0.9498\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2615 - acc: 0.9166 - val_loss: 0.1812 - val_acc: 0.9466\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2719 - acc: 0.9158 - val_loss: 0.1943 - val_acc: 0.9487\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2416 - acc: 0.9226 - val_loss: 0.1919 - val_acc: 0.9455\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2460 - acc: 0.9246 - val_loss: 0.1763 - val_acc: 0.9498\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2284 - acc: 0.9289 - val_loss: 0.1797 - val_acc: 0.9487\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2289 - acc: 0.9300 - val_loss: 0.1755 - val_acc: 0.9530\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2348 - acc: 0.9258 - val_loss: 0.1721 - val_acc: 0.9551\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2216 - acc: 0.9304 - val_loss: 0.1682 - val_acc: 0.9551\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2146 - acc: 0.9328 - val_loss: 0.1757 - val_acc: 0.9541\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2204 - acc: 0.9323 - val_loss: 0.1688 - val_acc: 0.9530\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 40s 308ms/step - loss: 0.2104 - acc: 0.9361 - val_loss: 0.1749 - val_acc: 0.9530\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2137 - acc: 0.9312 - val_loss: 0.1700 - val_acc: 0.9551\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2127 - acc: 0.9335 - val_loss: 0.1752 - val_acc: 0.9551\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2022 - acc: 0.9368 - val_loss: 0.1690 - val_acc: 0.9551\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2063 - acc: 0.9371 - val_loss: 0.1708 - val_acc: 0.9573\n",
      "CNN 2: Epochs=30, Train accuracy=0.93696, Validation accuracy=0.95726\n",
      "Net : 3\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 44s 337ms/step - loss: 0.9963 - acc: 0.7035 - val_loss: 0.6173 - val_acc: 0.7874\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.5605 - acc: 0.8303 - val_loss: 0.3313 - val_acc: 0.9038\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.4786 - acc: 0.8599 - val_loss: 0.3044 - val_acc: 0.9049\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.4228 - acc: 0.8747 - val_loss: 0.2572 - val_acc: 0.9177\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.3938 - acc: 0.8779 - val_loss: 0.2529 - val_acc: 0.9231\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.3566 - acc: 0.8909 - val_loss: 0.2446 - val_acc: 0.9274\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.3550 - acc: 0.8906 - val_loss: 0.2327 - val_acc: 0.9327\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.3438 - acc: 0.8906 - val_loss: 0.2305 - val_acc: 0.9359\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3107 - acc: 0.9051 - val_loss: 0.2344 - val_acc: 0.9359\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3070 - acc: 0.9082 - val_loss: 0.2110 - val_acc: 0.9391\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2918 - acc: 0.9125 - val_loss: 0.2255 - val_acc: 0.9359\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 44s 339ms/step - loss: 0.2922 - acc: 0.9109 - val_loss: 0.2024 - val_acc: 0.9391\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2791 - acc: 0.9151 - val_loss: 0.2058 - val_acc: 0.9391\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2666 - acc: 0.9144 - val_loss: 0.1985 - val_acc: 0.9412\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2533 - acc: 0.9227 - val_loss: 0.1979 - val_acc: 0.9412\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2594 - acc: 0.9209 - val_loss: 0.2003 - val_acc: 0.9455\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2417 - acc: 0.9255 - val_loss: 0.1963 - val_acc: 0.9466\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2425 - acc: 0.9250 - val_loss: 0.1944 - val_acc: 0.9444\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2451 - acc: 0.9243 - val_loss: 0.2007 - val_acc: 0.9423\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2349 - acc: 0.9286 - val_loss: 0.1912 - val_acc: 0.9423\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2336 - acc: 0.9277 - val_loss: 0.1911 - val_acc: 0.9434\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 40s 306ms/step - loss: 0.2320 - acc: 0.9268 - val_loss: 0.1948 - val_acc: 0.9455\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2296 - acc: 0.9303 - val_loss: 0.1962 - val_acc: 0.9434\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2262 - acc: 0.9290 - val_loss: 0.1924 - val_acc: 0.9444\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2261 - acc: 0.9316 - val_loss: 0.1923 - val_acc: 0.9444\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 40s 307ms/step - loss: 0.2258 - acc: 0.9321 - val_loss: 0.1926 - val_acc: 0.9444\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2216 - acc: 0.9321 - val_loss: 0.1909 - val_acc: 0.9455\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2305 - acc: 0.9290 - val_loss: 0.1915 - val_acc: 0.9434\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2257 - acc: 0.9319 - val_loss: 0.1909 - val_acc: 0.9455\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2292 - acc: 0.9289 - val_loss: 0.1902 - val_acc: 0.9455\n",
      "CNN 3: Epochs=30, Train accuracy=0.93230, Validation accuracy=0.94658\n",
      "Net : 4\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 45s 343ms/step - loss: 1.0175 - acc: 0.6875 - val_loss: 0.4328 - val_acc: 0.8697\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 43s 329ms/step - loss: 0.5602 - acc: 0.8315 - val_loss: 0.3717 - val_acc: 0.8932\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.4740 - acc: 0.8579 - val_loss: 0.3327 - val_acc: 0.8835\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.4155 - acc: 0.8772 - val_loss: 0.2788 - val_acc: 0.9188\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.3855 - acc: 0.8846 - val_loss: 0.2530 - val_acc: 0.9145\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.3605 - acc: 0.8942 - val_loss: 0.2517 - val_acc: 0.9252\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.3550 - acc: 0.8958 - val_loss: 0.2371 - val_acc: 0.9274\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.3449 - acc: 0.8971 - val_loss: 0.2204 - val_acc: 0.9359\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.3102 - acc: 0.9066 - val_loss: 0.2109 - val_acc: 0.9359\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.3127 - acc: 0.9060 - val_loss: 0.2120 - val_acc: 0.9327\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 43s 324ms/step - loss: 0.2994 - acc: 0.9123 - val_loss: 0.2062 - val_acc: 0.9338\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2797 - acc: 0.9151 - val_loss: 0.1893 - val_acc: 0.9412\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2771 - acc: 0.9153 - val_loss: 0.1905 - val_acc: 0.9412\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2883 - acc: 0.9108 - val_loss: 0.1924 - val_acc: 0.9423\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2710 - acc: 0.9168 - val_loss: 0.1937 - val_acc: 0.9412\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2643 - acc: 0.9190 - val_loss: 0.1923 - val_acc: 0.9391\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2548 - acc: 0.9203 - val_loss: 0.1915 - val_acc: 0.9391\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2489 - acc: 0.9242 - val_loss: 0.1896 - val_acc: 0.9402\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2564 - acc: 0.9211 - val_loss: 0.1834 - val_acc: 0.9391\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2551 - acc: 0.9234 - val_loss: 0.1825 - val_acc: 0.9434\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2490 - acc: 0.9262 - val_loss: 0.1876 - val_acc: 0.9412\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2547 - acc: 0.9218 - val_loss: 0.1820 - val_acc: 0.9412\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2411 - acc: 0.9262 - val_loss: 0.1815 - val_acc: 0.9444\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2468 - acc: 0.9254 - val_loss: 0.1782 - val_acc: 0.9423\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2344 - acc: 0.9275 - val_loss: 0.1878 - val_acc: 0.9412\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2375 - acc: 0.9276 - val_loss: 0.1862 - val_acc: 0.9412\n",
      "Epoch 27/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131/131 [==============================] - 40s 306ms/step - loss: 0.2356 - acc: 0.9296 - val_loss: 0.1775 - val_acc: 0.9434\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2322 - acc: 0.9278 - val_loss: 0.1772 - val_acc: 0.9423\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2258 - acc: 0.9316 - val_loss: 0.1832 - val_acc: 0.9391\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2295 - acc: 0.9302 - val_loss: 0.1793 - val_acc: 0.9391\n",
      "CNN 4: Epochs=30, Train accuracy=0.93158, Validation accuracy=0.94444\n",
      "Net : 5\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 44s 336ms/step - loss: 0.9994 - acc: 0.7012 - val_loss: 0.4350 - val_acc: 0.8729\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.5616 - acc: 0.8323 - val_loss: 0.3260 - val_acc: 0.9103\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 40s 308ms/step - loss: 0.4800 - acc: 0.8586 - val_loss: 0.3250 - val_acc: 0.9071\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.4101 - acc: 0.8742 - val_loss: 0.2973 - val_acc: 0.9113\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.3869 - acc: 0.8847 - val_loss: 0.2737 - val_acc: 0.9241\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.3623 - acc: 0.8905 - val_loss: 0.3320 - val_acc: 0.8878\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.3400 - acc: 0.8997 - val_loss: 0.2446 - val_acc: 0.9316\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.3186 - acc: 0.9056 - val_loss: 0.2368 - val_acc: 0.9359\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3161 - acc: 0.9050 - val_loss: 0.2457 - val_acc: 0.9338\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3154 - acc: 0.9052 - val_loss: 0.2307 - val_acc: 0.9370\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2977 - acc: 0.9094 - val_loss: 0.2341 - val_acc: 0.9359\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 40s 305ms/step - loss: 0.2811 - acc: 0.9154 - val_loss: 0.2283 - val_acc: 0.9327\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2788 - acc: 0.9146 - val_loss: 0.2334 - val_acc: 0.9359\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2647 - acc: 0.9201 - val_loss: 0.3404 - val_acc: 0.8835\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2563 - acc: 0.9238 - val_loss: 0.2357 - val_acc: 0.9284\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2355 - acc: 0.9301 - val_loss: 0.2148 - val_acc: 0.9380\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 40s 304ms/step - loss: 0.2626 - acc: 0.9202 - val_loss: 0.2093 - val_acc: 0.9370\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2424 - acc: 0.9264 - val_loss: 0.2132 - val_acc: 0.9402\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2394 - acc: 0.9276 - val_loss: 0.2044 - val_acc: 0.9412\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2327 - acc: 0.9302 - val_loss: 0.2121 - val_acc: 0.9370\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.2391 - acc: 0.9254 - val_loss: 0.2122 - val_acc: 0.9370\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2348 - acc: 0.9293 - val_loss: 0.2051 - val_acc: 0.9412\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2210 - acc: 0.9332 - val_loss: 0.2065 - val_acc: 0.9391\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2286 - acc: 0.9320 - val_loss: 0.2056 - val_acc: 0.9370\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2218 - acc: 0.9299 - val_loss: 0.2048 - val_acc: 0.9380\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 40s 307ms/step - loss: 0.2301 - acc: 0.9314 - val_loss: 0.2055 - val_acc: 0.9402\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2218 - acc: 0.9331 - val_loss: 0.2079 - val_acc: 0.9370\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2167 - acc: 0.9352 - val_loss: 0.2040 - val_acc: 0.9412\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2140 - acc: 0.9355 - val_loss: 0.2029 - val_acc: 0.9412\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2279 - acc: 0.9309 - val_loss: 0.2036 - val_acc: 0.9402\n",
      "CNN 5: Epochs=30, Train accuracy=0.93553, Validation accuracy=0.94124\n",
      "Net : 6\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 44s 336ms/step - loss: 0.9752 - acc: 0.7060 - val_loss: 0.5963 - val_acc: 0.8120\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.5780 - acc: 0.8293 - val_loss: 0.3779 - val_acc: 0.8932\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.4620 - acc: 0.8587 - val_loss: 0.3532 - val_acc: 0.8932\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.4284 - acc: 0.8719 - val_loss: 0.3067 - val_acc: 0.9081\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3908 - acc: 0.8820 - val_loss: 0.3594 - val_acc: 0.8814\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.3618 - acc: 0.8910 - val_loss: 0.2885 - val_acc: 0.9156\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3344 - acc: 0.8979 - val_loss: 0.2776 - val_acc: 0.9177\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 40s 303ms/step - loss: 0.3330 - acc: 0.9012 - val_loss: 0.2912 - val_acc: 0.9177\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3138 - acc: 0.9071 - val_loss: 0.2541 - val_acc: 0.9241\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3036 - acc: 0.9101 - val_loss: 0.2612 - val_acc: 0.9188\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2877 - acc: 0.9124 - val_loss: 0.2421 - val_acc: 0.9252\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2724 - acc: 0.9173 - val_loss: 0.2530 - val_acc: 0.9209\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.2809 - acc: 0.9155 - val_loss: 0.2590 - val_acc: 0.9209\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2707 - acc: 0.9187 - val_loss: 0.2408 - val_acc: 0.9252\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2581 - acc: 0.9217 - val_loss: 0.2276 - val_acc: 0.9316\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2501 - acc: 0.9235 - val_loss: 0.2356 - val_acc: 0.9231\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2402 - acc: 0.9287 - val_loss: 0.2304 - val_acc: 0.9263\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2421 - acc: 0.9276 - val_loss: 0.2313 - val_acc: 0.9284\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2343 - acc: 0.9264 - val_loss: 0.2339 - val_acc: 0.9284\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2301 - acc: 0.9314 - val_loss: 0.2227 - val_acc: 0.9284\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2317 - acc: 0.9306 - val_loss: 0.2162 - val_acc: 0.9306\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 40s 306ms/step - loss: 0.2263 - acc: 0.9322 - val_loss: 0.2173 - val_acc: 0.9263\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 40s 307ms/step - loss: 0.2311 - acc: 0.9302 - val_loss: 0.2167 - val_acc: 0.9284\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2180 - acc: 0.9342 - val_loss: 0.2143 - val_acc: 0.9306\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2191 - acc: 0.9295 - val_loss: 0.2172 - val_acc: 0.9306\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2263 - acc: 0.9313 - val_loss: 0.2193 - val_acc: 0.9316\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2179 - acc: 0.9329 - val_loss: 0.2178 - val_acc: 0.9306\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2147 - acc: 0.9352 - val_loss: 0.2182 - val_acc: 0.9295\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2120 - acc: 0.9333 - val_loss: 0.2189 - val_acc: 0.9295\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2168 - acc: 0.9311 - val_loss: 0.2170 - val_acc: 0.9295\n",
      "CNN 6: Epochs=30, Train accuracy=0.93517, Validation accuracy=0.93162\n",
      "Net : 7\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 44s 337ms/step - loss: 1.0217 - acc: 0.6955 - val_loss: 0.4072 - val_acc: 0.8878\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 40s 306ms/step - loss: 0.5755 - acc: 0.8262 - val_loss: 0.3224 - val_acc: 0.9049\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.4897 - acc: 0.8538 - val_loss: 0.2999 - val_acc: 0.9145\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.4253 - acc: 0.8743 - val_loss: 0.2778 - val_acc: 0.9252\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3995 - acc: 0.8784 - val_loss: 0.2898 - val_acc: 0.9135\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3766 - acc: 0.8875 - val_loss: 0.2707 - val_acc: 0.9220\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.3507 - acc: 0.8916 - val_loss: 0.2396 - val_acc: 0.9284\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.3385 - acc: 0.8962 - val_loss: 0.2330 - val_acc: 0.9338\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.3253 - acc: 0.9062 - val_loss: 0.2276 - val_acc: 0.9348\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 40s 305ms/step - loss: 0.3233 - acc: 0.9008 - val_loss: 0.2241 - val_acc: 0.9359\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3086 - acc: 0.9088 - val_loss: 0.2175 - val_acc: 0.9359\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3084 - acc: 0.9070 - val_loss: 0.2113 - val_acc: 0.9338\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2889 - acc: 0.9145 - val_loss: 0.2198 - val_acc: 0.9370\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2752 - acc: 0.9183 - val_loss: 0.2144 - val_acc: 0.9370\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2706 - acc: 0.9173 - val_loss: 0.2074 - val_acc: 0.9423\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2671 - acc: 0.9184 - val_loss: 0.2084 - val_acc: 0.9391\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2738 - acc: 0.9149 - val_loss: 0.2140 - val_acc: 0.9412\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2591 - acc: 0.9232 - val_loss: 0.1983 - val_acc: 0.9412\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2396 - acc: 0.9256 - val_loss: 0.1953 - val_acc: 0.9423\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2394 - acc: 0.9258 - val_loss: 0.1922 - val_acc: 0.9412\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2323 - acc: 0.9302 - val_loss: 0.1940 - val_acc: 0.9402\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 40s 306ms/step - loss: 0.2287 - acc: 0.9300 - val_loss: 0.1881 - val_acc: 0.9412\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2333 - acc: 0.9297 - val_loss: 0.1911 - val_acc: 0.9412\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 43s 330ms/step - loss: 0.2317 - acc: 0.9285 - val_loss: 0.1897 - val_acc: 0.9423\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2372 - acc: 0.9293 - val_loss: 0.1881 - val_acc: 0.9412\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2326 - acc: 0.9272 - val_loss: 0.1887 - val_acc: 0.9412\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2197 - acc: 0.9312 - val_loss: 0.1883 - val_acc: 0.9444\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2266 - acc: 0.9304 - val_loss: 0.1888 - val_acc: 0.9434\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2239 - acc: 0.9285 - val_loss: 0.1882 - val_acc: 0.9423\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2149 - acc: 0.9335 - val_loss: 0.1879 - val_acc: 0.9423\n",
      "CNN 7: Epochs=30, Train accuracy=0.93349, Validation accuracy=0.94444\n",
      "Net : 8\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 43s 328ms/step - loss: 1.0316 - acc: 0.6904 - val_loss: 0.6327 - val_acc: 0.7714\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.5674 - acc: 0.8291 - val_loss: 0.4098 - val_acc: 0.8803\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 40s 306ms/step - loss: 0.4754 - acc: 0.8565 - val_loss: 0.3697 - val_acc: 0.8942\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.4197 - acc: 0.8670 - val_loss: 0.4638 - val_acc: 0.8397\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.4003 - acc: 0.8805 - val_loss: 0.3200 - val_acc: 0.9060\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.3722 - acc: 0.8917 - val_loss: 0.3118 - val_acc: 0.9135\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.3415 - acc: 0.8969 - val_loss: 0.2927 - val_acc: 0.9124\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.3286 - acc: 0.9020 - val_loss: 0.2802 - val_acc: 0.9156\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.3301 - acc: 0.9021 - val_loss: 0.2833 - val_acc: 0.9113\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.3109 - acc: 0.9053 - val_loss: 0.2677 - val_acc: 0.9209\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2953 - acc: 0.9114 - val_loss: 0.2655 - val_acc: 0.9188\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2829 - acc: 0.9131 - val_loss: 0.2601 - val_acc: 0.9241\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2692 - acc: 0.9151 - val_loss: 0.2543 - val_acc: 0.9241\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2662 - acc: 0.9161 - val_loss: 0.2695 - val_acc: 0.9209\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.2668 - acc: 0.9199 - val_loss: 0.2534 - val_acc: 0.9220\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2504 - acc: 0.9227 - val_loss: 0.2404 - val_acc: 0.9306\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2404 - acc: 0.9276 - val_loss: 0.2481 - val_acc: 0.9274\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2528 - acc: 0.9242 - val_loss: 0.2383 - val_acc: 0.9263\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2380 - acc: 0.9244 - val_loss: 0.2403 - val_acc: 0.9284\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2284 - acc: 0.9299 - val_loss: 0.2292 - val_acc: 0.9316\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 40s 307ms/step - loss: 0.2248 - acc: 0.9294 - val_loss: 0.2310 - val_acc: 0.9284\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2235 - acc: 0.9304 - val_loss: 0.2247 - val_acc: 0.9316\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 40s 307ms/step - loss: 0.2212 - acc: 0.9307 - val_loss: 0.2317 - val_acc: 0.9274\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2140 - acc: 0.9348 - val_loss: 0.2297 - val_acc: 0.9306\n",
      "Epoch 25/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2094 - acc: 0.9374 - val_loss: 0.2308 - val_acc: 0.9295\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2140 - acc: 0.9346 - val_loss: 0.2273 - val_acc: 0.9316\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 40s 308ms/step - loss: 0.2278 - acc: 0.9316 - val_loss: 0.2269 - val_acc: 0.9295\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2174 - acc: 0.9343 - val_loss: 0.2254 - val_acc: 0.9306\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 40s 308ms/step - loss: 0.2187 - acc: 0.9333 - val_loss: 0.2268 - val_acc: 0.9306\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2162 - acc: 0.9338 - val_loss: 0.2274 - val_acc: 0.9295\n",
      "CNN 8: Epochs=30, Train accuracy=0.93732, Validation accuracy=0.93162\n",
      "Net : 9\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 44s 334ms/step - loss: 1.0024 - acc: 0.6933 - val_loss: 0.4409 - val_acc: 0.8878\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.5769 - acc: 0.8248 - val_loss: 0.4853 - val_acc: 0.8333\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.4656 - acc: 0.8612 - val_loss: 0.3318 - val_acc: 0.9103\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.4076 - acc: 0.8758 - val_loss: 0.3178 - val_acc: 0.9145\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 40s 305ms/step - loss: 0.3773 - acc: 0.8907 - val_loss: 0.2781 - val_acc: 0.9177\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.3669 - acc: 0.8905 - val_loss: 0.2724 - val_acc: 0.9167\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.3335 - acc: 0.8998 - val_loss: 0.2658 - val_acc: 0.9241\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3148 - acc: 0.9063 - val_loss: 0.2716 - val_acc: 0.9241\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.3138 - acc: 0.9081 - val_loss: 0.2519 - val_acc: 0.9220\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3028 - acc: 0.9099 - val_loss: 0.2656 - val_acc: 0.9252\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2990 - acc: 0.9059 - val_loss: 0.2656 - val_acc: 0.9209\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 43s 324ms/step - loss: 0.2805 - acc: 0.9156 - val_loss: 0.2448 - val_acc: 0.9263\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2700 - acc: 0.9185 - val_loss: 0.2451 - val_acc: 0.9274\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2631 - acc: 0.9193 - val_loss: 0.2328 - val_acc: 0.9359\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2562 - acc: 0.9218 - val_loss: 0.2311 - val_acc: 0.9274\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2476 - acc: 0.9253 - val_loss: 0.2484 - val_acc: 0.9252\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2432 - acc: 0.9236 - val_loss: 0.2229 - val_acc: 0.9306\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2405 - acc: 0.9253 - val_loss: 0.2181 - val_acc: 0.9359\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2337 - acc: 0.9282 - val_loss: 0.2194 - val_acc: 0.9348\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2257 - acc: 0.9313 - val_loss: 0.2176 - val_acc: 0.9359\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2117 - acc: 0.9346 - val_loss: 0.2167 - val_acc: 0.9359\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2230 - acc: 0.9299 - val_loss: 0.2180 - val_acc: 0.9359\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.1995 - acc: 0.9390 - val_loss: 0.2178 - val_acc: 0.9380\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2115 - acc: 0.9365 - val_loss: 0.2202 - val_acc: 0.9359\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2195 - acc: 0.9339 - val_loss: 0.2178 - val_acc: 0.9370\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2035 - acc: 0.9349 - val_loss: 0.2112 - val_acc: 0.9391\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2074 - acc: 0.9363 - val_loss: 0.2132 - val_acc: 0.9348\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2084 - acc: 0.9342 - val_loss: 0.2155 - val_acc: 0.9348\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2074 - acc: 0.9348 - val_loss: 0.2081 - val_acc: 0.9380\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2062 - acc: 0.9333 - val_loss: 0.2139 - val_acc: 0.9370\n",
      "CNN 9: Epochs=30, Train accuracy=0.93900, Validation accuracy=0.93910\n",
      "Net : 10\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 44s 337ms/step - loss: 1.0177 - acc: 0.6945 - val_loss: 0.4866 - val_acc: 0.8526\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.5743 - acc: 0.8277 - val_loss: 0.3205 - val_acc: 0.9113\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.4926 - acc: 0.8489 - val_loss: 0.2427 - val_acc: 0.9231\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.4295 - acc: 0.8675 - val_loss: 0.2410 - val_acc: 0.9252\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.4138 - acc: 0.8773 - val_loss: 0.2349 - val_acc: 0.9274\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 44s 335ms/step - loss: 0.3743 - acc: 0.8869 - val_loss: 0.2212 - val_acc: 0.9348\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3631 - acc: 0.8904 - val_loss: 0.2007 - val_acc: 0.9370\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.3425 - acc: 0.8950 - val_loss: 0.2079 - val_acc: 0.9380\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3287 - acc: 0.9028 - val_loss: 0.1928 - val_acc: 0.9391\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3055 - acc: 0.9055 - val_loss: 0.1877 - val_acc: 0.9434\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.3122 - acc: 0.9102 - val_loss: 0.1823 - val_acc: 0.9466\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2907 - acc: 0.9127 - val_loss: 0.1796 - val_acc: 0.9444\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 40s 308ms/step - loss: 0.2828 - acc: 0.9150 - val_loss: 0.1920 - val_acc: 0.9402\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2865 - acc: 0.9114 - val_loss: 0.1771 - val_acc: 0.9498\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2696 - acc: 0.9174 - val_loss: 0.1769 - val_acc: 0.9455\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 40s 308ms/step - loss: 0.2672 - acc: 0.9198 - val_loss: 0.1746 - val_acc: 0.9444\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2561 - acc: 0.9238 - val_loss: 0.1636 - val_acc: 0.9509\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2557 - acc: 0.9196 - val_loss: 0.1577 - val_acc: 0.9487\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2456 - acc: 0.9229 - val_loss: 0.1622 - val_acc: 0.9541\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2440 - acc: 0.9289 - val_loss: 0.1617 - val_acc: 0.9530\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2465 - acc: 0.9225 - val_loss: 0.1579 - val_acc: 0.9530\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2361 - acc: 0.9280 - val_loss: 0.1702 - val_acc: 0.9476\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2291 - acc: 0.9303 - val_loss: 0.1547 - val_acc: 0.9519\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2295 - acc: 0.9283 - val_loss: 0.1542 - val_acc: 0.9476\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2235 - acc: 0.9315 - val_loss: 0.1577 - val_acc: 0.9519\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2234 - acc: 0.9292 - val_loss: 0.1542 - val_acc: 0.9530\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2053 - acc: 0.9394 - val_loss: 0.1515 - val_acc: 0.9530\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 40s 304ms/step - loss: 0.2150 - acc: 0.9329 - val_loss: 0.1546 - val_acc: 0.9551\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2138 - acc: 0.9316 - val_loss: 0.1476 - val_acc: 0.9541\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2069 - acc: 0.9338 - val_loss: 0.1485 - val_acc: 0.9551\n",
      "CNN 10: Epochs=30, Train accuracy=0.93935, Validation accuracy=0.95513\n",
      "Net : 11\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 44s 335ms/step - loss: 1.0224 - acc: 0.6966 - val_loss: 0.5992 - val_acc: 0.7874\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.5865 - acc: 0.8294 - val_loss: 0.4134 - val_acc: 0.8686\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 40s 307ms/step - loss: 0.5098 - acc: 0.8518 - val_loss: 0.2668 - val_acc: 0.9284\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.4392 - acc: 0.8627 - val_loss: 0.2325 - val_acc: 0.9370\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3885 - acc: 0.8863 - val_loss: 0.2363 - val_acc: 0.9327\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.3705 - acc: 0.8872 - val_loss: 0.2094 - val_acc: 0.9391\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.3517 - acc: 0.8972 - val_loss: 0.2212 - val_acc: 0.9444\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3359 - acc: 0.9009 - val_loss: 0.2105 - val_acc: 0.9434\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.3289 - acc: 0.9077 - val_loss: 0.1846 - val_acc: 0.9466\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 43s 329ms/step - loss: 0.3049 - acc: 0.9087 - val_loss: 0.1884 - val_acc: 0.9412\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.3000 - acc: 0.9112 - val_loss: 0.1812 - val_acc: 0.9455\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 43s 332ms/step - loss: 0.2944 - acc: 0.9097 - val_loss: 0.2086 - val_acc: 0.9434\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2716 - acc: 0.9183 - val_loss: 0.1658 - val_acc: 0.9509\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 43s 328ms/step - loss: 0.2503 - acc: 0.9245 - val_loss: 0.1699 - val_acc: 0.9498\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2559 - acc: 0.9241 - val_loss: 0.1682 - val_acc: 0.9530\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 43s 328ms/step - loss: 0.2738 - acc: 0.9157 - val_loss: 0.1636 - val_acc: 0.9509\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2548 - acc: 0.9236 - val_loss: 0.1702 - val_acc: 0.9541\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2463 - acc: 0.9263 - val_loss: 0.1678 - val_acc: 0.9509\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2370 - acc: 0.9272 - val_loss: 0.1642 - val_acc: 0.9455\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2526 - acc: 0.9239 - val_loss: 0.1622 - val_acc: 0.9498\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2331 - acc: 0.9311 - val_loss: 0.1646 - val_acc: 0.9509\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2344 - acc: 0.9274 - val_loss: 0.1626 - val_acc: 0.9498\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2295 - acc: 0.9304 - val_loss: 0.1629 - val_acc: 0.9498\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2309 - acc: 0.9296 - val_loss: 0.1606 - val_acc: 0.9509\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 43s 328ms/step - loss: 0.2252 - acc: 0.9293 - val_loss: 0.1624 - val_acc: 0.9498\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2285 - acc: 0.9306 - val_loss: 0.1621 - val_acc: 0.9519\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2320 - acc: 0.9294 - val_loss: 0.1626 - val_acc: 0.9530\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2286 - acc: 0.9280 - val_loss: 0.1608 - val_acc: 0.9509\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 43s 330ms/step - loss: 0.2298 - acc: 0.9292 - val_loss: 0.1601 - val_acc: 0.9509\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2309 - acc: 0.9300 - val_loss: 0.1604 - val_acc: 0.9519\n",
      "CNN 11: Epochs=30, Train accuracy=0.93110, Validation accuracy=0.95406\n",
      "Net : 12\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 45s 343ms/step - loss: 1.0333 - acc: 0.6839 - val_loss: 0.4440 - val_acc: 0.8665\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.5531 - acc: 0.8386 - val_loss: 0.4169 - val_acc: 0.8665\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.4653 - acc: 0.8539 - val_loss: 0.3358 - val_acc: 0.8974\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.4171 - acc: 0.8761 - val_loss: 0.3480 - val_acc: 0.8974\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.3703 - acc: 0.8875 - val_loss: 0.2995 - val_acc: 0.9017\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.3529 - acc: 0.8946 - val_loss: 0.2703 - val_acc: 0.9252\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 45s 342ms/step - loss: 0.3364 - acc: 0.9022 - val_loss: 0.2486 - val_acc: 0.9252\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.3168 - acc: 0.9054 - val_loss: 0.2557 - val_acc: 0.9241\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.3102 - acc: 0.9052 - val_loss: 0.2695 - val_acc: 0.9188\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2944 - acc: 0.9116 - val_loss: 0.2438 - val_acc: 0.9220\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2822 - acc: 0.9160 - val_loss: 0.2404 - val_acc: 0.9263\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 43s 329ms/step - loss: 0.2827 - acc: 0.9170 - val_loss: 0.2332 - val_acc: 0.9252\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2837 - acc: 0.9142 - val_loss: 0.2307 - val_acc: 0.9316\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2627 - acc: 0.9181 - val_loss: 0.2244 - val_acc: 0.9306\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2618 - acc: 0.9217 - val_loss: 0.2263 - val_acc: 0.9295\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2614 - acc: 0.9187 - val_loss: 0.2224 - val_acc: 0.9274\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2510 - acc: 0.9188 - val_loss: 0.2117 - val_acc: 0.9327\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 43s 330ms/step - loss: 0.2493 - acc: 0.9246 - val_loss: 0.2158 - val_acc: 0.9316\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 43s 328ms/step - loss: 0.2409 - acc: 0.9272 - val_loss: 0.2186 - val_acc: 0.9284\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2491 - acc: 0.9228 - val_loss: 0.2086 - val_acc: 0.9327\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2431 - acc: 0.9271 - val_loss: 0.2102 - val_acc: 0.9327\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2294 - acc: 0.9328 - val_loss: 0.2091 - val_acc: 0.9316\n",
      "Epoch 23/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2355 - acc: 0.9283 - val_loss: 0.2110 - val_acc: 0.9327\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2396 - acc: 0.9261 - val_loss: 0.2106 - val_acc: 0.9327\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2280 - acc: 0.9326 - val_loss: 0.2110 - val_acc: 0.9327\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2347 - acc: 0.9290 - val_loss: 0.2095 - val_acc: 0.9338\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2246 - acc: 0.9297 - val_loss: 0.2086 - val_acc: 0.9338\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2306 - acc: 0.9264 - val_loss: 0.2079 - val_acc: 0.9338\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2274 - acc: 0.9312 - val_loss: 0.2066 - val_acc: 0.9338\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2369 - acc: 0.9273 - val_loss: 0.2089 - val_acc: 0.9316\n",
      "CNN 12: Epochs=30, Train accuracy=0.93278, Validation accuracy=0.93376\n",
      "Net : 13\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 46s 351ms/step - loss: 1.0371 - acc: 0.6864 - val_loss: 0.3465 - val_acc: 0.8953\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.5779 - acc: 0.8262 - val_loss: 0.2786 - val_acc: 0.9177\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.4897 - acc: 0.8531 - val_loss: 0.2526 - val_acc: 0.9209\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 43s 329ms/step - loss: 0.4311 - acc: 0.8689 - val_loss: 0.2875 - val_acc: 0.9177\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.3943 - acc: 0.8839 - val_loss: 0.2154 - val_acc: 0.9316\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.3723 - acc: 0.8870 - val_loss: 0.2146 - val_acc: 0.9348\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.3407 - acc: 0.8983 - val_loss: 0.1923 - val_acc: 0.9402\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.3310 - acc: 0.8999 - val_loss: 0.1871 - val_acc: 0.9423\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.3098 - acc: 0.9090 - val_loss: 0.1832 - val_acc: 0.9434\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2976 - acc: 0.9093 - val_loss: 0.1828 - val_acc: 0.9466\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2987 - acc: 0.9096 - val_loss: 0.1787 - val_acc: 0.9476\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2930 - acc: 0.9134 - val_loss: 0.1684 - val_acc: 0.9476\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2864 - acc: 0.9131 - val_loss: 0.1689 - val_acc: 0.9434\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2747 - acc: 0.9185 - val_loss: 0.1596 - val_acc: 0.9476\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2498 - acc: 0.9238 - val_loss: 0.1572 - val_acc: 0.9476\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 43s 328ms/step - loss: 0.2573 - acc: 0.9232 - val_loss: 0.1606 - val_acc: 0.9466\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2424 - acc: 0.9276 - val_loss: 0.1607 - val_acc: 0.9487\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2499 - acc: 0.9244 - val_loss: 0.1636 - val_acc: 0.9455\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2376 - acc: 0.9277 - val_loss: 0.1592 - val_acc: 0.9455\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2409 - acc: 0.9275 - val_loss: 0.1605 - val_acc: 0.9509\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2343 - acc: 0.9282 - val_loss: 0.1578 - val_acc: 0.9476\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2311 - acc: 0.9284 - val_loss: 0.1529 - val_acc: 0.9487\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2319 - acc: 0.9300 - val_loss: 0.1594 - val_acc: 0.9509\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2275 - acc: 0.9298 - val_loss: 0.1579 - val_acc: 0.9509\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2271 - acc: 0.9299 - val_loss: 0.1566 - val_acc: 0.9466\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2148 - acc: 0.9342 - val_loss: 0.1532 - val_acc: 0.9476\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 44s 336ms/step - loss: 0.2106 - acc: 0.9372 - val_loss: 0.1571 - val_acc: 0.9455\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2193 - acc: 0.9306 - val_loss: 0.1545 - val_acc: 0.9487\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 43s 331ms/step - loss: 0.2016 - acc: 0.9385 - val_loss: 0.1552 - val_acc: 0.9476\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2161 - acc: 0.9308 - val_loss: 0.1548 - val_acc: 0.9487\n",
      "CNN 13: Epochs=30, Train accuracy=0.93840, Validation accuracy=0.95085\n",
      "Net : 14\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 45s 346ms/step - loss: 1.0027 - acc: 0.6976 - val_loss: 0.4039 - val_acc: 0.8974\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.5841 - acc: 0.8226 - val_loss: 0.4218 - val_acc: 0.8643\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.5002 - acc: 0.8504 - val_loss: 0.3673 - val_acc: 0.8921\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 40s 306ms/step - loss: 0.4414 - acc: 0.8676 - val_loss: 0.3226 - val_acc: 0.9092\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.4004 - acc: 0.8844 - val_loss: 0.2957 - val_acc: 0.9124\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3704 - acc: 0.8867 - val_loss: 0.2658 - val_acc: 0.9295\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3609 - acc: 0.8930 - val_loss: 0.2669 - val_acc: 0.9252\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.3329 - acc: 0.8995 - val_loss: 0.2514 - val_acc: 0.9263\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.3232 - acc: 0.9044 - val_loss: 0.3291 - val_acc: 0.8942\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3063 - acc: 0.9053 - val_loss: 0.2359 - val_acc: 0.9359\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3010 - acc: 0.9089 - val_loss: 0.2321 - val_acc: 0.9359\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2829 - acc: 0.9144 - val_loss: 0.2315 - val_acc: 0.9370\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2841 - acc: 0.9144 - val_loss: 0.2273 - val_acc: 0.9370\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2677 - acc: 0.9203 - val_loss: 0.2214 - val_acc: 0.9402\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2758 - acc: 0.9190 - val_loss: 0.2304 - val_acc: 0.9370\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2743 - acc: 0.9177 - val_loss: 0.2349 - val_acc: 0.9338\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2618 - acc: 0.9199 - val_loss: 0.2244 - val_acc: 0.9380\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2578 - acc: 0.9238 - val_loss: 0.2243 - val_acc: 0.9423\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2609 - acc: 0.9219 - val_loss: 0.2223 - val_acc: 0.9391\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2540 - acc: 0.9219 - val_loss: 0.2192 - val_acc: 0.9380\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2549 - acc: 0.9246 - val_loss: 0.2210 - val_acc: 0.9423\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2486 - acc: 0.9251 - val_loss: 0.2184 - val_acc: 0.9412\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2350 - acc: 0.9295 - val_loss: 0.2217 - val_acc: 0.9402\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2426 - acc: 0.9264 - val_loss: 0.2193 - val_acc: 0.9412\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2478 - acc: 0.9245 - val_loss: 0.2182 - val_acc: 0.9423\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2389 - acc: 0.9257 - val_loss: 0.2175 - val_acc: 0.9434\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2303 - acc: 0.9306 - val_loss: 0.2168 - val_acc: 0.9423\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 43s 326ms/step - loss: 0.2433 - acc: 0.9244 - val_loss: 0.2164 - val_acc: 0.9412\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2489 - acc: 0.9247 - val_loss: 0.2167 - val_acc: 0.9423\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2404 - acc: 0.9276 - val_loss: 0.2163 - val_acc: 0.9412\n",
      "CNN 14: Epochs=30, Train accuracy=0.93062, Validation accuracy=0.94338\n",
      "Net : 15\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 45s 346ms/step - loss: 1.0127 - acc: 0.6862 - val_loss: 0.4619 - val_acc: 0.8739\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.5548 - acc: 0.8367 - val_loss: 0.3418 - val_acc: 0.9092\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.4684 - acc: 0.8553 - val_loss: 0.3043 - val_acc: 0.9135\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 43s 328ms/step - loss: 0.4200 - acc: 0.8718 - val_loss: 0.2911 - val_acc: 0.9199\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 43s 330ms/step - loss: 0.3767 - acc: 0.8851 - val_loss: 0.2906 - val_acc: 0.9124\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.3648 - acc: 0.8913 - val_loss: 0.2996 - val_acc: 0.9177\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.3423 - acc: 0.8957 - val_loss: 0.2582 - val_acc: 0.9295\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 43s 324ms/step - loss: 0.3218 - acc: 0.9018 - val_loss: 0.3539 - val_acc: 0.8835\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.3111 - acc: 0.9088 - val_loss: 0.2418 - val_acc: 0.9295\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2936 - acc: 0.9127 - val_loss: 0.2429 - val_acc: 0.9295\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2762 - acc: 0.9146 - val_loss: 0.2408 - val_acc: 0.9241\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2728 - acc: 0.9197 - val_loss: 0.2295 - val_acc: 0.9252\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2711 - acc: 0.9175 - val_loss: 0.2309 - val_acc: 0.9284\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 42s 322ms/step - loss: 0.2685 - acc: 0.9194 - val_loss: 0.2273 - val_acc: 0.9241\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2485 - acc: 0.9201 - val_loss: 0.2211 - val_acc: 0.9295\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2472 - acc: 0.9237 - val_loss: 0.2227 - val_acc: 0.9274\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 43s 325ms/step - loss: 0.2496 - acc: 0.9233 - val_loss: 0.2219 - val_acc: 0.9316\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2436 - acc: 0.9280 - val_loss: 0.2167 - val_acc: 0.9316\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2501 - acc: 0.9238 - val_loss: 0.2201 - val_acc: 0.9316\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2430 - acc: 0.9242 - val_loss: 0.2233 - val_acc: 0.9295\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2365 - acc: 0.9295 - val_loss: 0.2211 - val_acc: 0.9327\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2506 - acc: 0.9219 - val_loss: 0.2215 - val_acc: 0.9306\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2359 - acc: 0.9284 - val_loss: 0.2224 - val_acc: 0.9306\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 43s 327ms/step - loss: 0.2407 - acc: 0.9244 - val_loss: 0.2218 - val_acc: 0.9316\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2407 - acc: 0.9254 - val_loss: 0.2202 - val_acc: 0.9316\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2424 - acc: 0.9262 - val_loss: 0.2199 - val_acc: 0.9338\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 43s 330ms/step - loss: 0.2299 - acc: 0.9258 - val_loss: 0.2203 - val_acc: 0.9306\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 44s 334ms/step - loss: 0.2362 - acc: 0.9296 - val_loss: 0.2215 - val_acc: 0.9327\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 42s 324ms/step - loss: 0.2395 - acc: 0.9285 - val_loss: 0.2200 - val_acc: 0.9327\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2343 - acc: 0.9308 - val_loss: 0.2194 - val_acc: 0.9327\n",
      "CNN 15: Epochs=30, Train accuracy=0.93074, Validation accuracy=0.93376\n",
      "Net : 16\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 44s 333ms/step - loss: 1.0423 - acc: 0.6843 - val_loss: 0.3842 - val_acc: 0.8835\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.5918 - acc: 0.8204 - val_loss: 0.6256 - val_acc: 0.8152\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.4814 - acc: 0.8550 - val_loss: 0.2792 - val_acc: 0.9167\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.4231 - acc: 0.8743 - val_loss: 0.2735 - val_acc: 0.9188\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.3892 - acc: 0.8844 - val_loss: 0.2463 - val_acc: 0.9284\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3772 - acc: 0.8859 - val_loss: 0.2954 - val_acc: 0.8942\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3542 - acc: 0.8916 - val_loss: 0.2334 - val_acc: 0.9295\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.3391 - acc: 0.8980 - val_loss: 0.2549 - val_acc: 0.9284\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.3141 - acc: 0.9029 - val_loss: 0.2756 - val_acc: 0.9017\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 40s 308ms/step - loss: 0.3146 - acc: 0.9057 - val_loss: 0.2115 - val_acc: 0.9348\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2918 - acc: 0.9099 - val_loss: 0.2187 - val_acc: 0.9370\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2773 - acc: 0.9181 - val_loss: 0.2079 - val_acc: 0.9391\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2813 - acc: 0.9153 - val_loss: 0.1942 - val_acc: 0.9370\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2643 - acc: 0.9185 - val_loss: 0.1874 - val_acc: 0.9444\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2516 - acc: 0.9228 - val_loss: 0.1932 - val_acc: 0.9434\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2624 - acc: 0.9178 - val_loss: 0.1907 - val_acc: 0.9455\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2528 - acc: 0.9239 - val_loss: 0.1950 - val_acc: 0.9444\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2490 - acc: 0.9215 - val_loss: 0.1775 - val_acc: 0.9476\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2325 - acc: 0.9310 - val_loss: 0.1796 - val_acc: 0.9444\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2438 - acc: 0.9263 - val_loss: 0.1792 - val_acc: 0.9509\n",
      "Epoch 21/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2505 - acc: 0.9227 - val_loss: 0.3208 - val_acc: 0.9017\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2386 - acc: 0.9273 - val_loss: 0.1717 - val_acc: 0.9509\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2269 - acc: 0.9296 - val_loss: 0.1753 - val_acc: 0.9498\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 40s 307ms/step - loss: 0.2159 - acc: 0.9325 - val_loss: 0.1727 - val_acc: 0.9509\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2060 - acc: 0.9367 - val_loss: 0.1709 - val_acc: 0.9551\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2144 - acc: 0.9330 - val_loss: 0.1657 - val_acc: 0.9519\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2063 - acc: 0.9368 - val_loss: 0.1641 - val_acc: 0.9551\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2156 - acc: 0.9354 - val_loss: 0.1675 - val_acc: 0.9466\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2107 - acc: 0.9329 - val_loss: 0.1676 - val_acc: 0.9498\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.1889 - acc: 0.9411 - val_loss: 0.1631 - val_acc: 0.9498\n",
      "CNN 16: Epochs=30, Train accuracy=0.94103, Validation accuracy=0.95513\n",
      "Net : 17\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 45s 340ms/step - loss: 1.0188 - acc: 0.6923 - val_loss: 0.3591 - val_acc: 0.8878\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.5740 - acc: 0.8286 - val_loss: 0.2577 - val_acc: 0.9135\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.4737 - acc: 0.8582 - val_loss: 0.2275 - val_acc: 0.9241\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.4271 - acc: 0.8665 - val_loss: 0.2027 - val_acc: 0.9412\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3907 - acc: 0.8838 - val_loss: 0.1990 - val_acc: 0.9359\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 40s 306ms/step - loss: 0.3758 - acc: 0.8894 - val_loss: 0.1972 - val_acc: 0.9380\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.3360 - acc: 0.8969 - val_loss: 0.2018 - val_acc: 0.9412\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.3114 - acc: 0.9078 - val_loss: 0.1857 - val_acc: 0.9444\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2953 - acc: 0.9149 - val_loss: 0.1778 - val_acc: 0.9466\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2987 - acc: 0.9100 - val_loss: 0.1752 - val_acc: 0.9476\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2953 - acc: 0.9109 - val_loss: 0.1942 - val_acc: 0.9391\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2795 - acc: 0.9173 - val_loss: 0.1695 - val_acc: 0.9509\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2877 - acc: 0.9125 - val_loss: 0.1753 - val_acc: 0.9444\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2904 - acc: 0.9143 - val_loss: 0.1666 - val_acc: 0.9498\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2822 - acc: 0.9155 - val_loss: 0.1624 - val_acc: 0.9487\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2631 - acc: 0.9237 - val_loss: 0.1595 - val_acc: 0.9498\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2632 - acc: 0.9200 - val_loss: 0.1593 - val_acc: 0.9487\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2702 - acc: 0.9199 - val_loss: 0.1585 - val_acc: 0.9487\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 42s 323ms/step - loss: 0.2486 - acc: 0.9204 - val_loss: 0.1559 - val_acc: 0.9530\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2559 - acc: 0.9221 - val_loss: 0.1567 - val_acc: 0.9519\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2616 - acc: 0.9203 - val_loss: 0.1552 - val_acc: 0.9530\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2441 - acc: 0.9245 - val_loss: 0.1560 - val_acc: 0.9530\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2563 - acc: 0.9207 - val_loss: 0.1541 - val_acc: 0.9562\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2626 - acc: 0.9220 - val_loss: 0.1546 - val_acc: 0.9551\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2546 - acc: 0.9212 - val_loss: 0.1541 - val_acc: 0.9551\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2483 - acc: 0.9263 - val_loss: 0.1551 - val_acc: 0.9562\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2496 - acc: 0.9221 - val_loss: 0.1557 - val_acc: 0.9541\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2461 - acc: 0.9228 - val_loss: 0.1557 - val_acc: 0.9551\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2454 - acc: 0.9260 - val_loss: 0.1555 - val_acc: 0.9541\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2453 - acc: 0.9260 - val_loss: 0.1553 - val_acc: 0.9551\n",
      "CNN 17: Epochs=30, Train accuracy=0.92632, Validation accuracy=0.95620\n",
      "Net : 18\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 45s 346ms/step - loss: 0.9874 - acc: 0.7001 - val_loss: 0.3477 - val_acc: 0.9006\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 40s 305ms/step - loss: 0.5512 - acc: 0.8321 - val_loss: 0.3145 - val_acc: 0.9167\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.4589 - acc: 0.8591 - val_loss: 0.2531 - val_acc: 0.9209\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.4182 - acc: 0.8748 - val_loss: 0.2627 - val_acc: 0.9209\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3742 - acc: 0.8891 - val_loss: 0.2190 - val_acc: 0.9370\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3643 - acc: 0.8881 - val_loss: 0.2058 - val_acc: 0.9370\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.3402 - acc: 0.8977 - val_loss: 0.2062 - val_acc: 0.9380\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3234 - acc: 0.9024 - val_loss: 0.1954 - val_acc: 0.9412\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3072 - acc: 0.9058 - val_loss: 0.1967 - val_acc: 0.9391\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2935 - acc: 0.9135 - val_loss: 0.1875 - val_acc: 0.9391\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2842 - acc: 0.9139 - val_loss: 0.1911 - val_acc: 0.9444\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2871 - acc: 0.9131 - val_loss: 0.1804 - val_acc: 0.9455\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2735 - acc: 0.9162 - val_loss: 0.1737 - val_acc: 0.9509\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2645 - acc: 0.9209 - val_loss: 0.1757 - val_acc: 0.9498\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2598 - acc: 0.9202 - val_loss: 0.1640 - val_acc: 0.9498\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 41s 309ms/step - loss: 0.2535 - acc: 0.9228 - val_loss: 0.1651 - val_acc: 0.9466\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2562 - acc: 0.9203 - val_loss: 0.1592 - val_acc: 0.9509\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2295 - acc: 0.9309 - val_loss: 0.1607 - val_acc: 0.9487\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2359 - acc: 0.9248 - val_loss: 0.1570 - val_acc: 0.9487\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2228 - acc: 0.9314 - val_loss: 0.1572 - val_acc: 0.9476\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 41s 317ms/step - loss: 0.2366 - acc: 0.9279 - val_loss: 0.1625 - val_acc: 0.9455\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2190 - acc: 0.9336 - val_loss: 0.1572 - val_acc: 0.9498\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2133 - acc: 0.9344 - val_loss: 0.1560 - val_acc: 0.9509\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2103 - acc: 0.9376 - val_loss: 0.1560 - val_acc: 0.9498\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.2134 - acc: 0.9354 - val_loss: 0.1560 - val_acc: 0.9509\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2151 - acc: 0.9343 - val_loss: 0.1559 - val_acc: 0.9541\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2289 - acc: 0.9314 - val_loss: 0.1548 - val_acc: 0.9530\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.2152 - acc: 0.9348 - val_loss: 0.1545 - val_acc: 0.9509\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2073 - acc: 0.9366 - val_loss: 0.1554 - val_acc: 0.9509\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2103 - acc: 0.9363 - val_loss: 0.1546 - val_acc: 0.9519\n",
      "CNN 18: Epochs=30, Train accuracy=0.93756, Validation accuracy=0.95406\n",
      "Net : 19\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 45s 342ms/step - loss: 1.0382 - acc: 0.6861 - val_loss: 0.3439 - val_acc: 0.9038\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.5728 - acc: 0.8244 - val_loss: 0.4243 - val_acc: 0.8675\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.4874 - acc: 0.8489 - val_loss: 0.2562 - val_acc: 0.9241\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.4284 - acc: 0.8717 - val_loss: 0.3359 - val_acc: 0.9017\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3937 - acc: 0.8806 - val_loss: 0.2645 - val_acc: 0.9252\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.3623 - acc: 0.8909 - val_loss: 0.2175 - val_acc: 0.9359\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.3419 - acc: 0.8976 - val_loss: 0.2113 - val_acc: 0.9338\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.3390 - acc: 0.8986 - val_loss: 0.1905 - val_acc: 0.9391\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3030 - acc: 0.9095 - val_loss: 0.1993 - val_acc: 0.9380\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2999 - acc: 0.9088 - val_loss: 0.1891 - val_acc: 0.9444\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2865 - acc: 0.9145 - val_loss: 0.1828 - val_acc: 0.9444\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 310ms/step - loss: 0.2811 - acc: 0.9171 - val_loss: 0.1894 - val_acc: 0.9370\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2728 - acc: 0.9196 - val_loss: 0.1812 - val_acc: 0.9423\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2705 - acc: 0.9188 - val_loss: 0.2017 - val_acc: 0.9444\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2538 - acc: 0.9235 - val_loss: 0.1713 - val_acc: 0.9444\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2420 - acc: 0.9254 - val_loss: 0.1792 - val_acc: 0.9423\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2519 - acc: 0.9218 - val_loss: 0.1772 - val_acc: 0.9444\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2369 - acc: 0.9280 - val_loss: 0.1768 - val_acc: 0.9444\n",
      "Epoch 19/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2410 - acc: 0.9272 - val_loss: 0.1777 - val_acc: 0.9444\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 41s 315ms/step - loss: 0.2421 - acc: 0.9267 - val_loss: 0.1753 - val_acc: 0.9444\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2380 - acc: 0.9274 - val_loss: 0.1741 - val_acc: 0.9487\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2319 - acc: 0.9297 - val_loss: 0.1794 - val_acc: 0.9434\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2286 - acc: 0.9274 - val_loss: 0.1782 - val_acc: 0.9412\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 40s 308ms/step - loss: 0.2281 - acc: 0.9291 - val_loss: 0.1760 - val_acc: 0.9412\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2348 - acc: 0.9271 - val_loss: 0.1768 - val_acc: 0.9444\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2325 - acc: 0.9279 - val_loss: 0.1765 - val_acc: 0.9434\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2217 - acc: 0.9326 - val_loss: 0.1763 - val_acc: 0.9444\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2236 - acc: 0.9317 - val_loss: 0.1757 - val_acc: 0.9434\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2211 - acc: 0.9315 - val_loss: 0.1764 - val_acc: 0.9434\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2262 - acc: 0.9305 - val_loss: 0.1759 - val_acc: 0.9423\n",
      "CNN 19: Epochs=30, Train accuracy=0.93266, Validation accuracy=0.94872\n",
      "Net : 20\n",
      "Epoch 1/30\n",
      "131/131 [==============================] - 45s 346ms/step - loss: 1.0364 - acc: 0.6834 - val_loss: 0.7522 - val_acc: 0.7585\n",
      "Epoch 2/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.5968 - acc: 0.8175 - val_loss: 0.3345 - val_acc: 0.9060\n",
      "Epoch 3/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.4970 - acc: 0.8503 - val_loss: 0.2522 - val_acc: 0.9284\n",
      "Epoch 4/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.4300 - acc: 0.8724 - val_loss: 0.2998 - val_acc: 0.9135\n",
      "Epoch 5/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.4048 - acc: 0.8789 - val_loss: 0.2627 - val_acc: 0.9241\n",
      "Epoch 6/30\n",
      "131/131 [==============================] - 41s 314ms/step - loss: 0.3657 - acc: 0.8917 - val_loss: 0.2126 - val_acc: 0.9306\n",
      "Epoch 7/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3553 - acc: 0.8932 - val_loss: 0.2073 - val_acc: 0.9359\n",
      "Epoch 8/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.3367 - acc: 0.9005 - val_loss: 0.2029 - val_acc: 0.9391\n",
      "Epoch 9/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3258 - acc: 0.9025 - val_loss: 0.2194 - val_acc: 0.9263\n",
      "Epoch 10/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.3153 - acc: 0.9026 - val_loss: 0.2005 - val_acc: 0.9412\n",
      "Epoch 11/30\n",
      "131/131 [==============================] - 41s 313ms/step - loss: 0.2983 - acc: 0.9088 - val_loss: 0.1913 - val_acc: 0.9423\n",
      "Epoch 12/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2870 - acc: 0.9150 - val_loss: 0.1793 - val_acc: 0.9434\n",
      "Epoch 13/30\n",
      "131/131 [==============================] - 42s 318ms/step - loss: 0.2807 - acc: 0.9134 - val_loss: 0.2263 - val_acc: 0.9145\n",
      "Epoch 14/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2758 - acc: 0.9164 - val_loss: 0.1871 - val_acc: 0.9391\n",
      "Epoch 15/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2793 - acc: 0.9208 - val_loss: 0.1853 - val_acc: 0.9380\n",
      "Epoch 16/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2542 - acc: 0.9215 - val_loss: 0.1973 - val_acc: 0.9295\n",
      "Epoch 17/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2488 - acc: 0.9262 - val_loss: 0.1634 - val_acc: 0.9412\n",
      "Epoch 18/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2513 - acc: 0.9255 - val_loss: 0.1647 - val_acc: 0.9423\n",
      "Epoch 19/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2436 - acc: 0.9246 - val_loss: 0.1660 - val_acc: 0.9444\n",
      "Epoch 20/30\n",
      "131/131 [==============================] - 40s 309ms/step - loss: 0.2337 - acc: 0.9268 - val_loss: 0.1622 - val_acc: 0.9444\n",
      "Epoch 21/30\n",
      "131/131 [==============================] - 42s 321ms/step - loss: 0.2409 - acc: 0.9298 - val_loss: 0.1641 - val_acc: 0.9466\n",
      "Epoch 22/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2261 - acc: 0.9283 - val_loss: 0.1591 - val_acc: 0.9455\n",
      "Epoch 23/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2474 - acc: 0.9240 - val_loss: 0.1647 - val_acc: 0.9444\n",
      "Epoch 24/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2314 - acc: 0.9300 - val_loss: 0.1592 - val_acc: 0.9487\n",
      "Epoch 25/30\n",
      "131/131 [==============================] - 41s 311ms/step - loss: 0.2344 - acc: 0.9285 - val_loss: 0.1590 - val_acc: 0.9434\n",
      "Epoch 26/30\n",
      "131/131 [==============================] - 42s 320ms/step - loss: 0.2283 - acc: 0.9297 - val_loss: 0.1588 - val_acc: 0.9455\n",
      "Epoch 27/30\n",
      "131/131 [==============================] - 42s 317ms/step - loss: 0.2403 - acc: 0.9258 - val_loss: 0.1614 - val_acc: 0.9434\n",
      "Epoch 28/30\n",
      "131/131 [==============================] - 41s 316ms/step - loss: 0.2189 - acc: 0.9371 - val_loss: 0.1601 - val_acc: 0.9444\n",
      "Epoch 29/30\n",
      "131/131 [==============================] - 41s 312ms/step - loss: 0.2262 - acc: 0.9310 - val_loss: 0.1595 - val_acc: 0.9455\n",
      "Epoch 30/30\n",
      "131/131 [==============================] - 42s 319ms/step - loss: 0.2226 - acc: 0.9337 - val_loss: 0.1618 - val_acc: 0.9466\n",
      "CNN 20: Epochs=30, Train accuracy=0.93708, Validation accuracy=0.94872\n"
     ]
    }
   ],
   "source": [
    "# train the data and print some information\n",
    "history = [0] * nets\n",
    "epochs = 30\n",
    "\n",
    "datagen = ImageDataGenerator(\n",
    "    rotation_range=13,\n",
    "    zoom_range=0.11,\n",
    "    width_shift_range=0.1,\n",
    "    height_shift_range=0.1\n",
    ")\n",
    "\n",
    "datagen.fit(X_train)\n",
    "\n",
    "for j in range(nets):\n",
    "    print(f'Net : {j+1}')   \n",
    "    X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)\n",
    "    history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),\n",
    "        epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,  \n",
    "        validation_data = (X_val2,Y_val2), callbacks=[learning_rate_reduction], verbose=1)\n",
    "    \n",
    "    print(\"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n",
    "        j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc']) ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make submision\n",
    "results = np.zeros( (X_test.shape[0],10) ) \n",
    "for j in range(nets):\n",
    "    results = results + model[j].predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = np.argmax(results,axis = 1)\n",
    "results = pd.Series(results,name=\"target\")\n",
    "submission = pd.concat([pd.Series(range(0,9364),name = \"Id\"),results],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  target\n",
       "0   0       0\n",
       "1   1       7\n",
       "2   2       0\n",
       "3   3       0\n",
       "4   4       7"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show submission\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encode labels back into original values\n",
    "submission['target'] = enc.inverse_transform(submission['target'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>H</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>H</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id target\n",
       "0   0      A\n",
       "1   1      H\n",
       "2   2      A\n",
       "3   3      A\n",
       "4   4      H"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9364, 2)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check submission\n",
    "submission.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "J    980\n",
       "D    978\n",
       "G    955\n",
       "H    944\n",
       "E    940\n",
       "F    934\n",
       "B    923\n",
       "C    919\n",
       "A    910\n",
       "I    881\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save submission to csv\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "v_conda",
   "language": "python",
   "name": "v_conda"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}