nn_regression_batch_normalization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import necessary modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers, Model\n",
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load and prepare energy efficiency data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget https://archive.ics.uci.edu/ml/machine-learning-databases/00242/ENB2012_data.xlsx\n",
    "\n",
    "data_df = pd.read_excel('ENB2012_data.xlsx')\n",
    "data = np.array(data_df)\n",
    "\n",
    "N = data.shape[0]\n",
    "X = data[:,:8]\n",
    "y = data[:,8]\n",
    "\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(X, y, random_state=42)\n",
    "X_train, X_validate, y_train, y_validate = train_test_split(X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_validate = scaler.transform(X_validate)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Construct FFNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 8)]               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 30)                270       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 3,091\n",
      "Trainable params: 3,091\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "initializer_name = \"glorot_normal\"\n",
    "\n",
    "input_layer1 = layers.Input(shape=(8,))\n",
    "dense_layer1 = layers.Dense(30, activation=\"sigmoid\", kernel_initializer=initializer_name, input_shape=(8,))(input_layer1)\n",
    "dense_layer2 = layers.Dense(30, activation=\"sigmoid\", kernel_initializer=initializer_name)(dense_layer1)\n",
    "dense_layer3 = layers.Dense(30, activation=\"sigmoid\", kernel_initializer=initializer_name)(dense_layer2)\n",
    "dense_layer4 = layers.Dense(30, activation=\"sigmoid\", kernel_initializer=initializer_name)(dense_layer3)\n",
    "output_layer1 = layers.Dense(1, kernel_initializer=initializer_name)(dense_layer4)\n",
    "\n",
    "model = Model(inputs=input_layer1, outputs=output_layer1)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Configure and run training with batch size $N_b=10$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mean_squared_error\", optimizer=keras.optimizers.SGD(lr=1e-3))\n",
    "history = model.fit(X_train, y_train, batch_size=10, epochs=20, validation_data=(X_validate, y_validate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot training history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1800x1200 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAAEvCAYAAADvkw2zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gc1aH38e/ZIq20klW8ci9aYborGBsCGFNiIA4l9ZoQLiWYcElCSPJyCSkE3pv3BvBNDwkBwiUEEiAJwYQWQrBpAWPsuGIw4Ibcm2RZdct5/5iVtJJW9q602l1Jv8/zzLNTzsweDSP5x5k5Z4y1FhERERHJPle2KyAiIiIiDgUzERERkRyhYCYiIiKSIxTMRERERHKEgpmIiIhIjlAwExEREckRnmxXIF0CgYCtrKzs0++or6/H7/f36Xf0FzoX7XQuHDoP7XQu2ulctNO5cOg8OJYtW7bHWlvRef2ACWaVlZW89dZbffodixcvZvbs2X36Hf2FzkU7nQuHzkM7nYt2OhftdC4cOg8OY8zmROt1K1NEREQkRyiYiYiIiOQIBTMRERGRHDFgnjETERGRzAmFQlRXV9PU1JTSfiUlJaxbt66PapV7fD4fY8aMwev1JlVewUxERERSVl1dTXFxMZWVlRhjkt6vrq6O4uLiPqxZ7rDWsnfvXqqrqwkGg0nto1uZIiIikrKmpiaGDh2aUigbbIwxDB06NKVWRQUzERER6RGFssNL9RwpmImIiEi/VFRUlO0qpJ2CmYiIiEiOUDBL0pINe3l9Wzjb1RAREZFOrLXceOONTJw4kUmTJvHoo48CsH37dmbNmsXUqVOZOHEir7zyCpFIhCuuuKKt7I9//OMs174j9cpM0h+XVfPCuy3cnO2KiIiISAePP/44K1asYOXKlezZs4eTTjqJWbNm8fvf/55zzz2Xb3/720QiERoaGlixYgVbt25lzZo1ANTU1GS59h0pmCUpGPBT02ypbw7jz9dpExERaXXbX9fy9rYDSZWNRCK43e7Dljtu1BC+d8HxSR3z1Vdf5ZJLLsHtdjN8+HDOOOMMli5dykknncRVV11FKBTi4osvZurUqVRVVbFhwwa+8pWvMHfuXObMmZPUd2SKbmUmqSrgB2Djnvos10RERETiWWsTrp81axYvv/wyo0eP5rLLLuPBBx+krKyMlStXMnv2bO666y6uvvrqDNf20NT0k6RgRXswmzi6JMu1ERERyR3JtmxB3wwwO2vWLH79619z+eWXs2/fPl5++WUWLFjA5s2bGT16NPPnz6e+vp7ly5fzsY99jLy8PD71qU9xxBFHcMUVV6S1Lr2lYJakyqFqMRMREclFn/jEJ3j99deZMmUKxhjuvPNORowYwW9/+1sWLFiA1+ulqKiIBx98kK1bt3LllVcSjUYB+MEPfpDl2nekYJYkn9fNUJ9RMBMREckRBw8eBJxBXBcsWMCCBQs6bL/88su5/PLLu+y3fPnyjNSvJ/SMWQpG+A0bFMxERESkj+R0MDPGXGyMudcYs9AYk/VuE8P9LjbuPtjtQ4YiIiIivXHYYGaM8Rlj3jTGrDTGrDXG3NbTLzPG3G+M2WWMWZNg23nGmHeNMe8bY74JYK19wlo7H7gC+Leefm+6jCh0caApzL76lmxXRURERAagZFrMmoGzrLVTgKnAecaYk+MLGGOGGWOKO62bkOBYDwDndV5pjHEDdwHnA8cBlxhjjosr8p3Y9qwa4XdeRKrnzERERKQvHDaYWcfB2KI3NnW+l3cGsNAY4wMwxswHfpbgWC8D+xJ8zQzgfWvtBmttC/AIcJFx3AE8a63N+pN6I/zO6dJzZiIiItIXknrGzBjjNsasAHYBf7fWLonfbq39I/Ac8Igx5lLgKuCzKdRjNPBh3HJ1bN1XgHOATxtjru2mbhcYY+6pra1N4et6ZqjP4HWrZ6aIiIj0jaSCmbU2Yq2dCowBZhhjJiYocyfQBPwKuDCulS0ZJvHX2p9Za0+01l5rrb27m7r91Vp7TUlJ3w/66nYZxpUXsnG3gpmIiIikX0q9Mq21NcBiEj8ndjowEfgL8L0U61ENjI1bHgNsS/EYGREMFKnFTEREpJ8pKirqdtumTZuYOLFLm1NWJNMrs8IYUxqbL8C5tfhOpzLTgHuBi4ArgXJjzPdTqMdS4EhjTNAYkwfMA55MYf+Mqarws3FvPdGohswQERGR9EqmxWwksMgYswonQP3dWvtUpzKFwGestR9Ya6PA5cDmzgcyxvwBeB042hhTbYz5AoC1Ngx8GfgbsA54zFq7tqc/VF8KBvy0hKNsq23MdlVEREQGrZtuuolf/vKXbcu33nort912G2effTYnnHACkyZNYuHChSkft6mpiSuvvJJJkyYxbdo0Fi1aBMDatWuZMWMGU6dOZfLkybz33nvU19czd+5cpkyZwsSJE3n00Ud7/XMd9pVM1tpVwLTDlHmt03IIpwWtc7lLDnGMZ4BnDlefbAsG2t+ZOaasMMu1ERERyQHPfhN2rE6qaEEkDO4k3gg5YhKcf3u3m+fNm8cNN9zAddddB8Bjjz3Gc889x9e+9jWGDBnCnj17OPnkk7nwwgsxJtGj7InddZczOtfq1at55513mDNnDuvXr+fuu+/mq1/9KpdeeiktLS1EIhGeeeYZRo0axdNPPw1AOjoi5vTI/7koPpiJiIhIdkybNo1du3axbds2Vq5cSVlZGSNHjuRb3/oWkydP5pxzzmHr1q3s3LkzpeO++uqrXHbZZQAcc8wxjB8/nvXr13PKKafw3//939xxxx1s3ryZgoICJk2axAsvvMBNN93EK6+8Qjo6Iuol5ikaVpxPYZ6bDeqZKSIi4jhEy1ZnjXV1FBcXH75gEj796U/zpz/9iR07djBv3jwefvhhdu/ezbJly/B6vVRWVtLU1JTSMbt77eLnPvc5Zs6cydNPP825557Lfffdx1lnncWyZct45plnuPnmm5kzZw633HJLr34mBbMUGWMIBvxqMRMREcmyefPmMX/+fPbs2cNLL73EY489xrBhw/B6vSxatIjNm7s87n5Ys2bN4uGHH+ass85i/fr1bNmyhaOPPpoNGzZQVVXF9ddfz4YNG1i1ahXHHHMM5eXlfP7zn6eoqIgHHnig1z+TglkPBAN+VlX3/YC2IiIi0r3jjz+euro6Ro8ezciRI7n00ku54IILmD59OlOnTuWYY45J+ZjXXXcd1157LZMmTcLj8fDAAw+Qn5/Po48+ykMPPYTX62XEiBHccsstLF26lBtvvBGXy4XX6+VXv/pVr38mBbMeqAr4eWb1dprDEfI97mxXR0REZNBavbq900EgEOD1119PWO7gwe7Hva+srGTNmjUA+Hy+hC1fN998MzfffHOHdeeeey7nnntuD2rdPT383wPBCj9RCx/ua8h2VURERGQAUYtZDwQDzujBG3bXM2FYeh5gFBERkb61evXqth6XrfLz81myZEk3e2SeglkPBIdqyAwREZH+ZtKkSaxYsSLb1Tgk3crsgZJCL0P9eQpmIiIyqHU3tIS0S/UcKZj1UDDgZ4OCmYiIDFI+n4+9e/cqnB2CtZa9e/fi8/mS3ke3MnsoGPCzeP3ubFdDREQkK8aMGUN1dTW7d6f2b2FTU1NKQaW/8/l8jBkzJunyCmY9FKzw88dl1dQ1hSj2ebNdHRERkYzyer0Eg8GU91u8eDHTph3yFdyDmm5l9lBV7J2Zm/ZoyAwRERFJDwWzHmobMmNP9wPWiYiIiKRCwayHxg8txBi1mImIiEj6KJj1kM/rZlRJARvVYiYiIiJpomDWC1UVfo1lJiIiImmjYNYLrWOZaQwXERERSQcFs14IBvzUNYXZW9+S7aqIiIjIAKBg1gvBgN6ZKSIiIumjYNYLVbEhMzbuVjATERGR3lMw64XRZQV43UbvzBQREZG0UDDrBbfLMH6oX0NmiIiISFoomPVSMKAhM0RERCQ9FMx6qSrgZ9PeBiJRDZkhIiIivaNg1kvBgJ+WcJRtNY3ZroqIiIj0cwpmvaQhM0RERCRdFMx6KVihYCYiIiLpoWDWSxVF+RTlexTMREREpNcUzHrJGNP2zkwRERGR3lAwSwNnyAyNZSYiIiK9o2CWBsGAn+r9jTSHI9muioiIiPRjCmZpUFXhx1rYsrch21URERGRfkzBLA1ah8zQc2YiIiLSGwpmaVCpscxEREQkDRTM0mCIz0ugKI+NuxXMREREpOcUzNJELzMXERGR3lIwSxONZSYiIiK9pWCWJsFAEXsONnOgKZTtqoiIiEg/pWCWJq09Mzep1UxERER6SMEsTar0MnMRERHpJQWzNBlXXogxsEE9M0VERKSHFMzSxOd1M7q0QC1mIiIi0mMKZmmkITNERESkNxTM0qgqFsystdmuioiIiPRDCmZpFAz4OdgcZvfB5mxXRURERPohBbM0ClYUAejVTCIiItIjCmZpVKWXmYuIiEgvKJil0ajSAvLcLgUzERER6REFszRyuwzjhxYqmImIiEiPKJilmYbMEBERkZ5SMEuzYIWfzXsbiEQ1ZIaIiIikRsEszaoCfloiUbbVNGa7KiIiItLPKJilWTDgDJmxQbczRUREJEUKZmkWbB0yY/fBLNdERERE+hsFszQLFOVRnO9RBwARERFJmYJZmhljCFb4dStTREREUqZg1gc0ZIaIiIj0hIJZHwgG/GytaaQpFMl2VURERKQfyelgZoy52BhzrzFmoTFmTrbrk6xgwI+1sGVfQ7arIiIiIv3IYYOZMWasMWaRMWadMWatMearPf0yY8z9xphdxpg1CbadZ4x51xjzvjHmmwDW2iestfOBK4B/6+n3ZlpV65AZu3U7U0RERJKXTItZGPiGtfZY4GTgS8aY4+ILGGOGGWOKO62bkOBYDwDndV5pjHEDdwHnA8cBl3T6ju/EtvcLlYFCAD1nJiIiIik5bDCz1m631i6PzdcB64DRnYqdASw0xvgAjDHzgZ8lONbLwL4EXzMDeN9au8Fa2wI8AlxkHHcAz7bWoT8o9nmpKM5n4x6NZSYiIiLJ86RS2BhTCUwDlsSvt9b+0RgTBB4xxvwRuAr4aAqHHg18GLdcDcwEvgKcA5QYYyZYa+9OUKcLgAsmTEjUQJc96pkpIiIiqUr64X9jTBHwZ+AGa+2BztuttXcCTcCvgAuttak0F5kE66y19mfW2hOttdcmCmWxQn+11l5TUlKSwtf1vSoFMxEREUlRUsHMGOPFCWUPW2sf76bM6cBE4C/A91KsRzUwNm55DLAtxWPklGDAz56DLdQ2hrJdFREREeknkumVaYDfAOustT/qpsw04F7gIuBKoNwY8/0U6rEUONIYEzTG5AHzgCdT2D/ntL4zc5NazURERCRJybSYnQpcBpxljFkRmz7WqUwh8Blr7QfW2ihwObC584GMMX8AXgeONsZUG2O+AGCtDQNfBv6G07ngMWvt2h7/VDmgqiL2MnMFMxEREUnSYR/+t9a+SuJnwOLLvNZpOYTTgta53CWHOMYzwDOHq09/Mba8EJdB78wUERGRpOX0yP/9Wb7HzeiyArWYiYiISNIUzPpQMFCkscxEREQkaQpmfagq4Gfj7nqstdmuioiIiPQDCmZ9KBjwU98SYXddc7arIiIiIv2Aglkfah0yQx0AREREJBkKZn2oNZipA4CIiIgkQ8GsD40qLSDP41IwExERkaQomPUht8tQObSQDbsVzEREROTwFMz6WDDg15AZIiIikhQFsz4WDBSxZV8D4Ug021URERGRHKdg1seqAn5CEcvWmsZsV0VERERynIJZHwtWaMgMERERSY6CWR9rGzJDHQBERETkMBTM+thQfx7FPo+GzBAREZHDUjDrY8YY552ZCmYiIiJyGApmGRBUMBMREZEkKJhlQDBQxLbaRppCkWxXRURERHKYglkGBCv8WAub9zZkuyoiIiKSwxTMMqCq7WXmegOAiIiIdE/BLAMqAxrLTERERA5PwSwDivI9DCvO11hmIiIickgKZhminpkiIiJyOApmGVJVoWAmIiIih6ZgliHBgJ+99S3UNoSyXRURERHJUQpmGRIMFAGwca9azURERCQxBbMMCWrIDBERETkMBbMMGVdeiMugnpkiIiLSLQWzDMnzuBhbXqixzERERKRbCmYZpCEzRERE5FAUzDKoNZhZa7NdFREREclBCmYZVBXw09ASYVddc7arIiIiIjlIwSyDWofM2KAOACIiIpKAglkGBStah8xQMBMREZGuFMwyaOQQH/kel8YyExERkYQUzDLI5TJUDlXPTBEREUlMwSzDggG/xjITERGRhBTMMixY4WfL3gbCkWi2qyIiIiI5RsEsw4IBP+GopXp/Y7arIiIiIjlGwSzDqgLqmSkiIiKJKZhlWDAWzPScmYiIiHSmYJZh5f48hvg8GjJDREREulAwyzBjDMGKIt3KFBERkS4UzLKgKuBno17LJCIiIp0omCXrXw9R9cH/puVQwYCfbbVNNLZE0nI8ERERGRgUzJK1ax1jP3wSaqt7fajWDgCb9qrVTERERNopmCXrpKsBC2/d3+tDBTVkhoiIiCSgYJas8iB7h06HZQ9AqKlXh1IwExERkUQUzFKwdfTHoWEvrH28V8fx53sYPiSfDeoAICIiInEUzFKwv2wKBI6CJb8Ga3t1rGDAr7HMREREpAMFs1QYAzOuge0roHpprw4VDGgsMxEREelIwSxVU+ZBXrHTatYLVQE/+xtC7K9vSVPFREREpL9TMEtVfjFMuxTefgLqdvT4MG0dADRkhoiIiMQomPXESfMhGnZ6aPZQsCI2lpluZ4qIiEiMgllPBCbAhHOcMc3CPbsVObasELfL6DkzERERaaNg1lMzvggHd8K6J3u0e57HxdiyAjYomImIiEiMgllPTTgHyqt61QkgqJeZi4iISBwFs55yuZxnzarfhG3/6tEhWofMsL0cE01EREQGBgWz3ph2KXj9sOSeHu0erPDTGIqw80BzmismIiIi/ZGCWW/4Spxxzdb8Ger3pLx7VWzIjA16A4CIiIigYNZ7M66BSHOPhs7Qy8xFREQknoJZbw07BoJnOENnRMIp7TpiiA+f16UOACIiIgIomKXHzC/Cga3wzlMp7eZyGSqH+tViJiIiIoCCWXocdR6UjoM3U+8EUFWhYCYiIiIOBbN0cLnhpKth82uwY01KuwYDfrbsayAUifZR5URERKS/UDBLl2mXgacA3kxtwNlgoIhw1FK9v7GPKiYiIiL9hYJZuhSWw+TPwKo/QsO+pHdr75mpITNEREQGOwWzdJrxRQg3wr9+l/QubWOZqWemiIjIoKdglk4jJsL4U2HpfRCNJLVLmT+PkgKvOgCIiIiIglnazbgGarbA+ueS3iUYUM9MERERUTBLv2M+DkNGw5LkOwFUKZiJiIgICmbp5/bA9Ktg40uw652kdgkG/GyvbaKhJbU3B4iIiMjAomDWF068Atz5SQ84G6xwOgBs2tPQh5USERGRXKdg1hf8AZj4KVj5CDTVHra4XmYuIiIioGDWd2ZeA6F6+NfDhy1aOVRjmYmIiIiCWd8ZNQ3GzICl90L00K9b8ud7GDHExwa1mImIiAxqCmZ9aeYXYd8GeP+FwxbVkBkiIiKiYNaXjr0QioYn9f7MYIWCmYiIyGCnYNaXPHnO0BnvvwB73j9k0aqAn5qGEPvrWzJUOREREck1CmZ97cQrweV1njU7hNaemXrOTEREZPBSMOtrxcPh+Iud3pnNdd0W05AZIiIiomCWCTO+CC11zrhm3RhbXojbZTRkhoiIyCCmYJYJY6Y7w2e8eQ9Ym7CI1+1iXHmhWsxEREQGMQWzTDDGaTXbsx42LOq2WDDgZ8NuBTMREZHBSsEsUyZ+EgoDsKT792cGA3427a0nGk3cqiYiIiIDm4JZpnjynZebr38O9m9KWCQY8NMUirLjQFNGqyYiIiK5QcEsk6ZfBcYFS+9LuLlKPTNFREQGNQWzTCoZDcdeAMt/By0NXTYHKzSWmYiIyGCmYJZpM78ITTWw+rEum4YX+yjwutmoDgAiIiKDkoJZpo07BYZPcjoBdBo6w+UyVMY6AIiIiMjgo2CWacbAzGtg11rY/FqXzVUBvcxcRERksFIwy4ZJn4GCMljy6y6bggE/W/Y1EIpEs1AxERERySYFs2zwFsC0y+Cdp6G2usOmYMBPJGr5cF/XzgEiIiIysCmYZctJVwMWlv6mw+rWnpm6nSkiIjL4KJhlS9l4OOp8WP5bCLUPKNs6ltkHu/UycxERkcFGwSybZl4DDXthzZ/bVpUW5lE5tJCfvPAef3hzC7abl56LiIjIwKNglk3BM6DiGHjz1x2Gzvj9/JOZOraUmx9fzdW/fYvddc1ZrKSIiIhkioJZNhkDM+bD9pXw4Zttq0eVFvDQF2Zyy8eP49X393DuT17muTU7slhRERERyQQFs2ybPA/yS5xWszgul+Gq04I8ff1pjC4t4NqHlvGNx1ZyoCmUpYqKiIhIX1Mwy7b8Iph2Kby9EA5s77J5wrBiHr/uI1x/1gSeWLGV83/yCq9/sDcLFRUREZG+pmCWC066GqIRWPa/CTd73S6+Pudo/nTtKeR5XHzuvjf4/lNv0xSKZLiiIiIi0pcUzHLB0CPgyI/CW/8L4ZZui00bV8bT15/G52eO575XN3LhL15lzdbaDFZURERE+pKCWa6Y8UWo3wVvP3HIYoV5Hv7r4ok8cOVJ1DSE+MQvX+OuRe8TiWpYDRERkf5OwSxXHHEWlB+R8P2Zicw+ehh/u2EWc44fwYK/vctnf/06m/fqbQEiIiL9mYJZrnC5YMY1sPUt2LosqV3K/Hn84pJp/HTeVN7bWcf5P32F3y/RoLQiIiL9lYJZLpn6OcgrgiX3JL2LMYaLpo7mb1+bxQnjyvjWX1bzhd++xa66psPvLCIiIjlFwSyX+IbAlEtg7eNwcHdKu44sKeDBq2Zw6wXH8dr7ezj3xy/z7Oquw2+IiIhI7vJkuwLSyYxrYOm98IsTweNLYgfTNucCrgA+P8RS2xgi/KcotQs9FPs8uIzpfn9PHrjzwdM6+cCd53x68hIuj9+yDV5b1b5Ph/1bj9FpnbcQfCXOvIiIiHShYJZrKo6C8xfArrWHLneI58g8QJmFtdtqeHvbAQqsm5nBcoYPSRD0bBQiIQg3OUN1hJsg0gJNNe3L4WaINHcoE8TCph7+jB6fE9ASTqWHX+fJ6+EXi4iI5LacDGbGmIuBucAw4C5r7fNZrlJmzbym14dwAZOAyIc1fP3RFWxYU88XTgty47lH4/O6e3dwa3lp0T8449SZcWGu2QlwrVPbclP7ulADNNV2nRr2wb6NseUaiIYP/f2tLW+Hmtx5zqC9Ngo2Epu3zryNxpbj57vbFu10jGiXbcfv2we77gfjBpcnNrljk6fjul6X8TrLbq8z745b745ta5vvVNblcTqZiIhIzspYMDPG3A98HNhlrZ0Yt/484KeAG7jPWnu7tfYJ4AljTBnwP8DgCmZpNHVsKU9ffzo/eHYdv3l1Iy+v382P/20qE0eX9PygxmBdHsgvhnTflbS2+wDXGtyaaqGxpn3dwV2w5732ZZvEGxGMKxaA3O3zxuUEl7b5+G2mPTR12GYoaKyD3bVOoIzGAlw07Ew2Erc+3D5li3ElF+LcXvAWOK2b3X76wFPQ9lmxawO809BlfZdPt9c5nyIi0kUmW8weAH4BPNi6whjjBu4CPgpUA0uNMU9aa9+OFflObLv0QkGem/970UTOOXY4N/5pJRff9Ro3nHMk155xBB53jrWgGAN5fmcaMir1/a2FloNO+DlU+EqjtxYvZvbs2antFI12DGq2U6CLdhPoWqdICKIhZ3vrfKR1eyi2LhI3n0rZcHtrZ3Md1O+GUKOzHP9Jx9vpxwO8neiH7cS4ugY1l6djWOy83FqmdTpU2e72dXtj31vgtLp64+d9cesKnX1FRLIgY8HMWvuyMaay0+oZwPvW2g0AxphHgIuMMeuA24FnrbXLM1XHgW7WURX87YZZfHfhWv7n+fW8+M4ufvTZqVQG/Ekfw1qLtRCJWlrCUSzOctRaotbZ3vrZYX3sH3GPy4XHbfC4DB6XC6/bYNLZemKM05KX61wucOUB/fR5OWudZxFDjbEQ18ib/3yZGdMmQagJwo2H+YwLeW3BMdwxMEYjzq3yaEOnYBlOsNwpsPaWO69jgGsLdAUdA1w34W7Yzk2wri62HCuX548rW+h0glHLoYh0YjI5GGksmD3VeivTGPNp4Dxr7dWx5cuAmcB64HJgKbDCWnt3N8e7BrgGYPjw4Sc+8sgjfVr/gwcPUlRU1KffkSlvbAvz4NvNNEfA54HWNzpZC1GA2Ke1TrtI62dfMIDbBR4DLuPMu43BHZt3GWeb22Wc7a1Tp3Kt613GKRf/b56JTZiu823bTeKyYDDdlTUQCbVQmJ+H22Vi9Wz/eTwup36eWP2cQNpe/0Nt97g4RG/a3JNTvx82grFRXNEwxkZiUxh3pAVXtBlXtBl3pBlXtCX22ZzkZ3z5lg7bTYq/IRZD1JVPxO1MUZcv9hm/Lp+I2xf7zIubjy/ni00Fse2+2Gdu3DLOqesiy3QuHDoPjjPPPHOZtXZ65/XZfvg/0V8Na639GfCzw+1srb0HuAdg+vTpNuXbSSla3JNbVjlqNnB5bSMP/HMTzaFoLHg4gcblMrHw0R5wXMa0L2PYvHkjR1RVAR2DUGs5A12OZYFIJEo4ap2pdT5iCUWjRCLO+lAkSiRqCUUs4Wi0rWyHdZH2Y4SilsZI+37hiCUStW2tedAeLomtc5Zte+iMzZNoG9H2YBpr/WtdjlqLtQZIQytNAsaA1+0iz+20NHrdLrwug8fttDZ63S688dvcrS2R7ds9btNxf7cLj8t0KuMiz+0c1+MyuF0m5X/T1217h2NHT0jt58Pg87opyvdQ5PM4n7H5Qq8blyv7wSIp1rZ3cAk1suS1RcycNjm27KyjJW4+1IAJNeAONeIONXTZ5kz7oLl1W2x9KuHPuJ0Bq/P8kFcY+yxqf1TA62+fb9t2mHLewpR7RQ+kv5u9pXPh0Hk4tGwHs2pgbNzyGDC2JVsAABWDSURBVGBbluoy6IwsKeDm84/t0b6LF29l9uzU/hEeqF5ctIhTT59FKGIJhaOEolEnQEaihCJRWsJOmOw8H4qFyQ7zYSeItkSc8BmKRDvMt5YLx89H24/TFIoSjoRpibQH35ZwNPad7eVag23arVqZtkMZA/48J6j5890U+bwU5cdCXH5s3ufBn++hOBbm/HldA15RvocCrzvhLfNo7H8Sotb5jHSerCUSiX1Go0SiEI5GibZ+2tj/CNj4/TysrR+BaRpDYZ6bwgI3/nyPM5/nwd3TsGlt+y3gtjBXHwt99c6zlS31Caa49aEG55nB/Zs6bkulQ4rL0+l2bqET6OKX227fFjJ+6y5nzMPW7Xlx273+9tvB8bd69YyfDGLZDmZLgSONMUFgKzAP+Fx2qySSGpcx5Hvc5HtIfy/VPmRtewtkKGxjgTLa1uKYqiVLljBz5syU9olaS2MoQn1zhIPNIeqawm3zB5sjHGwKc7A5RH1zhLrmMPXNYfbUNXCwOdw2JVNXl4ECr5uohYi1bYGsTy17PeFqn9eFP89DQZ4bf56HwvzYZ54T4Jz1Tojz5zufhZ2W/flD8OeVU1DgpjDPjc/Ty9bFcEuCYHcwFgDjAlyoUwtefCtfS4PTWzp+W0sDwUhz6mMeev1QNAyKR0DRcGcqHg5FI9o/i4ZD4VANASMDTiaHy/gDzh20gDGmGvietfY3xpgvA3/DGS7jfmvtYUZWFZF0MMaQ5zHk4UpLH4RNfldKHUnSwVpLczgaC3ROUIufb5uawjSGIrFnGF2x5wBduI3B4za4jIlbB253bJvL4HJ1+oytd3c3GcPSZcs4duIU6pvDNLREqG8J09jiBNCGljD1LWEamiNt2xpaIuw52ExDi7Pd+Uxi2Jc4Pq+LAq+bAq8bX54T2Aq8bnze9vmCvI7Lvti69uV8CryFFOaNpKDI5Wz3OmEw3+PqUfhbvOgfzP7IzEPc1q3v1ArYGBsGZ6cz7VwLH7wIzQe6HtzlAf+wWFhrDXAjEs9rYGrpJzLZK/OSbtY/AzyTqXqIyMBhjPOMms/rpqI4d5or93/g5tQJgV4dIxqNtSbGhTgn1EVoaI59xgJfYyjS4bMhFKEptlzXFGZ3XXN7mdj2nrSK+vPcCW8Vt95O9sfWtc3ne/hgH5TvDlOUX0iRbwhF/u5vLR9SS0N7WKvbEfe5Cw7ugNqtsHUZ1O8h4bN4BeUJWt6GO8PyDBkDJWOcVjrdRpUsy/atTBERScDlMvhjAYc+GAEmFInS0BKhqTXMxYJcUyhuPvbZ0BKhsSXMweZIe8tkKreWl77W8WczdHw2MBbiimPPCRb7vJQUeCkp8FBamEdJgZchBV5KCysoKR9FyWgv3u7GYIyEnefoDu6Aup3O58FdHcPc3n866yMtHfd1eToGtZLRzmf8sq80J3q7ysClYCYiMgh53S5KClyUFHjTdsxEt5ZfeeMtjjx2Ypdby63z8begd9Q2tc0fbD50hwR/njsusLUGOW9ckCunpGA4pcUnUjKsdZuXYp/X6YBhLTTuhwPb4MBWqP3QaXWrrXaWP3wD1m7r2jHC6+8+tJWMdYKdtyBt51QGHwUzERFJi0S3lvcMdTP7uOEpHysUiXKgMURtbKppDLUt1zTErW9w1m/a00BNYwu1jSGaQtFDHrvY52kLcxVF+YwqHcWo0iMYNdTHqKoCRpUWMKLEh9dYp7WtS3Crdj53rIH6XV2/oDAQF9RiAa5kDIX1B50WPbf+6ZXu6eoQEZGc43W7GFqUz9Ci1J8dbApF2kNcY4jahrj5WMCraWihpjHEzgPNrPiwhv0NHcciNAaGF/sYWepjVGkBo0uPYVTJVEaOK2B0qRPeygq9mEhLLLh1Cm21W2HvB7DhJWipA5xX3fCvG2HYsTBiEoyY7HwOP75/vLFEMkLBTEREBpTWVrthQ3xJ79PQEmZbTRPbahrZXtvI1rj5t7cd4O9v76Ql3LElzud1MarECWmjSocwqvRERpWcxqiRBYyKBTqf1+30Mq3ZwrrFf+bYshDsWA3rnoLlD7YfrCzYMayNmOi0tul5tkFHwUxERAa9wjwPE4YVMWFY4lcFWWvZW9/C9pomttY0toW2bbHlxe/uZvfBZjq/5bDcn+eEtJICqD+Fk8ccxZjpBYwtK2Csp4aimndgxyonrO1YDeuebN+5oKxTy9pEqDga3Ol7LlByj4KZiIjIYRhjCBTlEyjKZ9KYkoRlWsJRdh6ID27t85v21rN5T5jnN7/dYZ/SQi9jyk5mTOlZjD2igMqplmPMZsa0fMDQuvV4dq2Bpfc5b30AcOdBxTFxLWuxW6EFpX19CiRDFMxERETSIM/jYmx5IWPLCxNuX7RoEVNmnMqH+xqo3t/Ih/sbqN7vzL+3q45F7+6iue12aRVQxVD/hYwrzeOEor1M9mxhQnQjIxvfY8i7z+JZ8VD7wUvHtYe1YcdBeRBKx4NvSJ//3JJeCmYiIiIZYIyh3J9HuT+PKWO7tnBZa9l9sJnq/Y1OcIsFuOr9Dby4t5zf7S+gJXIkMAewVFDDR/zbmO6r5vjIFio3raLsnacx8QPsFpRDWWVsGt8+Xzre6S2q26I5R8FMREQkBxhjGFbsY1ixjxPGlXXZHo1adtU1U72/wWlt2+e0uj27v5F79zey7UAj3mgjE8w2xrl2ceKQA0wq3M+40G7Kq5fjWfckJn5cNuN2wlnnwFYWdOYLy9X5IAsUzERERPoBl8swosTHiBIf0yvLu2wPR6LsrGvmvZ11LN9Sw6It+/nRlpq2wXqH+z2cOTbMqUPrmOivYSy78NRuhprN8O6zzhsT4uUVdQpsle0tb6XjNJBuH1EwExERGQA8bhejS51x1mYfPQyASNSyfmcdyzbvZ/mW/byxeT+PvAdQjNc9juNHnc2J48s4cWYZJ47IY3h0J+zf5IS1/Zucae8H8P4/INzY8QuLRkBwFsz+Jgw9IrM/7ADW74OZMeYC4IIJEyZkuyoiIiI5xe0yHDtyCMeOHMLnTx4PwJ6DzfxrS40T1jbv56E3NvObVzcCMLq0gBPGj+SEccdy4qQyjh05xHkvqbVOi1prWNu/GfZ9AG8vhLWPwwn/DmfcBMUjsvfDDhD9PphZa/8K/HX69Onzs10XERGRXBcoyuejxw3no7FXZbWEo6zbfqCtVW3Zpn38deU2wBlEd/KYUqdVbVwZJ4yfSvnYGe0HO+c2ePlOWPYArHwETv4P+Mj1Gr6jF/p9MBMREZGey/O4mDK2lCljS7mKIADbaxtZvtlpVVu2ZT/3vbKBX0Wc3p7BgJ8TxpVxwvhSTj0iQOXcH8IpX4IX/x+88kN463447eswY76eQ+sBBTMRERHpYGRJAXMnFzB38kjAef/o6q21bbc/X1q/iz8vrwbg+FFDmDt5JHPP/BnjT/0q/OM2+Pt3YcndzvNnUz6nF7enQGdKREREDsnndXNSZTknxXqDWmvZvLeBF9bt5OnV27nzuXe587l3mTS6hLmT7+STkzcybMkP4MmvwD9/Dmd9F469QMNvJEHBTERERFJijKEy4Ofq06u4+vQqqvc38OzqHTy1eju3P/sOtwNTxtzGlye9w+zqu/E+dhmMPhHOuTXLNc99CmYiIiLSK2PKCpk/q4r5s6r4cF8DT6/eztOrtjN/6Ujc3MLXKpZxxZ4/UPTbC5hcNg2O/imMnJLtauckBTMRERFJm7HlhVx7xhFce8YRbN5bHwtpZfx891Quc/+d6/cvhF/PovHoiymYc4vGQOtEwUxERET6xPihfq6bPYHrZk9g4556nl41kX/751nMbXqSq955lvC7f+X9MZ8iMPe7BEaOy3Z1c4KCmYiIiPS5YMDPl886komurYw57pf8ftlqhv/rp5z34Z8J3b2Qx4s/QcuML3P2tKOoKM7PdnWzRsFMREREMmrCsCImnH8KnH8Km9avoun5/+KTex5h/z+e4pfPX8T6cfOYM2U85x0/gqFFgyukKZiJiIhI1lQeNRmO+iN22wrynr2Fb3/4MLu2P8eCzZ/itoWzmFE1jLmTR3LBlFEU5Q/82OLKdgVEREREzKip+L/wJFz+VypGVbLAew+vl3yXyt0vcvPjqzjtjhf5xYvvcaAplO2q9ikFMxEREckdwVmYq/8Bn/0dQws9fL/5dtaOXcCnh+/kf55fz2m3v8hPXlhPbePADGgKZiIiIpJbjIHjLoTr3oALf46/aSff2XE9b520iFmVhfzkhfc47fYX+dHz71LT0JLt2qaVgpmIiIjkJrcHTvh3+NKbcOIVBFbfyy9qvsRLnzKcflSAn734Pqfe/iJ3PvcO++oHRkBTMBMREZHc5hsCH/8xXPE0uDyMf/oSfll0P3//j8mcdexwfvXSB5x2x4v84Jl17DnYnO3a9oqCmYiIiPQPlafBf7wGp94AK/7AkX88m59P+ZC/f20Wc44bzr2vbOC0O17k+0+9za4DTdmubY8omImIiEj/4S2Aj94G81+EomHw2GVMWHQdP5k7ihe+fgZzJ43if/+5idPvXMStT65lR23/CmgKZiIiItL/jJoK8xfB2bfA+r/BXTOoql7IDz8zmRe/cQYXTx3NQ29sZtadi/juE2vYWtOY7Ronpd8HM2PMBcaYe2pra7NdFREREckktxdO/4Zze3PYsbDwOvjdJxjv2s0dn57Mov8zm0+dOIZHlm5h9oJF3Pz4aj7c15DtWh9Svw9m1tq/WmuvKSkpyXZVREREJBsCR8IVz8DcH0L1UvjlKfD6Lxlbms8PPjmJxTeeybyTxvHnZdWc+T+LuelPq9i8tz7btU6o3wczEREREVwuOOlqZ+yzytPgbzfDb+bArnWMLi3gvy6eyMv/eSafP3k8T6zYylk/fIlvPLaSjXtyK6ApmImIiMjAUToWPvcYfPI+2LcB7j4dFt8O4RZGlPi49cLjeeU/z+TKj1Ty9OptnP3DxXzt0RW8v+tgtmsOKJiJiIjIQGMMTP4MfHkpHH8xLP4B3HMGVL8FwLAhPr7z8eN49aazmH96Fc+t2cFHf/wSX/nDv1i/sy6rVVcwExERkYHJH4BP3QeXPAqNNXDfOfDct6DFuX0ZKMrn5o8dy6s3ncl/nHEEL67byZwfv8yardnrUOjJ2jeLiIiIZMLR58H4j8ALt8Ibd8E7T8EFP4UjzgRgaFE+/3neMVwzq4qFK7Zx/KghWauqWsxERERk4PMNgY//yOm96fLA7y6GhV+Cxv1tRUoL87j8I5UYY7JWTQUzERERGTwqT+3wWifumglvP5ntWrVRMBMREZHBJcFrnXj081C3I9s1UzATERGRQarttU7fg/XPw10z4F8PgbVZq5KCmYiIiAxebi+c/vXYa52Oc547274ia9VRr0wRERGR1tc6bX4VRk3LWjXUYiYiIiICzmudgrOyW4WsfruIiIiItFEwExEREckRCmYiIiIiOULBTERERCRHKJiJiIiI5AgFMxEREZEcoWAmIiIikiMUzERERERyhIKZiIiISI7o98HMGHOBMeae2trabFdFREREpFeMzeIb1NPJGLMb2NzHXxMA9vTxd/QXOhftdC4cOg/tdC7a6Vy007lw6Dw4xltrKzqvHDDBLBOMMW9Za6dnux65QOeinc6FQ+ehnc5FO52LdjoXDp2HQ+v3tzJFREREBgoFMxEREZEcoWCWmnuyXYEconPRTufCofPQTueinc5FO50Lh87DIegZMxEREZEcoRYzERERkRyhYJaAMeY8Y8y7xpj3jTHfTLDdGGN+Ftu+yhhzQjbq2deMMWONMYuMMeuMMWuNMV9NUGa2MabWGLMiNt2Sjbr2NWPMJmPM6tjP+FaC7YPlmjg67r/1CmPMAWPMDZ3KDNhrwhhzvzFmlzFmTdy6cmPM340x78U+y7rZ95B/V/qbbs7FAmPMO7Hfgb8YY0q72feQv0/9TTfn4lZjzNa434OPdbPvgLkuujkPj8adg03GmBXd7DugrolesdZqipsAN/ABUAXkASuB4zqV+RjwLGCAk4El2a53H52LkcAJsfliYH2CczEbeCrbdc3AudgEBA6xfVBcE51+ZjewA2csnkFxTQCzgBOANXHr7gS+GZv/JnBHN+fqkH9X+tvUzbmYA3hi83ckOhexbYf8fepvUzfn4lbg/xxmvwF1XSQ6D522/xC4ZTBcE72Z1GLW1QzgfWvtBmttC/AIcFGnMhcBD1rHG0CpMWZkpiva16y12621y2PzdcA6YHR2a5WzBsU10cnZwAfW2r4e2DlnWGtfBvZ1Wn0R8NvY/G+BixPsmszflX4l0bmw1j5vrQ3HFt8AxmS8YlnQzXWRjAF1XRzqPBhjDPBZ4A8ZrVQ/pGDW1Wjgw7jlarqGkWTKDCjGmEpgGrAkweZTjDErjTHPGmOOz2jFMscCzxtjlhljrkmwfdBdE8A8uv8jOxiuiVbDrbXbwfmfGWBYgjKD8fq4CqcVOZHD/T4NFF+O3da9v5tb3IPpujgd2Gmtfa+b7YPlmjgsBbOuTIJ1nbuuJlNmwDDGFAF/Bm6w1h7otHk5zq2sKcDPgScyXb8MOdVaewJwPvAlY8ysTtsH2zWRB1wI/DHB5sFyTaRisF0f3wbCwMPdFDnc79NA8CvgCGAqsB3nNl5ng+m6uIRDt5YNhmsiKQpmXVUDY+OWxwDbelBmQDDGeHFC2cPW2sc7b7fWHrDWHozNPwN4jTGBDFezz1lrt8U+dwF/wbkFEW/QXBMx5wPLrbU7O28YLNdEnJ2tt61jn7sSlBk014cx5nLg48ClNvbwUGdJ/D71e9bandbaiLU2CtxL4p9xUFwXxhgP8Eng0e7KDIZrIlkKZl0tBY40xgRjrQLzgCc7lXkS+PdYT7yTgdrWWxkDSeyZgN8A66y1P+qmzIhYOYwxM3Cuqb2Zq2XfM8b4jTHFrfM4Dziv6VRsUFwTcbr9v9/BcE108iRweWz+cmBhgjLJ/F3p94wx5wE3ARdaaxu6KZPM71O/1+kZ00+Q+GccFNcFcA7wjrW2OtHGwXJNJC3bvQ9yccLpYbcep7fMt2PrrgWujc0b4K7Y9tXA9GzXuY/Ow2k4zeqrgBWx6WOdzsWXgbU4vYneAD6S7Xr3wXmoiv18K2M/66C9JmI/ayFO0CqJWzcorgmcMLodCOG0dnwBGAr8A3gv9lkeKzsKeCZu3y5/V/rz1M25eB/nmanWvxd3dz4X3f0+9eepm3Pxu9jfglU4YWvkQL8uEp2H2PoHWv8+xJUd0NdEbyaN/C8iIiKSI3QrU0RERCRHKJiJiIiI5AgFMxEREZEcoWAmIiIikiMUzERERERyhIKZiIiISI5QMBMRERHJEQpmIiIiIjni/wMufWpYwGFihwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(dpi=300)\n",
    "pd.DataFrame(history.history).plot(figsize=(10, 5),logy=True)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build FFNN model with batch normalization layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 8)]               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 8)                 32        \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 30)                270       \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 30)                120       \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 30)                120       \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 30)                120       \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 30)                930       \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 1)                 31        \n",
      "=================================================================\n",
      "Total params: 3,483\n",
      "Trainable params: 3,287\n",
      "Non-trainable params: 196\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "initializer_name = \"glorot_normal\"\n",
    "\n",
    "input_layer1 = layers.Input(shape=(8,))\n",
    "batch_layer1 = keras.layers.BatchNormalization()(input_layer1)\n",
    "dense_layer1 = layers.Dense(30, activation=\"sigmoid\", kernel_initializer=initializer_name, input_shape=(8,))(batch_layer1)\n",
    "batch_layer2 = keras.layers.BatchNormalization()(dense_layer1)\n",
    "dense_layer2 = layers.Dense(30, activation=\"sigmoid\", kernel_initializer=initializer_name)(batch_layer2)\n",
    "batch_layer3 = keras.layers.BatchNormalization()(dense_layer2)\n",
    "dense_layer3 = layers.Dense(30, activation=\"sigmoid\", kernel_initializer=initializer_name)(batch_layer3)\n",
    "batch_layer4 = keras.layers.BatchNormalization()(dense_layer3)\n",
    "dense_layer4 = layers.Dense(30, activation=\"sigmoid\", kernel_initializer=initializer_name)(batch_layer4)\n",
    "output_layer1 = layers.Dense(1, kernel_initializer=initializer_name)(dense_layer4)\n",
    "\n",
    "model = Model(inputs=input_layer1, outputs=output_layer1)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Configure and run training as above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mean_squared_error\", optimizer=keras.optimizers.SGD(lr=1e-3))\n",
    "history = model.fit(X_train, y_train, batch_size=10, epochs=20, validation_data=(X_validate, y_validate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot (improved) trainin history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1800x1200 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAEvCAYAAACHYI+LAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUZfrG8e+ZSe+BJARIQoDQEyChI1WRsogVCyCCrr2uu7rquqtbLKvsrq77szdQqWKlWFApIiglEHqH0EuA9J6c3x+HKi1lkjNh7s91zUUy5cyTIzJ33vc9z2uYpomIiIiIVJ/D7gJERERELhYKViIiIiIuomAlIiIi4iIKViIiIiIuomAlIiIi4iIKViIiIiIu4mV3AQARERFmfHx8jb5HXl4egYGBNfoedYXOxUk6Fxadh5N0Lk7SuThJ58Ki82BZvnx5hmmakWd7zC2CVXx8PMuWLavR95g3bx79+vWr0feoK3QuTtK5sOg8nKRzcZLOxUk6FxadB4thGOnnekxTgSIiIiIuomAlIiIi4iIKViIiIiIu4hZrrERERKT2lJSUsHv3bgoLCyv1utDQUNavX19DVbkfPz8/YmJi8Pb2rvBrFKxEREQ8zO7duwkODiY+Ph7DMCr8upycHIKDg2uwMvdhmiaHDx9m9+7dNG3atMKv01SgiIiIhyksLKR+/fqVClWexjAM6tevX+lRPQUrERERD6RQdWFVOUcKViIiIlLrgoKC7C6hRihYiYiIiLiIRwSrVbsz+XF3id1liIiIyK+Ypsmjjz5KYmIiSUlJTJ06FYB9+/bRp08fOnbsSGJiIj/++CNlZWWMHTv2xHNfeuklm6s/k0dcFTgjbS8T1hXzWGk5Pl4ekSVFRETqhE8//ZSVK1eSlpZGRkYGXbp0oU+fPkyaNIlBgwbx5JNPUlZWRn5+PitXrmTPnj2sWbMGgMzMTJurP5NHBKvkuHDe/nE76/Zl0zE2zO5yRERE3MbfZqxl3d7sCj23rKwMp9N5wee1bRTC08PaVeiYCxcuZMSIETidTho0aEDfvn1ZunQpXbp04bbbbqOkpISrr76ajh070qxZM7Zt28YDDzzA0KFDGThwYIXeozZ5xPBNSlw4AKnpR22uRERERE5lmuZZ7+/Tpw8LFiygcePGjB49mg8++IDw8HDS0tLo168fr776KrfffnstV3thto5YGYYxDBiWkJBQo+8THepHPT+DFbvcb8hQRETEThUdWYKaaRDap08f3nzzTcaMGcORI0dYsGAB48aNIz09ncaNG3PHHXeQl5dHamoqv/nNb/Dx8eG6666jefPmjB071qW1uIKtwco0zRnAjM6dO99R0+/VPMyhESsRERE3c80117B48WI6dOiAYRi8+OKLREdHM2HCBMaNG4e3tzdBQUF88MEH7Nmzh1tvvZXy8nIAnn/+eZurP5NHrLECaB7qZOn+Ag5mFxIV4md3OSIiIh4tNzcXsJpwjhs3jnHjxp32+JgxYxgzZswZr0tNTa2V+qrKI9ZYASSEWT9q6k5NB4qIiEjN8JhgFRfiwNtpsGKXpgNFRESkZnhMsPJxGrRrFMqKdI1YiYiISM3wmGAFVtuFVXsyKSkrt7sUERERuQh5VLBKjgujsKScDfty7C5FRERELkIeFaxSmhxrFLpT66xERETE9TwqWDUK9SMq2JcVClYiIiJSAzwqWBmGQUpcuFouiIiI1CFBQUHnfGzHjh0kJibWYjXn51HBCqx1VjuP5JORW2R3KSIiInKR8bhgdXyd1QqNWomIiNjiscce47XXXjvx/V//+lf+9re/cdlll5GSkkJSUhJffPFFpY9bWFjIrbfeSlJSEsnJycydOxeAtWvX0rVrVzp27Ej79u3ZvHkzeXl5DB06lA4dOpCYmMjUqVNd8rN5zJY2xyU1DsXLYbBi51Eub9vA7nJERETs9dXjsH91hZ7qX1YKzgpEh+gkGPLPcz5800038bvf/Y57770XgGnTpvH111/z8MMPExISQkZGBt27d+fKK6/EMIwK1Qbw6quvArB69Wo2bNjAwIED2bRpE2+88QYPPfQQo0aNori4mLKyMmbPnk2jRo2YNWsWAFlZWRV+n/PxuBErP28nbRuF6MpAERERmyQnJ3Pw4EH27t1LWloa4eHhNGzYkD/96U+0b9+eAQMGsGfPHg4cOFCp4y5cuJDRo0cD0Lp1a5o0acKmTZvo0aMHzz33HC+88ALp6en4+/uTlJTEd999x2OPPcaPP/5IaGioS342jxuxAkiODePj5bspLSvHy+lx2VJEROSk84ws/VpBTg7BwcEuedvhw4czffp09u/fz0033cTEiRM5dOgQy5cvx9vbm/j4eAoLCyt1TNM0z3r/yJEj6datG7NmzWLQoEG88847XHrppSxfvpzZs2fzxBNPMHDgQJ566qlq/1wemSpSmoSTX1zGxgNqFCoiImKHm266iSlTpjB9+nSGDx9OVlYWUVFReHt7M3fuXNLT0yt9zD59+jBx4kQANm3axM6dO2nVqhXbtm2jWbNmPPjgg1x55ZWsWrWKvXv3EhAQwM0338wjjzxCamqqS34ujxyxSok7uYC9XSPXDP2JiIhIxbVr146cnBwaN25Mw4YNGTVqFMOGDaNz58507NiR1q1bV/qY9957L3fffTdJSUl4eXkxfvx4fH19mTp1Kh999BHe3t5ER0fz1FNPsXTpUh599FEcDgfe3t68/vrrLvm5PDJYxYT7ExHkQ+rOo9zcvYnd5YiIiHik1atPLpqPiIhg8eLFZ31ebm7uOY8RHx/PmjVrAPDz82P8+PFnPOeJJ57giSeeOO2+QYMGMWjQoCpUfX4eORVoGAbJceFquSAiIiIu5ZEjVmA1Cp2z7gBH84oJD/SxuxwRERE5j9WrV5+44u84X19ffvnlF5sqOjuPDVYn1lntOsqlrdXPSkRExJ0lJSWxcuVKu8u4II+cCgRoHxOK02FoOlBERDzSuVoTyElVOUceG6wCfLxoHR2sRqEiIuJx/Pz8OHz4sMLVeZimyeHDh/Hz86vU6zx2KhCsdVafr9hLWbmJ01HxlvkiIiJ1WUxMDLt37+bQoUOVel1hYWGlg0Zd5ufnR0xMTKVe49HBKiUunI9+3snmgzm0jg6xuxwREZFa4e3tTdOmTSv9unnz5pGcnFwDFV08PHYqEE5vFCoiIiJSXR4drJrUD6BeoA+p6VpnJSIiItVna7AyDGOYYRhvZWVl2fX+JMeGsWKXRqxERESk+mwNVqZpzjBN887QUPv260uOC2PLwVyy8ktsq0FEREQuDh49FQinNwoVERERqQ6PCVZeJTlnvb99bBgOQwvYRUREpPo8I1h98ySdlv8BSgrPeCjI14uWDdQoVERERKrPM4JVwgD8Cw/AL2+c9eHkuHBW7sqkvFwdaEVERKTqPCNYNe9PRv0usOBfkHtml9mUuDByCkvZeijXhuJERETkYuEZwQrY2nwslBbA3GfPeCxZjUJFRETEBTwmWBUExEDn30LqBDiw9rTHmkUEEurvrXVWIiIiUi0eE6wA6Pc4+IbAN0/CKTt6OxwGyXFhGrESERGRavGsYBVQD/o+BtvmwuZvT3soOTacTQdzyC5Uo1ARERGpGs8KVgBdbod6za1Rq7KTISqlSRimCat22bO9joiIiNR9nhesvHxg4DNweDMse+/E3R1iwzAMtM5KREREqszzghVAqyHQtA/Mex4KrCAV4udNi6ggBSsRERGpMs8MVoYBg56DgkyY/+KJu5Njw1mxMxPTVKNQERERqTzPDFYA0UmQMhqWvAUZWwBrnVVWQQnbMvJsLk5ERETqIs8NVgD9/wxefjDnKUCNQkVERKR6PDtYBTeA3r+HjbNg+wISIoMI9vXSOisRERGpEs8OVgDd74PQOPj6Tzgop6MahYqIiEgVKVh5+8Hlf4UDq2HlRJLjwtm4P5vcolK7KxMREZE6RsEKoN21ENsNvv8HXRp6UW7Cqt0atRIREZHKUbCCY+0Xnoe8g3TZ/QGgBewiIiJSeQpWx8V0gqQb8Fv6Gj3q57FCC9hFRESkkhSsTjXgaTAc/NE5hVQ1ChUREZFKUrA6VWgM9HyA5Ozvic9fQ/rhfLsrEhERkTpEwerXLnmIkoAo/uL9ESt2Hra7GhEREalDFKx+zTcI54CnSXZsoTRtut3ViIiISB2iYHUWjo4j2e6dQN9dr0GxpgNFRESkYhSszsbh4JeWjxBVfojiha/YXY2IiIjUEQpW5xCZeClflXXBuei/kL3P7nJERESkDlCwOofkuHCeLx0JZSXwwzN2lyMiIiJ1gILVOdQL9MFZvxnfhVwDKyfC3pV2lyQiIiJuTsHqPJJjw3gu9wrMgHrwzZOghqEiIiJyHgpW55HcJJz0PC+Odn0U0hfChpl2lyQiIiJuTMHqPJJjwwBYGPIbiGwD3/4FSotsrkpERETclYLVebSODsbf20nq7lwY9Awc3Q5L3rK7LBEREXFTClbn4eV00D4mlBU7j0LCAEi4HOaPg7wMu0sTERERN6RgdQEpTcJZuzebwpIyGPQsFOfCvOftLktERETckILVBSTHhlFabrJmTxZEtoLOt8Gy9+HgBrtLExERETejYHUByXHhAKTuPGrd0e8J8AmCb5+0sSoRERFxRwpWFxAZ7EtsPX9W7My07gisD33/CFu+g83f2VuciIiIuBUFqwpIiQsndedRzOMNQrveCfWaWaNWZaX2FiciIiJuQ8GqAlLiwjmQXcS+rELrDi8fuPwfcGgDLH/f3uJERETEbShYVUBynNUo9MQ6K4DWQyG+N8x9DgoybapMRERE3ImCVQW0aRiCr5fj5DorAMOAQc9BwVFYMM6+4kRERMRtuDxYGYZxtWEYbxuG8YVhGANdfXw7eB9rFHraiBVAw/aQPAp+eRMOb7WnOBEREXEbFQpWhmG8ZxjGQcMw1vzq/sGGYWw0DGOLYRiPA5im+blpmncAY4EbXV6xTVLiwlm7J5ui0rLTH7j0L+D0gTlP2VOYiIiIuI2KjliNBwafeodhGE7gVWAI0BYYYRhG21Oe8udjj18UkuPCKC4rZ+3e7NMfCI6G3g/Dhpmw/Ud7ihMRERG3UKFgZZrmAuDIr+7uCmwxTXObaZrFwBTgKsPyAvCVaZqpri3XPicahaYfPfPBHvdDaCx88ycoLzvzcREREfEIxoneTBd6omHEAzNN00w89v1wYLBpmrcf+3400A3YBIwBlgIrTdN84xzHuxO4E6BBgwadpkyZUq0f5EJyc3MJCgqq1jH+MC+f5mEO7u3od8ZjUQcW0Hb9v9nQ6gH2NxxQrfepaa44FxcLnQuLzsNJOhcn6VycpHNh0Xmw9O/ff7lpmp3P9phXNY5rnOU+0zTNV4BXLvRi0zTfAt4C6Ny5s9mvX79qlHJh8+bNo7rv0WNvKqnpR89+HLMvvLuA1num0frax8HXff/iueJcXCx0Liw6DyfpXJykc3GSzoVF5+HCqnNV4G4g9pTvY4C91SvHvaXEhbM3q5D9xxuFnsowYNDzkHsAJt0Iy8dD1p5ar1FERETsU51gtRRoYRhGU8MwfICbgC9dU5Z7Ot4odMWv2y4cF9vF6siemQ4zHoKX2sLrl8B3f4X0Rdr+RkRE5CJXoalAwzAmA/2ACMMwdgNPm6b5rmEY9wPfAE7gPdM019ZYpW6gXaNQfLwcrNiVyZCkhmd/0iUPQs8H4NBG2PytdVv0P1j4EviFQvNLocVASBgAQVG1+wOIiIhIjapQsDJNc8Q57p8NzHZpRW7Mx8tBYqOQs18ZeCrDgKjW1u2SB6EwG7bNPRa05sDaz6znNUq2QlaLgdbXDmfN/xAiIiJSY6qzeN0jpcSF8+HP6RSXluPjVcGZVL8QaHuVdTNN2L/qZMhaMA7mvwAB9a1RrBYDrVGtgHo1+4OIiIiIy9karAzDGAYMS0hIsLOMSkmOC+edhdtZvy+bDrFhlT+AYUDDDtatz6OQfwS2/mAFrS3fwaqpYDggpgu0uNwKWtHtrdeJiIiIW7M1WJmmOQOY0blz5zvsrKMyUpqcXMBepWD1awH1IGm4dSsvg70rTq7N+uEZ6xYUfTJkNetnjYCJiIiI29FUYCU1DPUnOsSP1J2ZjL3ExQd3OCGms3Xr/yfIPWiNYm3+FtZ9CSs+BIcXxPU4uTYrqrWLixAREZGqUrCqgpQmYaSeq+WCKwVFQceR1q2sFHYvObk2a85frFuTXtD3UWjaV9OFIiIiNqtOHyuPlRwbzu6jBRzMOUuj0Jri9IImPWHAX+Gen+DhdTDoOTiyDT64Ct69HDZ9Yy2OFxEREVsoWFXByXVWmfYVEdoYetwHD62EK1461vH9BnizjzVtWF5uX20iIiIeSsGqCto1CsXbadgbrI7z8oXOt8EDqXDVa1CcB9NGw+s9YNXH6vYuIiJSixSsqsDP20nbRqG1s86qopzekDwK7l8K170LGPDp7fBqF0j9EMpK7K5QRETkoqdgVUUpcWGs2p1JaZmbTbk5nFbrhnsWwY0fgW8wfHk/vJIMS9+BklpcFyYiIuJhbA1WhmEMMwzjraysLDvLqJLkuHAKS8rZsD/H7lLOzuGANsPgzvkwajoEN4RZf4D/diBm1xfWlKGIiIi4lK3ByjTNGaZp3hkaGmpnGVWSEneyUahbMwyruehvv4VbvoSIFiRsfQ9ebg8//sfax1BERERcQlOBVdQ4zJ/IYF9S3WEBe0UYBjTrC2Nnkpr8T2vT5+//Bi8nwtznra11REREpFoUrKrIMAxS4sLcf8TqLLJD28DN0+GOuRDfG+b/0xrB+u6vkHvI7vJERETqLAWrakiOC2fH4XwO5xbZXUrVNE6BmyZaC91bDoSFL8PLSfD1E5C91+7qRERE6hwFq2pIiQsHbG4U6goN2sHw96xWDe2ugV/ehP92gJkPw9F0u6sTERGpMxSsqiGpcSheDoMVu+redOBZRbSAa16HB1Oh4yir/9X/UuDz+yBji93ViYiIuD0Fq2rw93HSpmEIqel1fMTq18LjYdjL8FAadLkd1kyH/+sEH14LG2apm7uIiMg5KFhVU0pcGGm7Mykrvwg3Pw5tDENegN+thr6Pw8F1MGWkNU04fxzkHLC7QhEREbeiBqHVlBwXTn5xGRvdtVGoKwRFQf8nrIB1w4cQkQBzn4GX2sK0MbB9AZgXYbAUERGpJDUIraYTC9gvlnVW5+P0hrZXwi1fwP3LodvdsG0eTBgGr3aFn1+HgotsWlRERKQSNBVYTbH1/Kkf6HPxrbO6kIgEGPQs/GEDXPWatSfh14/Dv1vDF/fD3hV2VygiIlLrvOwuoK4zDIPkuHDPGLE6G29/SB5l3fauhGXvwurpsOJDaJQCXX4L7a4FnwC7KxUREalxGrFygeS4MLYdyiMzv9juUuzVqCNc+T9rFGvIi9ZGz1/cB/9pbTUdzdhsd4UiIiI1SsHKBU6us/Kw6cBz8QuFbnfBfb/A2FnQ/DJY8jb8X2eYcCWs+wLKSuyuUkRExOU0FegC7WNCcRiwIv0o/VtF2V2O+zAMiO9l3XIPQuoHsHw8TLsFgqKh0xhIGWO1dRAREbkIaMTKBQJ9vWgdHUJqXd/apiYFRUGfR6ymoyOmQHQSzH/R2ptwyijY8j2Ul9tdpYiISLVoxMpFkuPC+GLlXsrKTZwOw+5y3JfDCa2GWLejO2DZ+9ZC9w0zoV4z6HQrpIwG/3C7KxUREak0jVi5SEpcOLlFpWw5mGt3KXVHeDxc/jf4/Xq49h0IagBz/gJvX2pNHYqIiNQxClYuktLk2AL2nR7adqE6vHyh/fVw29fWYvec/fDRtVBYdzvyi4iIZ9KWNi4SXz+A8ABvUhWsqie+F9z4IRzcAJNHQkmh3RWJiIhUmLa0cZETjUK1gL36EgbANW9A+k8w/TYoK7W7IhERkQrRVKALJceGsflgLlkF6tFUbUnD4TfjYOMsmPGQNnkWEZE6QcHKhY6vs0pTo1DX6HoH9HsCVn4Ec56yuxoREZELUrByofYxoRgGWmflSn0fg653wqJXYOHLdlcjIiJyXgpWLhTs5037xqF8vWY/pqauXMMwYPALkHgdfPc0pH5od0UiIiLnpGDlYiO7xbFhfw6/bD9idykXD4cDrn7D2nNwxoOwfqbdFYmIiJyVgpWLXdWxMWEB3kxYtMPuUi4uXj5WG4bGnawrBbf/aHdFIiIiZ1CwcjE/byc3donl23UH2JtZYHc5FxefQBg5Deo1hckjYF+a3RWJiIicRsGqBozu3gTTNPno53S7S7n4BNSDmz8F/zD48Fo4vNXuikRERE5QsKoBMeEBDGjTgClLd1FYUmZ3ORef0MYw+nPr6w+vhux99tYjIiJyjIJVDRnbM54jecXMXKUP/RoRkQA3T4f8o9a+gvm6WEBEROynYFVDejSvT4uoICYs2qHWCzWlUTKMmASHt8CkG6E4z+6KRETEw2kT5hpiGAa39Ixn9Z4sUrV/YM1p2geuexf2LINpY6BM2wmJiIh9tAlzDbo2uTHBfl5qvVDT2l4JV7wMW+bA5/dAebndFYmIiIfSVGANCvT14vpOscxevY+D2YV2l3Nx6zQGLnsaVn8MXz+uTZtFRMQWClY17JYeTSgzTSb+stPuUi5+vR6GHvfDkjdhwTi7qxEREQ+kYFXD4iMC6dcykklLdlJcqimqGmUYcPk/oMMImPssLH3H7opERMTDKFjVgjE94zmUU8RXa9R6ocY5HHDl/6DlEJj1CKz51O6KRETEgyhY1YI+LSJpGhGoRey1xekN178PcT3g0zthy/d2VyQiIh5CwaoWOBwGo7s3IXVnJqt3X3ytJdyStz+MmAyRrWHqaNi9zO6KRETEAyhY1ZLhnWMI8HEyXqNWtcc/DG7+BIIiYeJwOLTR7opEROQip2BVS0L8vLkuJYYZq/ZyOLfI7nI8R3ADa19Bpw98eA1k7rK7IhERuYgpWNWiMT2bUFxazpSl+nCvVfWaws2fQlGuFa7yMuyuSERELlIKVrUoISqYXgkRfPRzOqVlar1Qq6ITYeQUyNplTQsW5dhdkYiIXIQUrGrZmJ7x7Msq5Nt1B+wuxfM06QnXT4B9q2DKKCjVlKyIiLiWglUtu7R1FDHh/mq9YJdWg+GqV2H7fPjkdjDL7K5IREQuIgpWtcx5rPXCL9uPsH5ftt3leKaOI2DQc7D+S1psfkf7CoqIiMsoWNngxi6x+Hk7+GDxDrtL8Vw97oOeD9B472xY/H92VyMiIhcJW4OVYRjDDMN4KyvLs5pmhgX4cHXHxny2Yg+Z+cV2l+O5Bvydg5GXwLd/hrWf212NiIhcBGwNVqZpzjBN887Q0FA7y7DFmJ7xFJaUM22ZWi/YxuFgQ+vfQWw3a+ubnb/YXZGIiNRxmgq0SZuGIXRtWo8PFqdTVq41PnYpd/rATZMhNAYm3wSHt9pdkoiI1GEKVjYa2zOe3UcL+GHDQbtL8WyB9WHUx2AYVo+rvMN2VyQiInWUgpWNBrZtQMNQP7VecAf1m8OIKZC1xxq5KimwuyIREamDFKxs5OV0MKpbHAu3ZLDloDqB2y62K1z7FuxeCp/dBeXqji8iIpWjYGWzm7rG4eN08MHidLtLEYB2V8PAf8C6L+C7p+yuRkRE6hgFK5tFBPlyRYeGfLJ8NzmFJXaXIwA97ocud8Ci/8GSt+2uRkRE6hAFKzcwtmc8ecVlTF++2+5SBKxF7ENegJZD4Ks/wsav7K5IRETqCAUrN9A+JozkuDA+WJxOuVovuAeHE4a/Cw07wPTbYE+q3RWJiEgdoGDlJsb2jGd7Rh4LNh+yuxQ5zicQRkyFgAiYdCMc1To4ERE5PwUrNzEksSERQb5qveBughvAzdOhrAgmXg8FR+2uSERE3JiClZvw8XIwslsc8zYdYkdGnt3lyKkiW8GNH8GRbTB1NJQW2V2RiIi4KQUrNzKqWxxOw+DDnzXl5Haa9oGrXoUdP8KXD4CptXAiInImBSs30iDEjyFJDZm2bBd5RaV2lyO/1uFG6P9nWDUV5j5rdzUiIuKGFKzczNieTcgpLOWzFXvsLkXOps8jkDwaFoyD1A/trkZERNyMgpWbSYkLJ7FxCB8s3oGp6Sb3YxhwxUvQ/FKY+TvY+oPdFYmIiBtRsHIzhmEwpkc8mw7ksnjrYbvLkbNxesP1EyCyNUy9BfavsbsiERFxEwpWbmhYh0aEB3gzXq0X3JdfCIycBr7BVhuG7L12VyQiIm5AwcoN+Xk7ualrHN+tP8Duo/l2lyPnEtoYRk2DomyYeAMUZttdkYiI2MzWYGUYxjDDMN7Kysqyswy3dHP3JgB89PNOmyuR84pOghsmwMF18PFYKNNG2iIinszWYGWa5gzTNO8MDQ21swy31DjMn4Fto5mydCeFJWV2lyPnkzDAWtC+9XuY9Xv1uBIRscOR7dbswcENtpahqUA3NqZnPJn5JXy5Uut33F6nMdD7D5D6Afz4b7urERHxPKumweZvrH1ebaRg5ca6N6tHqwbBjF+k1gt1wqV/gaTr4Yd/wKqP7a5GRMRzmCakTYb43hAWa2spClZuzDAMxvSMZ92+bJala/Nft2cY1rY3TXrBF/fCjoV2VyQi4hl2LYGj26HDCLsrUbByd1cnNyLEz0utF+oKL1+46SMIj4cpI+HQRrsrEndkmjD+Cpj7nN2ViFwc0iaDlz+0vdLuShSs3F2Ajxc3donl6zX72Z9VaHc5UhH+4TDqY3D6wMThkHvQ7orE3exJtTb0XvoulGlfUJFqKSmEtZ9Cm2FWb0GbKVjVAaO7x1Numkz8Jd3uUqSiwuNh5FTIPQSTboTiPLsrEneycqL1Z34GbJ9naykidd6mr6EwCzrcZHclgIJVnRBXP4BLW0UxeclOikrVeqHOaNwJhr8Le1fAJ7erx5VYSgphzSfQ9irwDYXVn9hdUd0093n46RW7qxB3kDYFgqKhWT+7KwEUrOqMMT3jycgtZvbqfXaXIpXReigMeRE2zobJIzRyJbDpKyjMhE63WlMX62dASYHdVdUtuQfhx3/BvH9aIxXiuXIPwZY50P4GcDjtrgZQsKozeiVE0CwykPGLNOOX8MgAACAASURBVB1Y53S7E6542WogOv4KyMuwuyKx08pJENIYmvaBpOugOAc2f2t3VXXLyklQXgoleVbvIvFcaz6x/i64wdWAxylY1REOh8GYHvGk7cpk5a5Mu8uRyup8K9w40dr65t3LrQ7B4nly9sOW76y1IA4nxPeBwEhYPd3uyuoO04TUCdDkEmjYAZa9r90OPFnaZIhuDw3a2l3JCQpWdch1nWII8vViglov1E2tfwO3fAkFR+HdgbB3pd0VSW1bNQ3Mcugw0vre6QXtroFN32gT74ra8SMc2QYpY6DzbXBwLexeandVYoeD62HfSrcarQIFqzolyNeL4Z1imLlqL4dyiuwuR6oirhvc9o3V72r8UNj6g90VVZ1GCSrHNK0prNhuEJFw8v7E4VBWBBtm2VdbXbJ8PPiFWv2KEoeDT7A1aiWeJ20KGE5IGm53JadRsKpjRvdoQkmZyeQlO+0uRaoqshX89lsIawITr6+ba0TWfg7/aqmrsipj7wo4tP7M365ju0JoHKzRdOAF5R22Fvt3GAHe/uAbZC1aXvupNRIsnqO8zPq3M2EABEXZXc1pFKzqmOaRQfRuEcHEX9IpKSu3uxypqpBGcOtsiO0On94Bi/5nd0UVU5gFn94FH4+xPsh+ellXtFXUykng5WdN/Z3KMKxF7Fvn6sKGC1k1BcqKrWnA4zrfCqWF1uiFeI7tCyBnr9v0rjqVglUdNLZnPAeyi/hm7X6XHM80TfKLS9mbWcD6fdks3nqYr9fsZ+rSnby1YCsvfr2B1+ZtobBEPbRcyj8Mbv4E2l4N3/4Zvv4TlLtxWN6+AF7rCas/hn5PwKhpkH+4bo641bbSIuu8tb7C+u/+a4nDwSyDdZ/Xfm11hWla04AxXU5fqBydZN237D1NT3uStClWH7hWQ+yu5AxedhcgldevVRRx9QKYsGgHV7RvdOL+krJysgtKyCwoITO/5NjXxWTlW/dlFZSQlV/C1t2F/G/9IrKOPS+roJiSsnP/g+TlMCgtN5mZto/Xb06hSf3A2vgxPYO3Hwx/H75uAD+/Crn74erXrTVY7qKkEL7/u1Vf/QT47RyI6WR9iDVIgp9fh5RbrJEXObtNX1u9qzqOPPvjDdpBZGurWWiX22u3trpi58+Qscna6PzXOt1qbXye/hPE96r92qR2FeXC+i8h6XprStjNKFjVQU6HwS09mvDMrPUMfnkBOYWlZBWUkFt0/j3Hgn29CA3wxllmEhPqoEFIEKH+PoT6exMW4G396W/9GRrgTViA9Vigj5O5Gw/y8NQ0rvjfQv51fQcGtYuupZ/WAzgcMOQFCGkI3/0V8g5ZrRn8QuyuDPalWVN/h9ZDlzvg8r+DT4D1mGFA93usD7Rtc6H5pfbW6s5WToLgRufuDG0Y1qjV3GcgazeExtRmdXVD6gTwDTlzKhWs+755wlrErmB18Vs/A0ry3e5qwOMUrOqoG7rEsmT7EcpN87RwdDwgHb8dD0chfl54Oa2Z33nz5tGvX/dKvd+lrRsw84Fe3D8plbs+XM4dvZvyx8Gt8XZqNtklDAN6PWxty/Dl/fD+b+Dm6RBsU4AtL7PWT819HgLqW1OWCQPOfF7ScCsMLn5Nwepccg7A5jlwyYPn7wydeK0VrNZ8Apc8VHv11QUFR2HtZ9aIn89ZRsx9AqwP2aXvQt4LEBhR+zVK7UmbbF38E1e5z7HaomBVR4X4efPWLZ1r9T1j6wUw7e4ePDtrPW//uJ0VOzP5v5EpRIf61WodF7WOI6yGkdNusRqJ3vwpRLSo3RqObIfP7oZdP1vrv654CQLqnf25Xr7W1NW85+DQJohsWbu11gWrp1nrpzqcYxrwuPrNoVGK1SxUwep0qz62Fqh3Gnvu53S6FX55w9rgWufv4pW121rv2fcxt11+oOEGqRRfLyd/vyqRV0Yks25fNkNf+ZGFm3Ulk0u1GABjZ0JxvtVIdPey2nnf44uDX7/Earx37dtw/fhzh6rjOt8GTl/45fXaqLJuOd67KqZLxUJn0nDYvwoyNtd8bXXF8b+XDTtandbPJao1xPW0pgPd+SIQqZ5V0wATOtxodyXnpGAlVXJlh0Z8ef8l1Av0YfR7v/DK95spL9cVOS7TOMXqdeUXYu0vuOmbGn077+JMmHwTzHjIWph+7yKrP1BFfiMMioT218PKyZB/pEbrrHP2pVnbGFV0LUi7awFDW9ycas9yq7t6pzEXfm7n2+Dodtg+v+brktpnmtbVgLHdoV4zu6s5JwUrqbKEqGC+uP8SrurQiP/M2cTY8Us5kldsd1kVUlxazvxNh8gucuMwWL+5dQVeZCuYPAJSP6yZ91k/ky5LH7D6KA3+J4z+ovKLp7vfC6UF1siCnLRykjWal3htxZ4f0tBafL1muloHHLd8PHgHWov7L6TtleBfz2q9IBefvSsgY6Nb9q46lYKVVEuAjxcv3diRZ69J5Oethxn6yo+k7nTfDshFpWVM/CWd/v+ax5j3lvDEwnw+XrYL010/xIKirGnBZn2tRe3zx7nuA7cwGz6/D6aOosg3Eu5aYF3l56jCPwsN2kHTvrDkbSgrcU19dV1p8bHeVUPBP7zir0saDoe3WKNdnq4wG9Z8agXTilwl6+ULyaNg42xrw2u5uKRNsX5RaXe13ZWcl4KVVJthGIzq1oRP7umJl9PghjcW897C7W4VVopKy/hw8Q76j5vHk5+tISrEl5du7EDjIAePTl/Fze/+QvrhPLvLPDvfYBgxFdrfaF01NusP1lV71bHjJ3jjEkibBL0fITXlBWuNSnX0uM/qhLxWTS4B2PwNFByBjqMq97o2V4LDW1vcgHUOSvLOv2j91zrdCuWlsKKGRnjFHqXF1t+HVkMq94uKDRSsxGWSYkKZeX9v+rWK4u8z13HfpFRyCu0dvSgsKWPCoh30fXEef/liLQ3D/Pngtq58ek9PrkmO4fGufjxzdSJpu7IY9PIC3py/lVJ33CrIyweufsO62mnZu9ZVg1XZSqa0COY8ZW0AbTitDaEv+wumw7v6NSZcbjUQ/flVTWOBNQ0YFA3N+1fudQH1IOEya6TG0xdhL58ADRKhcaeKv6Z+c2v0dPkH1f8FRNzHlu+snR7ctHfVqRSsxKVCA7x5+5ZOPDGkNd+sPcCV//cTG/Zn13odhSVlvP/TdvqOm8vTX64lrl4AE2/vxvS7e9CnZSTGsUXZDsPg5u5N+O73fendIpLnv9rAVa/+xJo9WbVe8wU5HFaDzsH/hA2z4MNrKrfx7P418Pal8NN/rRGAuxdaGwC7sr5ud1vrIHb94rrj1kW5B60LDjrceP7eVeeSOByy98DOxa6vra7YuxL2rbT2BazsZfWdb4WsnbDl+5qpTWpf2mQIiLB+6XBztgYrwzCGGYbxVlaWG36ISZUZhsFdfZsz6fZu5BWVcvWrPzF9+e5aee+C4jLeXbid3i/O5W8z1hFfP5BJd3Rj6l3duSQh4kSg+rXoUD/eGt2J10elcDCniKte/YnnZ6+noNgNf+Ptfg8Mf8+6Wuq9wVZfl/MpL7PC1Nv9rQ/8kdNg2MvgG+T62jqOBL8wWHyWbUc8yeqPK9a76lxaDQEvf8+eDkydYG1a3f76yr+21VAIjILl77u+Lql9+UesbaGSrgenC0bXa5itwco0zRmmad4ZGhpqZxlSQ7o1q8+sB3uTHBvOIx+n8dj0VTW2kXN+cSlvL9hG7xfn8o+Z60iIDGLKnd2ZelcPejY/d6A6lWEYDElqyHcP9+WGzjG8uWAbg15ewE9b3LBPV+K1Vjf07L3wzuVwYN3Zn3c0HSYMs6b/WgyEexdDy0E1V5dPoDUatmGm9d6eauUka/qqquvWfIOscLX2c8+8GKA4z2oK2u6aqq2n8fKBlNHWh/GFfvEQ97f2MygrdvurAY/TVKDUqMhgXz66vRv3909g6rJdXPPaInZkuG6ReH5xKW/O30rvF+by7Oz1tIoOYtpdPZh8Z3e6N6tfpWOGBnjz/LXtmXxHd5wOg1Hv/MIjH6eRme9mrSSa9oFbZ4NZDu8PthakH2easGKi1exz3yprY+cbP6qdrT663gEYsOStmn8vd7RvFRxYU/21IEnDrcXv2+a5pKw6Ze1nUJxjTQNWVcoY6/+DmmpTIrUnbQpEtjl/g1g3omAlNc7pMHhkUCveH9uFfVkFDPvfQr5es69ax8wrKuX1eVvp9cJcnv9qA20bhTD97h5MvL07XZteoFN4BfVoXp+vHurNvf2a8/mKPQz4z3xmpO11q6sdiU6C2+dY0x4fXgPrvoS8DJh6s7U5csP2cM9P1hRdbW3/EBpjXQ6d+gEU5dTOe7qTlZPA6QOJ11XvOAkDwC/UM5uFLh8PEa2qtxdceBPrHKZOgLLzb1AvbuzwVti9xBqtctMtbH5NwUpqTf/WUcx8oBfNooK4+6NUnpm5jpJKXoGXU1jCq3O30OuFH3jh6w0kNQ7lk3t68uFvu9E53jWB6lR+3k7+OLg1X97fi0Zh/jwweQW/nbCMPZlVuCKvpoTFWV3aG7a3rhb8vy6w+Vu4/B8wZob1AVPbut8HRdnWqJknKS229gZs9ZsLbwV0IV6+0GaYNa1alStA66oD62D3UqvTenU/SDvfCjn7rNYXUjelTQEMayeIOkLBSmpVTHgAH9/Vg7E943ln4XZueutn9mVd+EMju7CE/32/md4vzmXcNxvpGBvGZ/f2ZMJtXenUpOZ7mrRtFMJn917Cn4e2YfHWwwz8z3wmLNpBmbts4xNQD2750uo8HR4Pd86DSx6s2hVprhDTCWK6WvsHetIl75u/tS4Jr2zvqnNJuh6Kc2t8SyO3kjrBGvFr74L1NC0GQXAjdWKvq8rLYdUUaNYPQhrZXU2FedldgHgeHy8Hf72yHZ2ahPP4J6sY+spC/ntTR3q3iDzjuVkFJYz/aQfvLtxGdmEpA9pE8eBlLWgfE1brdTsdBrf3bsagdtE8+fkanv5yLZ+v3MML17WnZYPgWq/nDD4BcMMHNf42u4/mM23Zbj5fsYeSsnIig32JCPIlMsiXyGDfE9+3bH4LLebfT8HamfglXlmhCwjqvLTJENQAml/qmuPF97aOt2a623ebdomSAuscthkGgVVbI3kapxek3ALzX4CjO6xfOqTu2LkYMndC/z/bXUmlKFiJbYZ1aESbhiHcO3E5t7y3hIcua8EDl7bA6TDIyi/hvZ+2895P28kpLOXytg146LIWJDa2/wrS2HoBTLi1C5+v3MPfZ6xj6Cs/ck+/BO7r3xxfL5tGiGpYSVk5368/wOQlu1iw+RAAvRIiaBDix6GcIvZnFbJmTxYZuUUcH8RzEsp83wh2f/w8Y6f5WKEr6GTwOh7CIoN8iTjlMT/vOnoO8zKsq9C632N9oLuCw2ldGbfsfSjMstZcXczWfWn9nJXptH4hKbfAghetZqMDnnbdcaXmpU229olsc4XdlVSKgpXYKiEqiM/vu4Q/f7aGl7/bzPL0o3SMDWP8TzvIKSplULsGPHhZC9o1cq8PFMMwuCY5hj4tInlm1npe+X4zs1bt5YXr2tfIWi+7bM/IY8rSnXyyfDcZucU0DPXjgUtbcH2nGGLrBZzx/LJyk6P5xRzKKSIjt4ijK8bSfd2/+EPbQtaWR3Mot4jtGXks2X6Eo/lnbyMQ7Od1MmwdC1yNw/y5rlMM9QJ9avpHrrrVH1tbqVS1d9W5JA6HX96A9TOtffAuZqkToF4za6TOVUIbQ8vB1hY3/Z6wWjGI+yspsNqNtL3KauNShyhYie0CfLz49w0d6NK0Hk9/uZYfN2cwJDGaBy9rQZuGFdh41Ub1g3x56caOXNWxEU9+tobhbyzm5u5x/HFwa0L83L+R3dkUlpTxzdr9TF6yk5+3HcHpMLi0dRQjusbSt2UUTse5p/ScDoOIIGtECoCYB2Dz69zh9TVc++Zpzy0pK+dwrhXCDuUWkpFTzKHcIuv7nCIO5Raxfm82C3KKyCkq5ZUfNvPApQnc0iPePUe1Vk6ERsnQoK1rjxvTGcKaWNOBF3OwOrQJ0n+CAX91/dVfnW+zNmbeOMsaART3t2GW1XKjjvSuOpWClbgFwzAY0TWObk3rUW6aJES5wZqlSujXKopvH+7Df+Zs4v2ftvPduoP8/ap2DGwXbXdpFbbpQA6Tl+zksxV7yMwvIbaeP48OasXwTjE0CPGr2kH9wyD5Zmvx8OV/g+CT58Pb6SA61I/oUD/g/COSmw7k8Pzs9Tw3ewMfLE7nscGtuaJ9Q/dZt7V/tXUbMs71xzYMq3XDT/+F3EMQdOZaxItC6gRweLlu4f+pml8KoXHW30MFq7ohbQqExLh29LKW6KpAcSvNIoPqXKg6LtDXi79c0ZZP772EsABv7vxwOfdOXM7BnEK7Szun/OJSpi3dxT8WFzDwpQV89HM6vRIimHh7N+Y/0p/7+idUPVQd1+0ua4ps6TtVPkTLBsG8f2tXPvptN4J8vXhg8gqueW0Ry3YcqV5trrJyMji8raaeNSFpuLVFzrrPa+b4diststbTtPoNBEVd+Oll5aQfzqv4Ju8Op9W+YfsCyNhSzWKlxuUcgK3fWy0WHHUvpmjESsTFOsaGMeOBXry1YBv//X4zCzdn8IeBrejerD5N6ge4xTTW6t1ZTF66ky9X7iW3qJSGgQZ/HtqGa1NqYB1T/ebW9izL3oPefwBv/yofqleLCGY92JtPUnfzr282MvyNxQxJjObxIa1pUt+mdRhlJbBqqvUzVrd31bk0aGd1nl49/Vhn+8rJzC8mwMcLHy83/ZDaMNNqU9HpZKf1snKTvZkFbM/IY8fhPOvPjDx2HM5n15F8SstNgv28uK9/AmN7VmB6OHk0zHve2j9w0LM1/ANJtaz+2NpRog5OA4KClUiN8HY6uK9/AkMSo3ni09U8/eVawJrVaRTqT3xEAPH1A2kacfIWWy8Ab2fNffBlF5bwxcq9TFmyk7V7s/HzdvCbpIaM6BpH7vY0+vduVmPvTfd7rTUuq6ad9uFZFU6HwQ2dY7mifUPeXrCdNxds5bv1BxjdPZ4HL0sgLKCWFydvngP5GTUzhXWqpOvgh2cgcxeExZ73qYUlZSzZfoR5Gw8xf9NBth6ytpGqF+hDgxA/GoT4Eh3iR1SIH9HHvrfu96N+oA+O86yjc6XycpO9WQUELXwHL79GvLQ+mh0Ll7LjcB67jhRQfEoDYX9vJ03qB9A6OpjBidHE1Qvg27X7+edXG/hwcTqPDGrJVR0an7v24AbQeqjVGf/Sv4B3NUdipeakTYFGKRDZyu5KqkTBSqQGNYsMYvId3Vm3L5uth3LZkZHPjsN5bMvIY+aqfWQVnJzKcDoMYsL9aRoReEboahTmf95F4+dimibL048yeckuZq3eS2FJOW0bhvCPq9pxZcfGhPpbC+zn7ajhD9L4XtAgCX5+3br83QVrowJ8vHhoQAtGdI3lP3M2MX7RdqYv38WDl7VgdI8mtdf6YuVECIyEhMtq9n0Sh1vBas0n0Ot3Zzy8PSOP+RsPMn/TIRZvO0xhSTk+Xg66Na3HdZ1iKCk1OZBTyMHsQvZnF7J2bzYZuUX8eocmL4dBVLAvDUL9cBQVMi97LVHHgliDEzdfgit4cUZ5ucn+7EJ2HM478ff/+OhT+pF8osv2scB3Ef8quZ6PftlFfP1AEqKCGNC2AU3rBxJ/7P+BqGDfM9bUjegax6ItGTw7ez0PT03j3YXb+dOQNvRMOMeemJ1uhXVfwPov61Qnb4+yfzUcqKH1irVEwUqkhjkcBomNQ8/ag+toXjHbTkxx5J34eun2I+QVn+xY7uN0EFf/+ChXAE0jgoiPCKBpRCDRIX5nfOAcySvm09TdTF26i80Hcwn0cXJNcgwjusaS1Di09hd9Gwb0uBc+vwe2/uDSEBIV4sc/r2vP2EvieXbWep6ZtZ4PFqfz+JDWDEmMrtmfNe+w1RW9213grOGrQOs1hcadrasDe/2O/OJSft52+Nio1CHSD+cDEF8/gJu6xNG3ZSTdm9XH3+fcAbOkrJyMXKsP2YHsIg5kF3LgWPA6mF3EtsPlbErdTU7hmXvtBfo4TwtaDUL9aBDsh5+3k/Qjx/5OZ+STfiSPwpKTI08+Xg6a1AsgPiKQ/q2juCpjDuZ2B6Pv/RO/b9i00qNlPRMimHF/L75I28O/vtnEyHd+4dLWUTw+pPWZjXub9rXaOSx7X8HKXaVNsS5iqO5emzZSsBKxUXigD50Cfc7Ylsc0TQ7lFJ1YX3IifGXks2DzIYpLz5wiOT66tetoAd+s2U9xWTkdY8N44bokrmjfiEBfm/93T7wO5jxtjVrVwOhO6+gQPvxtN+ZvOsRzs9Zz78RUOjUJ58mhbUiJq6Ftj9ZMh/IS6DCiZo5/CtM0yYi/gsif/sofX/+Yz3cHUVxWjp+3g57NI/htr6b0aRFJfETF15p5Ox00DPWnYejZ173NmzePfv36kV9cyoFsK4AdzCk8GcRyCjmQVcjynUc5kFV0YurOx+kgtp41+tq7RcSJUaf4iEAahvidDE9lJfCfGdByMA0aV30q2uGw+soNSWzI+EU7eHXuFga/vIAbOsfy+8tbEnX8AgyHw2o+OucpOLgeotpU+T2lBpSVWuurWgxyTed9myhYibghwzCIOrYGpluz0/+BKS832ZddyPZDeWw/bAWu7Rl5bNyfw5x1Bwj09WJktzhu6hpL62g36gPm5Qtdbod5z8GhjTW2fqJvy0h6JUTw8bJd/HvOJq59bRFD2zfk8cGtz9rUtFpWToSGHSA60bXHPSansIRFW61RqQWbDlGcGcnPvgZJR+cQ0uN39G0VSZf4ejV+QUSAjxdNI7xoep7QZpommfkl5JeUER3iV7Gp641fQd5BSKneurvj/Lyd3N23OTd0juV/P2zmo5/T+WLlXu7s04w7+zSzfrnoOMqaUl0+Hoa84JL3FRfZNg9yD9TZRevHKViJ1DEOh0HjMH8ah/nTq8Xpa0lKj40YeNXgIvhq6Xwb/Phva9Rq2Ms19jZOh8FNXeMY1qERby7YxlsLtjJn7QHGXhLPff0SCA1wwbTdgbWwLw2GvFj9Yx1jmibr9+Uwf9Mh5m08yPL0o5SWmwT6OLkkIYJ+/RMoWX0Jo/OXwtA2rm+kWQ2GYRAe6EOlxgZTJ1ibJCcMcGkt9QJ9eHpYO8b2jOfFrzfy3+83M2nJTh4e0JIbOsfg1eZKq0XGZU9be2yKe0ibDH5h0HKQ3ZVUi4KVyEXEbQPVcUGR1tqWtClw2VM1157gmEBfL35/eUtGdo3j399u5O0ftzFt2S4euqwFo7o1qV77gZWTrN5VidXrXZWVX8LCLRnMO7bw/GBOEQBtGoZwe+9m9GsVSUpc+MlavW6AGQ/C3hXQOKVa722rzJ2w5Xvo+0fX7a34K03qB/LqqBRuSz/Kc7PX86fPVvPeT9t5odM1dFozHdZ+dnF3s69LCrOtthsdR1mj23WYgpWI1K7u91j7ti1/3+prVQuiQ/0Yd30Hxl4Sz3Oz1/O3GeuYsGgHjw9pw6B2DSq/wL2sxGod0bJia0FM0+RIXjH7sqz1SfuyCtiTWciyHUdI3XmUchNC/Lzo3TKSvsdu52zM2vZKmPUH6+rAuhysUj+0/ky+ucbfqlOTcKbf3YNv1h7gha83cN1XsDAojnqL3iZAwco9rPsCSgtrZb1iTVOwEpHa1aAdNOsHS96GHg/U6qa47RqF8tFvuzFv4yGenb2euz9aTtf4ejw5tA0dYsMqfqAt31trgzqOwjRNDucVHwtMVmg6HqD2ZhawP9u6/9QLDsBqa9CmYQj39U+gX6tIOsSEVWzE0T/cmjpb8ylc/o862ZmaslJY8ZF1EUNYXK28pWEYDE6M5rI2UUxespPJ317Go4fe58Xx0xhx5RWuX38nlZM2Beo1t/bGrOMUrESk9nW/DyZdb/2W2v76Wn1rwzDo3zqK3i0imLpsFy/N2cRVr/7EVR0b8eig0xfUl5efGpoKjgWnQgave5WmjlCu/sKbPdlfn9bIEsDbadAgxI+GoX60jwljcDtrT8SGoX7HrsLzo36Qb5V6kwHWFjebvoKdi6weYXXNlu8gZy/8xnXr0yrK2+nglh7xZLd5kpL/TiJ221Qu+3ewa9ffSeUcTYf0hdD/z261brCqFKxEpPYlDID6LeDnV62QYMM/pl5OB6O6NeHKDo14c/423v5xG1+t2U9iPYPXNixmX3bBaS0Ejot05vJ770XMCRhGUmwEA48FpuhQfxqFWQEqItC3ZruXtxoC3gHWFjd1MVgtHw+BUdBysG0lhIRFQvvruHHdF6xu9siJ9Xf390+o3QazYk2rw0XTW0zBSkRqn8MB3e+21grt/Bma9LCtlGA/bx4Z1IqR3eL497eb+HHDHuIDISUunOhQPxqF+p822lR/3QQcX5Uy9JY/MDQ6yZ6ifQKtDYvXfW5dlViL06nVlr0XNn8DlzxU801VL6TzbTjSJvFciw2M6nst//xqA8/MWs+ExTt4bHBrhiY1rP1mup7GNK2rAZv0gvAmdlfjEgpWImKPDiPg+3/Az6/ZGqyOaxTmz79v6MC8eUfp1+889aRNgugk62anpOFWg9Jtc+vW5ekrJlob7KbcYncl1nqeBomw7D3a3TX2RIPZ52ev5/5JK3gndjtPDm1Dl/iavXrVo+1eBke2Qq+H7a7EZergqkcRuSj4BFpdsDfMhKM77K6mYg6ss9oc1PSGyxXR/DKr58/q6XZXUnHl5ZD6wcmtZexmGND5Vti/CvamAlaD2VkP9ubF4e3Zl1XA9W8s5o4PljEnvYTPV+xh3saDrNyVSfrhPLLySygvNy/wJnJeaZPByw/aXmV3JS6jESsRsU/XO2Hx/1lXCA561u5qLixtkrWPWVLtLrg/Ky8fq/XC6k+gOL9uNLrc9gNk7YTL/2p3JScl3QDfPgXL3oPGnQCrwewNUcZ5rAAAFMxJREFUnWMZ1r4R7y7cxpvzt5FTVMrE9SvPeLnDgFB/b8ICfAgL8CbM35vwAB9CA6w/wwKOPeZ/6vfeBPl6aZqxtMhqG9L6CvBzo10iqknBSkTsE9oY2l5tjWL0exx8gy/8GruUlVqLbFsMgsCICz+/NiQOt87dpq8h8Vq7q7mw5RMgoL71Qeou/EKsadXVH8PAZ8H/ZNsNfx8n91/agrv7Nuer7+fTNrkLmfklZBUUczSvhMyCEjLzi8nML+FofjFZBSUcyi1i88FcMvNLyC06c/Pq47wcBmEB3oT6nx7Agv288HE68HIaOB0OvBwGXk7D+tPhOPa1db/zxGOOE89xOgy8nY5jf579GMefExbgXePbIZ3Xpm+gMPOi6F11KgUrEbFX93uttUIrPrKah7qrrT9Y+5h1HGl3JSfF94KgaOu3fncPVrkHYeNs6Ha3+3XW7nyrtb3OqmnQ7c4zHvZyOgj2MWgeGVSpw5aUlZN1Wvg6+XVmQTFH80vIOvb13sxC1u/LIbughJLycsrKTUrKanaa0d/bSf/WkQxObEj/VpEE+9XyxQRpUyCogdXX7iKiYCUi9orpBLHd4Jc3rKlBh5te5r5yojXa0mKg3ZWc5HBCu2tg2btQkHnaaIvbWTkRyktdtuGySzVKtm7L34eud7is/Ye300FEkC8RQVULkqZpUm5aAa2s3KS0zKS0vJzSctO6lR37+vj9ZSfv///27jw6qipP4Pj3l5UQQhIIJJEAAUS2sDTEgLIYWnQCKraKPfR4WpSeYWzbMzKtZ/SMPXbby4zd2j3dzjD2oNLaaoO7jYqgDdK4DLssAQOyCmEJQUjYEkhy54/7ihRFFSSkql4tv88571TVe6/ybt3cqvrVvff9XqNnH+/1Ps/ZvL+GRZsOsmDjAVISExjbN4eyojyuG5hLVvsQn2l64rA9O3TkPSG7pJFbYuvVKKWi06h74bVpsOV9GBBBw0QeJ7+2vS3F0yMvtcHgKbDiaXsSQBguD3NJPJPWe46GLle4XRr/Rtxtr8G4ZwX0GOV2aQCbzDZRIDGEPzYem1zE2q+OsLD8AAvLD7C4oorEBOGq3p0pK8rj+kG5dM0IcHmltih/wwbaMTYMCHpWoFIqEvS/ETJ72NQLkWjTm9B4OrKGAT26jYDsQjtHKFLt+hi+3hGZvVUeRbdBakdY/Qe3SxJWiQnClYWd+LcbB/LJQ+OZf99oZozrTeXRU/zo7XJG/vtibv/9Zzz3yU4qj54K3oHXz4XcwZBXFLy/GSE0sFJKuS8xyc5t2f0p7Dv/zCvXrfuTzXeUN8TtkpxPxE5i37kMjh10uzT+rX3BpoYYONntkgSW2sFm/t70lu2hjEMiwpCCLB4q68+SB65h0cxx3H9tX47VNfCzdzcz+vElPPbZKf5n6TZ2Vp+49AMd2mLTWwydGrzCRxANrJRSkWH4nZDSAZY/7XZJzlVVAZVrbG9VpJ4eP3iKTbq5+W23S3K+E4fhi3fsl2hymtulubARd0Njve1NiXMiQr+8DGZOuIKFM8fx0YOlPFTWHwR+tXAL459cStlvl/GfH26l4kAtxrRiov36eSAJkZG2JAR0jpVSKjK0y7SJN1fPgeseg4w8t0tkrf8TSGJkfwl0HQBdB9lkoSP/0e3SnGv9XDuMGsnDgB55RVBQYocDR90buYG0C3rlpPP90j4MYA99h41kkTMn66klX/K7xV/SKyedsqI8ygblMaQgM3COrqYm2PCKTXCbkXvJ5THGUFvXcE66C8/t5KGX0fkSTxgIBg2slFKRY9Q9sHI2rHoWvvkjt0sDTY1O7qrroUNXt0tzYYNvg8U/hSO7I+eaa8bYYcCCEsgd6HZpWqZ4Orx9D+z6BHqNdbs0EalbVhrTx/Ri+pheVB2r48PNB1lYfoDZy3bw9NLtdMtK428G5VFWlMeIntkkel+QfNfHUFsJ1/307Kq6M40cOenkBjt5mqOnvAKlE6fP5gs74skXdtLmEGsMkPV+SEGmBlZKKQXYy5z0m2R7rcY+4P7Q0faP4Nh+e6HjSFfkBFblb8DYH7pdGuur5VC9FW6e5XZJWm7Qt2Dhw7YNamB1UV0z2nHHyJ7cMbInR0+e5sPNB1m06QAvrdjNnE93ktMhlesH5ZKVlsyRk2eYvHMWQ6U9U/+SSdU7izly8jT1DU0B/35acuLZ5KnZ7ZMZkNfRyWrvSazqZLVP9+yTQmaauxf31sBKKRVZRn0ftrxnhwtG3OVuWda9DGmd4Ioyd8vREtmFUHBlZAVWa563Z9oNusXtkrRccpqdT7fyGTh+CDp0cbtEUSOrfQq3F3fn9uLuHK9vYElFFYvKD/D255XUNzSRn9bIo41/5dN219C1Uzb92ieTnZ5yNvt8dvvmSwN5stG7mhn+EmlgpZSKLIVjIG+wncQ+fJp781xOHYGK92DEtMjLXRVI0RRY+JCdcN+1v7tlOXXETqYfdoe94HY0GXGXTf2x7iUY889ulyYqdUhNYvLQy5g89DIamwwJArLhVXirjglT72dCYbHbRQwZPStQKRVZRGDUD+BQhb2MjFvK37RniEVi7qpABt1iz7Yqf93tksCG16Chzgam0aZLP+g5xva4NQUeplItk5ggdjL7+rmQ1QN6XOV2kUJKAyulVOQpuhXSu7qbMHT9XOg6EPKHuVeG1srIhV7jbLLQ1pz+HmzG2KAkfxjkD3WvHG1RfDcc2QU7PnK7JMHT1AR7VsFfHoNZo+DX/eGlKfZx+ZtQvS10gWTtPtixFIZMhYTYDj10KFApFXmSUu012z76hU0m2KVfeI9/aCvsXQXX/zz6TrkvmgLz74PKtfY6jG6oXANVm+DG37pz/GAYcJO9NuTqOXD5tW6X5tKdOQU7/movybR1ob2QuCRCz6tt0Huw3AaPTQ12/+R0yB0E+UPskHzeYPsDo60nkmx4FTAxmxTUmwZWSqnIVDwdlj1p51rdFOYv6LO5q74d3uMGw4Cb4L0f2uFAtwKrNc/bL+jBU9w5fjAkpdr5Yf83C2r3u12a1vFc4LjiPTucfuakTb57+QTofwP0vQ7Sspv3b6i3Q+8HNjYvG161aU/ADi/nXAF5g+l+oj1sb7JXIUjPaVl5jLE9wAUl0LlP8F9vhNHASikVmdJz7CVG1s+Dax+F9p3Cc1zTaI95+YQ2JTB0TVoWXH6dHdq5/ucQwgv4+lVXa89MLLoNUjPCe+xgG3EXfPYUfP4iUOJ2aS7s8HbbK1WxAPYst5n4My6zFznuPwkKx9pg0Z+kVNt75T1sawwc3X1usPXVcvrU7IEdL9h9MvKbe7XynB6u7F7nD/XtX28Dtxt+E5rXHmGCHliJSG/gESDTGBPFP1eUUq4bda/9UlvzB5vXKgyyj2ywuavKHg/L8UJi8G02ZcXuT+2cq3Aqf932kIy4O7zHDYXOfaD3eFjzAnzDpd6/QJqaoHJ1czBVvcWuzy2CsQ/aYCp/2KUPZYvYFB7ZhbYX1PHJh/MZ0yfz3IBr22L7gwRsz1hukVfANRg+fwkSU6Ir7UYbtCiwEpE5wI1AlTGmyGt9GfA7IBF41hjzuDFmB/A9EYmA01KUUlEtd6D9Ylv5DHTua3tj2mXaC/qmZUFKRtAnwuYdWGL/fr+JQf27YXXFRDsUt/H18AdWa563X6zdhof3uKFSfDe8eifD1j0Kx96GjgWQWQCZ3Zz73cKXyPbsfKn3YMtCOFFlh6wLR9uh834TQ551vyG5I/S+xi5ny1V3/lDi+nmw6pnmfQZMDl+vs8ta2mP1PPDfwB89K0QkEZgFXAfsBVaJyHxjzOZgF1IpFcdG/xO8eAu8+t3zt0mCE2h5BVvtnODLc9/vumybuDLR5yPw1FFyqpfDiDsDD5tEg5T2tsdi859h0pPhycN1pg6+/MAO+0x8Ivom/QfSbxKUzCBx82LY8j6cOHT+PmmdnGCrADp28wq6nMArIx8SLzEb+Ilq2LrI9kydnS+VAX0nQL8b7K33fCk3JLeDy4bZxaOpqXko8VAFDLrVvfKFWYsCK2PMMhEp9FldAmxzeqgQkXnAzYAGVkqp4OnzTXhgCxyvgrqjUFcDp47a+6ecx2fvH7WndXvuN56+8N9OyTg3AGuoI7HpdHTlrgqkaIpNu7B9CfQLQeb441WwZ4WzrIR9n9v6bp8DQyL4gtWtlZgMk55gTfullJaW2gDy2D6o2Qs1lVDrua206Rl2fQr1NT5/ROxFxTt2CxCAdbPpRTy9r9XbbCC1ZYGtX9Nk9x/2dzbQKxwb+UlrExKgUy+7MNnt0oSVmBbmOnECq3c9Q4EiMgUoM8b8vfP4u8BI4MfAL7A9Wc8aY/4jwN+bAcwAyM3NHTFv3rw2vZCLOX78OB06dAjpMaKF1kUzrQsrJuvBGBKaTpPUcJykhhMkn7G39nHz/eQz566rTcll65B/jfoeF2k6w9Wf3cXXnYbzxcBLm592tl2YRtJP7CGz5gs61laQWVNBWt0BAJokiWMZl1OTOYDajv05mjXQDhfFmNa8RxIbTpJaX027umpS66tJrT/k87jaBvBemiSJ+tTOgJyt22MdenG4cwnVOSM53qF3RLTJmPysuATjx49fY4zxmz6+LZPX/f2HjTHmMHDPxZ5sjJkNzAYoLi42paWlbSjKxS1d6vzaUFoXXrQuLK2HZqtjqS5OTCF3w6vkXn1l6y4rU38M9q5m58fv0uvIQdi7Gupr7bb0rlBYAj1GQfeRJOQPJTMplczQvIKIEdT3iDFw8uvm3q6avSTU7iWtptIO9fUuhX4TycjqQQZQGJyjBoV+VlxcWwKrvUB3r8cFwL62FUcppVTQFE2xk8m3vB84p5TntPo9K+Gr5fa2ahOYJgoRmyxy8O3QfSR0L7FniUVAz0lUE4H0znaJ1sz0KqC2BFargL4i0guoBKYCMTAxQSmlYkTPq+3E6fI3mgOrhnrYv8GZG+UEUscP2m0pGVBQDOP+BXqM5JOddYydMMm98isVhVqabmEuUArkiMhe4MfGmOdE5D5gETbdwhxjzKaQlVQppVTrJCTas7FWzoYPfmSH9CrX2otLg+196l1qe6K6j4KuA85JKNq4Z6kLhVYqurX0rMDvBFi/AFgQ1BIppZQKnqF/ay9mveJ/bcLIkn+w86MKSqIzs7xSEU4vaaOUUrEsfyjM3AjpXWy+IaVUSAU3ZXErichNIjK7psY354dSSqmgyequQZVSYeJqYGWMeccYMyMzM9ZP1FVKKaVUPHA1sFJKKaWUiiUaWCmllFJKBYkGVkoppZRSQaKBlVJKKaVUkGhgpZRSSikVJBpYKaWUUkoFiQZWSimllFJBoglClVJKKaWCRBOEKqWUUkoFiRhj3C4DInII2B3iw+QA1SE+RrTQumimdWFpPTTTumimddFM68LSerB6GmO6+NsQEYFVOIjIamNMsdvliARaF820Liyth2ZaF820LpppXVhaDxenk9eVUkoppYJEAyullFJKqSCJp8BqttsFiCBaF820Liyth2ZaF820LpppXVhaDxcRN3OslFJKKaVCLZ56rJRSSimlQirmAisRKRORLSKyTUQe9rNdROQpZ/sGERnuRjlDTUS6i8hHIvKFiGwSkfv97FMqIjUiss5ZHnWjrKEmIrtEZKPzGlf72R4vbaKf1/96nYjUishMn31itk2IyBwRqRKRcq91nUTkQxH50rnNDvDcC36uRJsAdfGEiFQ474G3RCQrwHMv+H6KNgHq4iciUun1PpgU4Lkx0y4C1MMrXnWwS0TWBXhuTLWJNjPGxMwCJALbgd5ACrAeGOizzyTgfUCAUcAKt8sdorrIB4Y79zOArX7qohR41+2yhqEudgE5F9geF23C5zUnAgewuVjiok0A44DhQLnXul8BDzv3HwZ+GaCuLvi5Em1LgLq4Hkhy7v/SX1042y74foq2JUBd/AR48CLPi6l24a8efLb/Gng0HtpEW5dY67EqAbYZY3YYY04D84Cbffa5GfijsZYDWSKSH+6ChpoxZr8xZq1z/xjwBdDN3VJFrLhoEz6uBbYbY0KdmDdiGGOWAV/7rL4ZeMG5/wLwLT9PbcnnSlTxVxfGmA+MMQ3Ow+VAQdgL5oIA7aIlYqpdXKgeRESAbwNzw1qoKBVrgVU3YI/X472cH0y0ZJ+YIiKFwDeAFX42XyUi60XkfREZFNaChY8BPhCRNSIyw8/2uGsTwFQCf0jGQ5vwyDXG7Af7YwTo6mefeGwf07G9uP5c7P0UK+5zhkXnBBgijqd2MRY4aIz5MsD2eGkTLRJrgZX4Wed72mNL9okZItIBeAOYaYyp9dm8FjsUNBT4L+DtcJcvTEYbY4YDE4EfiMg4n+3x1iZSgMnAa342x0ubaI14ax+PAA3AywF2udj7KRY8DfQBhgH7scNgvuKpXXyHC/dWxUObaLFYC6z2At29HhcA+y5hn5ggIsnYoOplY8ybvtuNMbXGmOPO/QVAsojkhLmYIWeM2efcVgFvYbvwvcVNm3BMBNYaYw76boiXNuHloGfY17mt8rNP3LQPEZkG3AjcYZzJM75a8H6KesaYg8aYRmNME/AM/l9jXLQLEUkCbgVeCbRPPLSJ1oi1wGoV0FdEejm/yqcC8332mQ/c6ZwJNgqo8QwFxBJnTPw54AtjzG8C7JPn7IeIlGDbw+HwlTL0RCRdRDI897ETdMt9douLNuEl4K/PeGgTPuYD05z704A/+9mnJZ8rUU9EyoCHgMnGmJMB9mnJ+ynq+cyxvAX/rzEu2gUwAagwxuz1tzFe2kSruD17PtgL9gyvrdizNR5x1t0D3OPcF2CWs30jUOx2mUNUD2Ow3dIbgHXOMsmnLu4DNmHPZlkOXO12uUNQD72d17feea1x2yac19oeGyhleq2LizaBDSb3A2ewvQ3fAzoDi4EvndtOzr6XAQu8nnve50o0LwHqYht2zpDn8+L3vnUR6P0UzUuAunjR+SzYgA2W8mO9XfirB2f9857PB699Y7pNtHXRzOtKKaWUUkESa0OBSimllFKu0cBKKaWUUipINLBSSimllAoSDayUUkoppYJEAyullFJKqSDRwEoppZRSKkg0sFJKKaWUChINrJRSSimlguT/AVK5thlYjSioAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(dpi=300)\n",
    "pd.DataFrame(history.history).plot(figsize=(10, 5),logy=True)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}